diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 4c45b8f80d23..9d8b5c5077c3 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -18,6 +18,10 @@ acceptedBreaks: - code: "java.method.removed" old: "method org.apache.iceberg.RowDelta org.apache.iceberg.RowDelta::validateNoConflictingAppends(org.apache.iceberg.expressions.Expression)" justification: "Deprecations for 1.0 release" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.PartitionKey" + new: "class org.apache.iceberg.PartitionKey" + justification: "Serialization across versions is not supported" release-base-0.13.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/api/src/main/java/org/apache/iceberg/PartitionKey.java b/api/src/main/java/org/apache/iceberg/PartitionKey.java index 0f696b59c477..fc56d1a45347 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionKey.java +++ b/api/src/main/java/org/apache/iceberg/PartitionKey.java @@ -22,8 +22,9 @@ import java.lang.reflect.Array; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.util.SerializableFunction; /** * A struct of partition values. @@ -36,7 +37,7 @@ public class PartitionKey implements StructLike, Serializable { private final PartitionSpec spec; private final int size; private final Object[] partitionTuple; - private final Transform[] transforms; + private final SerializableFunction[] transforms; private final Accessor[] accessors; @SuppressWarnings("unchecked") @@ -46,7 +47,7 @@ public PartitionKey(PartitionSpec spec, Schema inputSchema) { List fields = spec.fields(); this.size = fields.size(); this.partitionTuple = new Object[size]; - this.transforms = new Transform[size]; + this.transforms = new SerializableFunction[size]; this.accessors = (Accessor[]) Array.newInstance(Accessor.class, size); Schema schema = spec.schema(); @@ -57,7 +58,7 @@ public PartitionKey(PartitionSpec spec, Schema inputSchema) { accessor != null, "Cannot build accessor for field: " + schema.findField(field.sourceId())); this.accessors[i] = accessor; - this.transforms[i] = field.transform(); + this.transforms[i] = field.transform().bind(accessor.type()); } } @@ -101,7 +102,7 @@ public String toPath() { @SuppressWarnings("unchecked") public void partition(StructLike row) { for (int i = 0; i < partitionTuple.length; i += 1) { - Transform transform = transforms[i]; + Function transform = transforms[i]; partitionTuple[i] = transform.apply(accessors[i].get(row)); } } diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index e984fc69d8ce..20f5795f6886 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -59,6 +59,7 @@ public class PartitionSpec implements Serializable { private final PartitionField[] fields; private transient volatile ListMultimap fieldsBySourceId = null; private transient volatile Class[] lazyJavaClasses = null; + private transient volatile StructType lazyPartitionType = null; private transient volatile List fieldList = null; private final int lastAssignedFieldId; @@ -123,16 +124,23 @@ public List getFieldsBySourceId(int fieldId) { /** Returns a {@link StructType} for partition data defined by this spec. */ public StructType partitionType() { - List structFields = Lists.newArrayListWithExpectedSize(fields.length); + if (lazyPartitionType == null) { + synchronized (this) { + if (lazyPartitionType == null) { + List structFields = Lists.newArrayListWithExpectedSize(fields.length); - for (int i = 0; i < fields.length; i += 1) { - PartitionField field = fields[i]; - Type sourceType = schema.findType(field.sourceId()); - Type resultType = field.transform().getResultType(sourceType); - structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType)); + for (PartitionField field : fields) { + Type sourceType = schema.findType(field.sourceId()); + Type resultType = field.transform().getResultType(sourceType); + structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType)); + } + + this.lazyPartitionType = Types.StructType.of(structFields); + } + } } - return Types.StructType.of(structFields); + return lazyPartitionType; } public Class[] javaClasses() { @@ -175,9 +183,11 @@ private String escape(String string) { public String partitionToPath(StructLike data) { StringBuilder sb = new StringBuilder(); Class[] javaClasses = javaClasses(); + List outputFields = partitionType().fields(); for (int i = 0; i < javaClasses.length; i += 1) { PartitionField field = fields[i]; - String valueString = field.transform().toHumanString(get(data, i, javaClasses[i])); + Type type = outputFields.get(i).type(); + String valueString = field.transform().toHumanString(type, get(data, i, javaClasses[i])); if (i > 0) { sb.append("/"); @@ -412,10 +422,7 @@ Builder identity(String sourceName, String targetName) { checkAndAddPartitionName(targetName, sourceColumn.fieldId()); PartitionField field = new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.identity(sourceColumn.type())); + sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.identity()); checkForRedundantPartitions(field); fields.add(field); return this; @@ -429,11 +436,7 @@ public Builder year(String sourceName, String targetName) { checkAndAddPartitionName(targetName); Types.NestedField sourceColumn = findSourceColumn(sourceName); PartitionField field = - new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.year(sourceColumn.type())); + new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.year()); checkForRedundantPartitions(field); fields.add(field); return this; @@ -447,11 +450,7 @@ public Builder month(String sourceName, String targetName) { checkAndAddPartitionName(targetName); Types.NestedField sourceColumn = findSourceColumn(sourceName); PartitionField field = - new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.month(sourceColumn.type())); + new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.month()); checkForRedundantPartitions(field); fields.add(field); return this; @@ -465,11 +464,7 @@ public Builder day(String sourceName, String targetName) { checkAndAddPartitionName(targetName); Types.NestedField sourceColumn = findSourceColumn(sourceName); PartitionField field = - new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.day(sourceColumn.type())); + new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.day()); checkForRedundantPartitions(field); fields.add(field); return this; @@ -483,11 +478,7 @@ public Builder hour(String sourceName, String targetName) { checkAndAddPartitionName(targetName); Types.NestedField sourceColumn = findSourceColumn(sourceName); PartitionField field = - new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.hour(sourceColumn.type())); + new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.hour()); checkForRedundantPartitions(field); fields.add(field); return this; @@ -502,10 +493,7 @@ public Builder bucket(String sourceName, int numBuckets, String targetName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add( new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.bucket(sourceColumn.type(), numBuckets))); + sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.bucket(numBuckets))); return this; } @@ -518,10 +506,7 @@ public Builder truncate(String sourceName, int width, String targetName) { Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add( new PartitionField( - sourceColumn.fieldId(), - nextFieldId(), - targetName, - Transforms.truncate(sourceColumn.type(), width))); + sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.truncate(width))); return this; } @@ -545,16 +530,10 @@ public Builder alwaysNull(String sourceName) { // add a partition field with an auto-increment partition field id starting from // PARTITION_DATA_ID_START - Builder add(int sourceId, String name, String transform) { + Builder add(int sourceId, String name, Transform transform) { return add(sourceId, nextFieldId(), name, transform); } - Builder add(int sourceId, int fieldId, String name, String transform) { - Types.NestedField column = schema.findField(sourceId); - Preconditions.checkNotNull(column, "Cannot find source column: %s", sourceId); - return add(sourceId, fieldId, name, Transforms.fromString(column.type(), transform)); - } - Builder add(int sourceId, int fieldId, String name, Transform transform) { checkAndAddPartitionName(name, sourceId); fields.add(new PartitionField(sourceId, fieldId, name, transform)); diff --git a/api/src/main/java/org/apache/iceberg/SortOrder.java b/api/src/main/java/org/apache/iceberg/SortOrder.java index ee02318de6fd..43da6057b5bc 100644 --- a/api/src/main/java/org/apache/iceberg/SortOrder.java +++ b/api/src/main/java/org/apache/iceberg/SortOrder.java @@ -36,7 +36,6 @@ import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; /** A sort order that defines how data and delete files should be ordered in a table. */ public class SortOrder implements Serializable { @@ -253,14 +252,6 @@ private Builder addSortField(Term term, SortDirection direction, NullOrder nullO return this; } - Builder addSortField( - String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) { - Types.NestedField column = schema.findField(sourceId); - ValidationException.check(column != null, "Cannot find source column: %s", sourceId); - Transform transform = Transforms.fromString(column.type(), transformAsString); - return addSortField(transform, sourceId, direction, nullOrder); - } - Builder addSortField( Transform transform, int sourceId, SortDirection direction, NullOrder nullOrder) { SortField sortField = new SortField(transform, sourceId, direction, nullOrder); @@ -293,7 +284,7 @@ SortOrder buildUnchecked() { private Transform toTransform(BoundTerm term) { if (term instanceof BoundReference) { - return Transforms.identity(term.type()); + return Transforms.identity(); } else if (term instanceof BoundTransform) { return ((BoundTransform) term).transform(); } else { diff --git a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java index 530d3d442c58..27ed7388820e 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java @@ -20,6 +20,8 @@ import java.util.List; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.Transforms; public class UnboundPartitionSpec { @@ -52,9 +54,9 @@ private PartitionSpec.Builder copyToBuilder(Schema schema) { for (UnboundPartitionField field : fields) { if (field.partitionId != null) { - builder.add(field.sourceId, field.partitionId, field.name, field.transformAsString); + builder.add(field.sourceId, field.partitionId, field.name, field.transform); } else { - builder.add(field.sourceId, field.name, field.transformAsString); + builder.add(field.sourceId, field.name, field.transform); } } @@ -94,13 +96,17 @@ UnboundPartitionSpec build() { } static class UnboundPartitionField { - private final String transformAsString; + private final Transform transform; private final int sourceId; private final Integer partitionId; private final String name; + public Transform transform() { + return transform; + } + public String transformAsString() { - return transformAsString; + return transform.toString(); } public int sourceId() { @@ -117,7 +123,7 @@ public String name() { private UnboundPartitionField( String transformAsString, int sourceId, Integer partitionId, String name) { - this.transformAsString = transformAsString; + this.transform = Transforms.fromString(transformAsString); this.sourceId = sourceId; this.partitionId = partitionId; this.name = name; diff --git a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java index 1181b665f87c..c9d7d0b2e89e 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java +++ b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java @@ -21,6 +21,8 @@ import java.util.Collections; import java.util.List; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.Transforms; public class UnboundSortOrder { private static final UnboundSortOrder UNSORTED_ORDER = @@ -38,8 +40,7 @@ public SortOrder bind(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - builder.addSortField( - field.transformAsString, field.sourceId, field.direction, field.nullOrder); + builder.addSortField(field.transform, field.sourceId, field.direction, field.nullOrder); } return builder.build(); @@ -49,8 +50,7 @@ SortOrder bindUnchecked(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - builder.addSortField( - field.transformAsString, field.sourceId, field.direction, field.nullOrder); + builder.addSortField(field.transform, field.sourceId, field.direction, field.nullOrder); } return builder.buildUnchecked(); @@ -114,21 +114,21 @@ UnboundSortOrder build() { } static class UnboundSortField { - private final String transformAsString; + private final Transform transform; private final int sourceId; private final SortDirection direction; private final NullOrder nullOrder; private UnboundSortField( String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) { - this.transformAsString = transformAsString; + this.transform = Transforms.fromString(transformAsString); this.sourceId = sourceId; this.direction = direction; this.nullOrder = nullOrder; } public String transformAsString() { - return transformAsString; + return transform.toString(); } public int sourceId() { diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java b/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java index 32f91018c604..22271aaed9d5 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java +++ b/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java @@ -21,6 +21,7 @@ import org.apache.iceberg.StructLike; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Type; +import org.apache.iceberg.util.SerializableFunction; /** * A transform expression. @@ -31,15 +32,17 @@ public class BoundTransform implements BoundTerm { private final BoundReference ref; private final Transform transform; + private final SerializableFunction func; BoundTransform(BoundReference ref, Transform transform) { this.ref = ref; this.transform = transform; + this.func = transform.bind(ref.type()); } @Override public T eval(StructLike struct) { - return transform.apply(ref.eval(struct)); + return func.apply(ref.eval(struct)); } @Override diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java index fe180a4f0506..9c3bec48b66f 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java +++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java @@ -18,17 +18,17 @@ */ package org.apache.iceberg.expressions; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; /** Expression utility methods. */ public class ExpressionUtil { - private static final Transform HASH_FUNC = - Transforms.bucket(Types.StringType.get(), Integer.MAX_VALUE); + private static final Function HASH_FUNC = + Transforms.bucket(Integer.MAX_VALUE).bind(Types.StringType.get()); private static final Pattern DATE = Pattern.compile("\\d\\d\\d\\d-\\d\\d-\\d\\d"); private static final Pattern TIME = Pattern.compile("\\d\\d:\\d\\d(:\\d\\d(.\\d{1,6})?)?"); private static final Pattern TIMESTAMP = diff --git a/api/src/main/java/org/apache/iceberg/expressions/Expressions.java b/api/src/main/java/org/apache/iceberg/expressions/Expressions.java index 63212fa842b5..7fad8324c4ae 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/Expressions.java +++ b/api/src/main/java/org/apache/iceberg/expressions/Expressions.java @@ -24,7 +24,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; -import org.apache.iceberg.types.Types; /** Factory methods for creating {@link Expression expressions}. */ public class Expressions { @@ -74,37 +73,32 @@ public static Expression not(Expression child) { @SuppressWarnings("unchecked") public static UnboundTerm bucket(String name, int numBuckets) { - Transform transform = - (Transform) Transforms.bucket(Types.StringType.get(), numBuckets); + Transform transform = (Transform) Transforms.bucket(numBuckets); return new UnboundTransform<>(ref(name), transform); } @SuppressWarnings("unchecked") public static UnboundTerm year(String name) { - return new UnboundTransform<>( - ref(name), (Transform) Transforms.year(Types.TimestampType.withZone())); + return new UnboundTransform<>(ref(name), (Transform) Transforms.year()); } @SuppressWarnings("unchecked") public static UnboundTerm month(String name) { - return new UnboundTransform<>( - ref(name), (Transform) Transforms.month(Types.TimestampType.withZone())); + return new UnboundTransform<>(ref(name), (Transform) Transforms.month()); } @SuppressWarnings("unchecked") public static UnboundTerm day(String name) { - return new UnboundTransform<>( - ref(name), (Transform) Transforms.day(Types.TimestampType.withZone())); + return new UnboundTransform<>(ref(name), (Transform) Transforms.day()); } @SuppressWarnings("unchecked") public static UnboundTerm hour(String name) { - return new UnboundTransform<>( - ref(name), (Transform) Transforms.hour(Types.TimestampType.withZone())); + return new UnboundTransform<>(ref(name), (Transform) Transforms.hour()); } public static UnboundTerm truncate(String name, int width) { - return new UnboundTransform<>(ref(name), Transforms.truncate(Types.LongType.get(), width)); + return new UnboundTransform<>(ref(name), Transforms.truncate(width)); } public static UnboundPredicate isNull(String name) { diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java index cd92fce3916e..cae84733c8d5 100644 --- a/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java +++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java @@ -20,7 +20,6 @@ import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.transforms.Transform; -import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; public class UnboundTransform implements UnboundTerm, Term { @@ -41,18 +40,13 @@ public Transform transform() { return transform; } - @SuppressWarnings("unchecked") @Override public BoundTransform bind(Types.StructType struct, boolean caseSensitive) { BoundReference boundRef = ref.bind(struct, caseSensitive); - Transform typeTransform; try { - // TODO: Avoid using toString/fromString - typeTransform = - (Transform) Transforms.fromString(boundRef.type(), transform.toString()); ValidationException.check( - typeTransform.canTransform(boundRef.type()), + transform.canTransform(boundRef.type()), "Cannot bind: %s cannot transform %s values from '%s'", transform, boundRef.type(), @@ -63,7 +57,7 @@ public BoundTransform bind(Types.StructType struct, boolean caseSensitive) transform, boundRef.type(), ref.name()); } - return new BoundTransform<>(boundRef, typeTransform); + return new BoundTransform<>(boundRef, transform); } @Override diff --git a/api/src/main/java/org/apache/iceberg/transforms/Bucket.java b/api/src/main/java/org/apache/iceberg/transforms/Bucket.java index 32540bb923d8..912bcd271725 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Bucket.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Bucket.java @@ -18,52 +18,53 @@ */ package org.apache.iceberg.transforms; -import static org.apache.iceberg.types.Type.TypeID; - +import java.io.Serializable; import java.math.BigDecimal; import java.nio.ByteBuffer; -import java.util.Set; import java.util.UUID; +import java.util.function.Function; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundTransform; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; -import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Objects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.relocated.com.google.common.hash.HashFunction; -import org.apache.iceberg.relocated.com.google.common.hash.Hashing; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.BucketUtil; +import org.apache.iceberg.util.SerializableFunction; -abstract class Bucket implements Transform { - private static final HashFunction MURMUR3 = Hashing.murmur3_32_fixed(); +class Bucket implements Transform, Serializable { + static Bucket get(int numBuckets) { + Preconditions.checkArgument( + numBuckets > 0, "Invalid number of buckets: %s (must be > 0)", numBuckets); + return new Bucket<>(numBuckets); + } @SuppressWarnings("unchecked") - static Bucket get(Type type, int numBuckets) { + static & SerializableFunction> B get( + Type type, int numBuckets) { Preconditions.checkArgument( numBuckets > 0, "Invalid number of buckets: %s (must be > 0)", numBuckets); switch (type.typeId()) { case DATE: case INTEGER: - return (Bucket) new BucketInteger(numBuckets); + return (B) new BucketInteger(numBuckets); case TIME: case TIMESTAMP: case LONG: - return (Bucket) new BucketLong(numBuckets); + return (B) new BucketLong(numBuckets); case DECIMAL: - return (Bucket) new BucketDecimal(numBuckets); + return (B) new BucketDecimal(numBuckets); case STRING: - return (Bucket) new BucketString(numBuckets); + return (B) new BucketString(numBuckets); case FIXED: case BINARY: - return (Bucket) new BucketByteBuffer(numBuckets); + return (B) new BucketByteBuffer(numBuckets); case UUID: - return (Bucket) new BucketUUID(numBuckets); + return (B) new BucketUUID(numBuckets); default: throw new IllegalArgumentException("Cannot bucket by type: " + type); } @@ -79,8 +80,16 @@ public Integer numBuckets() { return numBuckets; } - @VisibleForTesting - abstract int hash(T value); + @Override + public SerializableFunction bind(Type type) { + Preconditions.checkArgument(canTransform(type), "Cannot bucket by type: %s", type); + return get(type, numBuckets); + } + + protected int hash(T value) { + throw new UnsupportedOperationException( + "hash(value) is not supported on the base Bucket class"); + } @Override public Integer apply(T value) { @@ -90,6 +99,24 @@ public Integer apply(T value) { return (hash(value) & Integer.MAX_VALUE) % numBuckets; } + @Override + public boolean canTransform(Type type) { + switch (type.typeId()) { + case INTEGER: + case LONG: + case DATE: + case TIME: + case TIMESTAMP: + case STRING: + case BINARY: + case FIXED: + case DECIMAL: + case UUID: + return true; + } + return false; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -114,6 +141,7 @@ public String toString() { @Override public UnboundPredicate project(String name, BoundPredicate predicate) { + Function function = this.bind(predicate.term().type()); if (predicate.term() instanceof BoundTransform) { return ProjectionUtil.projectTransformPredicate(this, name, predicate); } @@ -122,10 +150,10 @@ public UnboundPredicate project(String name, BoundPredicate predicat return Expressions.predicate(predicate.op(), name); } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) { return Expressions.predicate( - predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value())); + predicate.op(), name, function.apply(predicate.asLiteralPredicate().literal().value())); } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.IN) { // notIn can't be projected - return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this); + return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), function); } // comparison predicates can't be projected, notEq can't be projected @@ -136,6 +164,7 @@ public UnboundPredicate project(String name, BoundPredicate predicat @Override public UnboundPredicate projectStrict(String name, BoundPredicate predicate) { + Function function = this.bind(predicate.term().type()); if (predicate.term() instanceof BoundTransform) { return ProjectionUtil.projectTransformPredicate(this, name, predicate); } @@ -145,9 +174,9 @@ public UnboundPredicate projectStrict(String name, BoundPredicate pr } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.NOT_EQ) { // TODO: need to translate not(eq(...)) into notEq in expressions return Expressions.predicate( - predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value())); + predicate.op(), name, function.apply(predicate.asLiteralPredicate().literal().value())); } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.NOT_IN) { - return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this); + return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), function); } // no strict projection for comparison or equality @@ -159,111 +188,61 @@ public Type getResultType(Type sourceType) { return Types.IntegerType.get(); } - private static class BucketInteger extends Bucket { + private static class BucketInteger extends Bucket + implements SerializableFunction { + private BucketInteger(int numBuckets) { super(numBuckets); } @Override - public int hash(Integer value) { + protected int hash(Integer value) { return BucketUtil.hash(value); } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.INTEGER || type.typeId() == TypeID.DATE; - } } - private static class BucketLong extends Bucket { - private BucketLong(int numBuckets) { - super(numBuckets); - } - - @Override - public int hash(Long value) { - return BucketUtil.hash(value); - } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.LONG - || type.typeId() == TypeID.TIME - || type.typeId() == TypeID.TIMESTAMP; - } - } + private static class BucketLong extends Bucket + implements SerializableFunction { - // bucketing by Double is not allowed by the spec, but this has the float hash implementation - static class BucketFloat extends Bucket { - // used by tests because the factory method will not instantiate a bucket function for floats - BucketFloat(int numBuckets) { + private BucketLong(int numBuckets) { super(numBuckets); } @Override - public int hash(Float value) { + protected int hash(Long value) { return BucketUtil.hash(value); } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.FLOAT; - } } - // bucketing by Double is not allowed by the spec, but this has the double hash implementation - static class BucketDouble extends Bucket { - // used by tests because the factory method will not instantiate a bucket function for doubles - BucketDouble(int numBuckets) { - super(numBuckets); - } - - @Override - public int hash(Double value) { - return BucketUtil.hash(value); - } + private static class BucketString extends Bucket + implements SerializableFunction { - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.DOUBLE; - } - } - - private static class BucketString extends Bucket { private BucketString(int numBuckets) { super(numBuckets); } @Override - public int hash(CharSequence value) { + protected int hash(CharSequence value) { return BucketUtil.hash(value); } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.STRING; - } } - private static class BucketByteBuffer extends Bucket { - private static final Set SUPPORTED_TYPES = Sets.newHashSet(TypeID.BINARY, TypeID.FIXED); + private static class BucketByteBuffer extends Bucket + implements SerializableFunction { private BucketByteBuffer(int numBuckets) { super(numBuckets); } @Override - public int hash(ByteBuffer value) { + protected int hash(ByteBuffer value) { return BucketUtil.hash(value); } - - @Override - public boolean canTransform(Type type) { - return SUPPORTED_TYPES.contains(type.typeId()); - } } - private static class BucketUUID extends Bucket { + private static class BucketUUID extends Bucket + implements SerializableFunction { + private BucketUUID(int numBuckets) { super(numBuckets); } @@ -272,26 +251,18 @@ private BucketUUID(int numBuckets) { public int hash(UUID value) { return BucketUtil.hash(value); } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.UUID; - } } - private static class BucketDecimal extends Bucket { + private static class BucketDecimal extends Bucket + implements SerializableFunction { + private BucketDecimal(int numBuckets) { super(numBuckets); } @Override - public int hash(BigDecimal value) { + protected int hash(BigDecimal value) { return BucketUtil.hash(value); } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == TypeID.DECIMAL; - } } } diff --git a/api/src/main/java/org/apache/iceberg/transforms/Dates.java b/api/src/main/java/org/apache/iceberg/transforms/Dates.java index b313a2f154e9..0543a0c8915a 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Dates.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Dates.java @@ -27,44 +27,66 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.SerializableFunction; enum Dates implements Transform { YEAR(ChronoUnit.YEARS, "year"), MONTH(ChronoUnit.MONTHS, "month"), DAY(ChronoUnit.DAYS, "day"); + static class Apply implements SerializableFunction { + private final ChronoUnit granularity; + + Apply(ChronoUnit granularity) { + this.granularity = granularity; + } + + @Override + public Integer apply(Integer days) { + if (days == null) { + return null; + } + + if (granularity == ChronoUnit.DAYS) { + return days; + } + + if (days >= 0) { + LocalDate date = EPOCH.plusDays(days); + return (int) granularity.between(EPOCH, date); + } else { + // add 1 day to the value to account for the case where there is exactly 1 unit between the + // date and epoch because the result will always be decremented. + LocalDate date = EPOCH.plusDays(days + 1); + return (int) granularity.between(EPOCH, date) - 1; + } + } + } + private static final LocalDate EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC).toLocalDate(); private final ChronoUnit granularity; private final String name; + private final Apply apply; Dates(ChronoUnit granularity, String name) { this.granularity = granularity; this.name = name; + this.apply = new Apply(granularity); } @Override public Integer apply(Integer days) { - if (days == null) { - return null; - } - - if (granularity == ChronoUnit.DAYS) { - return days; - } + return apply.apply(days); + } - if (days >= 0) { - LocalDate date = EPOCH.plusDays(days); - return (int) granularity.between(EPOCH, date); - } else { - // add 1 day to the value to account for the case where there is exactly 1 unit between the - // date and epoch - // because the result will always be decremented. - LocalDate date = EPOCH.plusDays(days + 1); - return (int) granularity.between(EPOCH, date) - 1; - } + @Override + public SerializableFunction bind(Type type) { + Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type); + return apply; } @Override @@ -113,7 +135,7 @@ public UnboundPredicate project(String fieldName, BoundPredicate projected = - ProjectionUtil.truncateInteger(fieldName, pred.asLiteralPredicate(), this); + ProjectionUtil.truncateInteger(fieldName, pred.asLiteralPredicate(), apply); if (this != DAY) { return ProjectionUtil.fixInclusiveTimeProjection(projected); } @@ -122,7 +144,7 @@ public UnboundPredicate project(String fieldName, BoundPredicate projected = - ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply); if (this != DAY) { return ProjectionUtil.fixInclusiveTimeProjection(projected); } @@ -144,7 +166,7 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< } else if (pred.isLiteralPredicate()) { UnboundPredicate projected = - ProjectionUtil.truncateIntegerStrict(fieldName, pred.asLiteralPredicate(), this); + ProjectionUtil.truncateIntegerStrict(fieldName, pred.asLiteralPredicate(), apply); if (this != DAY) { return ProjectionUtil.fixStrictTimeProjection(projected); } @@ -153,7 +175,7 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) { UnboundPredicate projected = - ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply); if (this != DAY) { return ProjectionUtil.fixStrictTimeProjection(projected); } @@ -165,7 +187,7 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< } @Override - public String toHumanString(Integer value) { + public String toHumanString(Type outputType, Integer value) { if (value == null) { return "null"; } diff --git a/api/src/main/java/org/apache/iceberg/transforms/Days.java b/api/src/main/java/org/apache/iceberg/transforms/Days.java new file mode 100644 index 000000000000..f69d5d6110ed --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/transforms/Days.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.transforms; + +import java.io.ObjectStreamException; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +public class Days extends TimeTransform { + private static final Days INSTANCE = new Days<>(); + + @SuppressWarnings("unchecked") + static Days get() { + return (Days) INSTANCE; + } + + @Override + @SuppressWarnings("unchecked") + protected Transform toEnum(Type type) { + switch (type.typeId()) { + case DATE: + return (Transform) Dates.DAY; + case TIMESTAMP: + return (Transform) Timestamps.DAY; + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + @Override + public Type getResultType(Type sourceType) { + return Types.DateType.get(); + } + + @Override + public boolean satisfiesOrderOf(Transform other) { + if (this == other) { + return true; + } + + if (other instanceof Timestamps) { + return Timestamps.DAY.satisfiesOrderOf(other); + } else if (other instanceof Dates) { + return Dates.DAY.satisfiesOrderOf(other); + } else if (other instanceof Days || other instanceof Months || other instanceof Years) { + return true; + } + + return false; + } + + @Override + public String toHumanString(Type alwaysDate, Integer value) { + return value != null ? TransformUtil.humanDay(value) : "null"; + } + + @Override + public String toString() { + return "day"; + } + + Object writeReplace() throws ObjectStreamException { + return SerializationProxies.DaysTransformProxy.get(); + } +} diff --git a/api/src/main/java/org/apache/iceberg/transforms/Hours.java b/api/src/main/java/org/apache/iceberg/transforms/Hours.java new file mode 100644 index 000000000000..afc14516f3cd --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/transforms/Hours.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.transforms; + +import java.io.ObjectStreamException; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +public class Hours extends TimeTransform { + private static final Hours INSTANCE = new Hours<>(); + + @SuppressWarnings("unchecked") + static Hours get() { + return (Hours) INSTANCE; + } + + @Override + @SuppressWarnings("unchecked") + protected Transform toEnum(Type type) { + if (type.typeId() == Type.TypeID.TIMESTAMP) { + return (Transform) Timestamps.HOUR; + } + + throw new IllegalArgumentException("Unsupported type: " + type); + } + + @Override + public boolean canTransform(Type type) { + return type.typeId() == Type.TypeID.TIMESTAMP; + } + + @Override + public Type getResultType(Type sourceType) { + return Types.IntegerType.get(); + } + + @Override + public boolean satisfiesOrderOf(Transform other) { + if (this == other) { + return true; + } + + if (other instanceof Timestamps) { + return other == Timestamps.HOUR; + } else if (other instanceof Hours + || other instanceof Days + || other instanceof Months + || other instanceof Years) { + return true; + } + + return false; + } + + @Override + public String toHumanString(Type alwaysInt, Integer value) { + return value != null ? TransformUtil.humanHour(value) : "null"; + } + + @Override + public String toString() { + return "hour"; + } + + Object writeReplace() throws ObjectStreamException { + return SerializationProxies.HoursTransformProxy.get(); + } +} diff --git a/api/src/main/java/org/apache/iceberg/transforms/Identity.java b/api/src/main/java/org/apache/iceberg/transforms/Identity.java index 78586543a24f..365abdf0b94a 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Identity.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Identity.java @@ -18,31 +18,49 @@ */ package org.apache.iceberg.transforms; -import java.nio.ByteBuffer; +import java.io.ObjectStreamException; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; -import org.apache.iceberg.relocated.com.google.common.base.Objects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.SerializableFunction; class Identity implements Transform { + private static final Identity INSTANCE = new Identity<>(); + @SuppressWarnings("unchecked") - public static Identity get(Type type) { - return new Identity<>(type); + public static Identity get() { + return (Identity) INSTANCE; } - private final Type type; + private static class Apply implements SerializableFunction { + private static final Apply APPLY_INSTANCE = new Apply<>(); + + @SuppressWarnings("unchecked") + private static Apply get() { + return (Apply) APPLY_INSTANCE; + } - private Identity(Type type) { - this.type = type; + @Override + public T apply(T t) { + return t; + } } + private Identity() {} + @Override public T apply(T value) { return value; } + @Override + public SerializableFunction bind(Type type) { + Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type); + return Apply.get(); + } + @Override public boolean canTransform(Type maybePrimitive) { return maybePrimitive.isPrimitiveType(); @@ -87,56 +105,12 @@ public boolean isIdentity() { return true; } - @Override - public String toHumanString(T value) { - if (value == null) { - return "null"; - } - - switch (type.typeId()) { - case DATE: - return TransformUtil.humanDay((Integer) value); - case TIME: - return TransformUtil.humanTime((Long) value); - case TIMESTAMP: - if (((Types.TimestampType) type).shouldAdjustToUTC()) { - return TransformUtil.humanTimestampWithZone((Long) value); - } else { - return TransformUtil.humanTimestampWithoutZone((Long) value); - } - case FIXED: - case BINARY: - if (value instanceof ByteBuffer) { - return TransformUtil.base64encode(((ByteBuffer) value).duplicate()); - } else if (value instanceof byte[]) { - return TransformUtil.base64encode(ByteBuffer.wrap((byte[]) value)); - } else { - throw new UnsupportedOperationException("Unsupported binary type: " + value.getClass()); - } - default: - return value.toString(); - } - } - @Override public String toString() { return "identity"; } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } else if (!(o instanceof Identity)) { - return false; - } - - Identity that = (Identity) o; - return type.equals(that.type); - } - - @Override - public int hashCode() { - return Objects.hashCode(type); + Object writeReplace() throws ObjectStreamException { + return SerializationProxies.IdentityTransformProxy.get(); } } diff --git a/api/src/main/java/org/apache/iceberg/transforms/Months.java b/api/src/main/java/org/apache/iceberg/transforms/Months.java new file mode 100644 index 000000000000..8fa4d42385f7 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/transforms/Months.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.transforms; + +import java.io.ObjectStreamException; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +public class Months extends TimeTransform { + private static final Months INSTANCE = new Months<>(); + + @SuppressWarnings("unchecked") + static Months get() { + return (Months) INSTANCE; + } + + @Override + @SuppressWarnings("unchecked") + protected Transform toEnum(Type type) { + switch (type.typeId()) { + case DATE: + return (Transform) Dates.MONTH; + case TIMESTAMP: + return (Transform) Timestamps.MONTH; + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + @Override + public Type getResultType(Type sourceType) { + return Types.IntegerType.get(); + } + + @Override + public boolean satisfiesOrderOf(Transform other) { + if (this == other) { + return true; + } + + if (other instanceof Timestamps) { + return Timestamps.MONTH.satisfiesOrderOf(other); + } else if (other instanceof Dates) { + return Dates.MONTH.satisfiesOrderOf(other); + } else if (other instanceof Months || other instanceof Years) { + return true; + } + + return false; + } + + @Override + public String toHumanString(Type alwaysInt, Integer value) { + return value != null ? TransformUtil.humanMonth(value) : "null"; + } + + @Override + public String toString() { + return "month"; + } + + Object writeReplace() throws ObjectStreamException { + return SerializationProxies.MonthsTransformProxy.get(); + } +} diff --git a/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java b/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java index eee174bc2d39..e4796478bf28 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java +++ b/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java @@ -121,13 +121,17 @@ static R visit(Schema schema, PartitionField field, PartitionSpecVisitor } else if (transform instanceof Truncate) { int width = ((Truncate) transform).width(); return visitor.truncate(field.fieldId(), sourceName, field.sourceId(), width); - } else if (transform == Dates.YEAR || transform == Timestamps.YEAR) { + } else if (transform == Dates.YEAR + || transform == Timestamps.YEAR + || transform instanceof Years) { return visitor.year(field.fieldId(), sourceName, field.sourceId()); - } else if (transform == Dates.MONTH || transform == Timestamps.MONTH) { + } else if (transform == Dates.MONTH + || transform == Timestamps.MONTH + || transform instanceof Months) { return visitor.month(field.fieldId(), sourceName, field.sourceId()); - } else if (transform == Dates.DAY || transform == Timestamps.DAY) { + } else if (transform == Dates.DAY || transform == Timestamps.DAY || transform instanceof Days) { return visitor.day(field.fieldId(), sourceName, field.sourceId()); - } else if (transform == Timestamps.HOUR) { + } else if (transform == Timestamps.HOUR || transform instanceof Hours) { return visitor.hour(field.fieldId(), sourceName, field.sourceId()); } else if (transform instanceof VoidTransform) { return visitor.alwaysNull(field.fieldId(), sourceName, field.sourceId()); diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java index a359c9b88492..732336d51094 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java +++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java @@ -23,6 +23,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Set; +import java.util.function.Function; import org.apache.iceberg.expressions.BoundLiteralPredicate; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundSetPredicate; @@ -39,7 +40,7 @@ class ProjectionUtil { private ProjectionUtil() {} static UnboundPredicate truncateInteger( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { int boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -60,7 +61,7 @@ static UnboundPredicate truncateInteger( } static UnboundPredicate truncateIntegerStrict( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { int boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -83,7 +84,7 @@ static UnboundPredicate truncateIntegerStrict( } static UnboundPredicate truncateLongStrict( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { long boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -106,7 +107,7 @@ static UnboundPredicate truncateLongStrict( } static UnboundPredicate truncateLong( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { long boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -127,7 +128,7 @@ static UnboundPredicate truncateLong( } static UnboundPredicate truncateDecimal( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { BigDecimal boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -152,7 +153,7 @@ static UnboundPredicate truncateDecimal( } static UnboundPredicate truncateDecimalStrict( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { BigDecimal boundary = pred.literal().value(); BigDecimal minusOne = @@ -182,7 +183,7 @@ static UnboundPredicate truncateDecimalStrict( } static UnboundPredicate truncateArray( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { S boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -203,7 +204,7 @@ static UnboundPredicate truncateArray( } static UnboundPredicate truncateArrayStrict( - String name, BoundLiteralPredicate pred, Transform transform) { + String name, BoundLiteralPredicate pred, Function transform) { S boundary = pred.literal().value(); switch (pred.op()) { case LT: @@ -251,7 +252,7 @@ private static UnboundPredicate removeTransform( } static UnboundPredicate transformSet( - String fieldName, BoundSetPredicate predicate, Transform transform) { + String fieldName, BoundSetPredicate predicate, Function transform) { return predicate( predicate.op(), fieldName, diff --git a/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java b/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java index 5ae7235f60cf..90f59622a76a 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java +++ b/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java @@ -43,4 +43,79 @@ Object readResolve() throws ObjectStreamException { return VoidTransform.get(); } } + + static class IdentityTransformProxy implements Serializable { + private static final IdentityTransformProxy INSTANCE = new IdentityTransformProxy(); + + static IdentityTransformProxy get() { + return INSTANCE; + } + + /** Constructor for Java serialization. */ + IdentityTransformProxy() {} + + Object readResolve() throws ObjectStreamException { + return Identity.get(); + } + } + + static class YearsTransformProxy implements Serializable { + private static final YearsTransformProxy INSTANCE = new YearsTransformProxy(); + + static YearsTransformProxy get() { + return INSTANCE; + } + + /** Constructor for Java serialization. */ + YearsTransformProxy() {} + + Object readResolve() throws ObjectStreamException { + return Years.get(); + } + } + + static class MonthsTransformProxy implements Serializable { + private static final MonthsTransformProxy INSTANCE = new MonthsTransformProxy(); + + static MonthsTransformProxy get() { + return INSTANCE; + } + + /** Constructor for Java serialization. */ + MonthsTransformProxy() {} + + Object readResolve() throws ObjectStreamException { + return Months.get(); + } + } + + static class DaysTransformProxy implements Serializable { + private static final DaysTransformProxy INSTANCE = new DaysTransformProxy(); + + static DaysTransformProxy get() { + return INSTANCE; + } + + /** Constructor for Java serialization. */ + DaysTransformProxy() {} + + Object readResolve() throws ObjectStreamException { + return Days.get(); + } + } + + static class HoursTransformProxy implements Serializable { + private static final HoursTransformProxy INSTANCE = new HoursTransformProxy(); + + static HoursTransformProxy get() { + return INSTANCE; + } + + /** Constructor for Java serialization. */ + HoursTransformProxy() {} + + Object readResolve() throws ObjectStreamException { + return Hours.get(); + } + } } diff --git a/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java b/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java index ed3327571f74..680e095270fb 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java +++ b/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java @@ -84,16 +84,22 @@ static List visit(SortOrder sortOrder, SortOrderVisitor visitor) { results.add( visitor.truncate( sourceName, field.sourceId(), width, field.direction(), field.nullOrder())); - } else if (transform == Dates.YEAR || transform == Timestamps.YEAR) { + } else if (transform == Dates.YEAR + || transform == Timestamps.YEAR + || transform instanceof Years) { results.add( visitor.year(sourceName, field.sourceId(), field.direction(), field.nullOrder())); - } else if (transform == Dates.MONTH || transform == Timestamps.MONTH) { + } else if (transform == Dates.MONTH + || transform == Timestamps.MONTH + || transform instanceof Months) { results.add( visitor.month(sourceName, field.sourceId(), field.direction(), field.nullOrder())); - } else if (transform == Dates.DAY || transform == Timestamps.DAY) { + } else if (transform == Dates.DAY + || transform == Timestamps.DAY + || transform instanceof Days) { results.add( visitor.day(sourceName, field.sourceId(), field.direction(), field.nullOrder())); - } else if (transform == Timestamps.HOUR) { + } else if (transform == Timestamps.HOUR || transform instanceof Hours) { results.add( visitor.hour(sourceName, field.sourceId(), field.direction(), field.nullOrder())); } else if (transform instanceof UnknownTransform) { diff --git a/api/src/main/java/org/apache/iceberg/transforms/TimeTransform.java b/api/src/main/java/org/apache/iceberg/transforms/TimeTransform.java new file mode 100644 index 000000000000..01ea8130aa60 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/transforms/TimeTransform.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.transforms; + +import org.apache.iceberg.expressions.BoundPredicate; +import org.apache.iceberg.expressions.BoundTransform; +import org.apache.iceberg.expressions.UnboundPredicate; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.util.SerializableFunction; + +abstract class TimeTransform implements Transform { + protected abstract Transform toEnum(Type type); + + @Override + public SerializableFunction bind(Type type) { + return toEnum(type).bind(type); + } + + @Override + public boolean preservesOrder() { + return true; + } + + @Override + public boolean canTransform(Type type) { + return type.typeId() == Type.TypeID.DATE || type.typeId() == Type.TypeID.TIMESTAMP; + } + + @Override + public UnboundPredicate project(String name, BoundPredicate predicate) { + if (predicate.term() instanceof BoundTransform) { + return ProjectionUtil.projectTransformPredicate(this, name, predicate); + } + + return toEnum(predicate.term().type()).project(name, predicate); + } + + @Override + public UnboundPredicate projectStrict(String name, BoundPredicate predicate) { + if (predicate.term() instanceof BoundTransform) { + return ProjectionUtil.projectTransformPredicate(this, name, predicate); + } + + return toEnum(predicate.term().type()).projectStrict(name, predicate); + } + + @Override + public String dedupName() { + return "time"; + } +} diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java index 476033707293..e63ee39ac19e 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java @@ -27,8 +27,10 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.UnboundPredicate; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.SerializableFunction; enum Timestamps implements Transform { YEAR(ChronoUnit.YEARS, "year"), @@ -36,39 +38,59 @@ enum Timestamps implements Transform { DAY(ChronoUnit.DAYS, "day"), HOUR(ChronoUnit.HOURS, "hour"); + static class Apply implements SerializableFunction { + private final ChronoUnit granularity; + + Apply(ChronoUnit granularity) { + this.granularity = granularity; + } + + @Override + public Integer apply(Long timestampMicros) { + if (timestampMicros == null) { + return null; + } + + if (timestampMicros >= 0) { + OffsetDateTime timestamp = + Instant.ofEpochSecond( + Math.floorDiv(timestampMicros, 1_000_000), + Math.floorMod(timestampMicros, 1_000_000) * 1000) + .atOffset(ZoneOffset.UTC); + return (int) granularity.between(EPOCH, timestamp); + } else { + // add 1 micro to the value to account for the case where there is exactly 1 unit between + // the timestamp and epoch because the result will always be decremented. + OffsetDateTime timestamp = + Instant.ofEpochSecond( + Math.floorDiv(timestampMicros, 1_000_000), + Math.floorMod(timestampMicros + 1, 1_000_000) * 1000) + .atOffset(ZoneOffset.UTC); + return (int) granularity.between(EPOCH, timestamp) - 1; + } + } + } + private static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC); private final ChronoUnit granularity; private final String name; + private final SerializableFunction apply; Timestamps(ChronoUnit granularity, String name) { this.granularity = granularity; this.name = name; + this.apply = new Apply(granularity); } @Override public Integer apply(Long timestampMicros) { - if (timestampMicros == null) { - return null; - } + return apply.apply(timestampMicros); + } - if (timestampMicros >= 0) { - OffsetDateTime timestamp = - Instant.ofEpochSecond( - Math.floorDiv(timestampMicros, 1_000_000), - Math.floorMod(timestampMicros, 1_000_000) * 1000) - .atOffset(ZoneOffset.UTC); - return (int) granularity.between(EPOCH, timestamp); - } else { - // add 1 micro to the value to account for the case where there is exactly 1 unit between the - // timestamp and epoch - // because the result will always be decremented. - OffsetDateTime timestamp = - Instant.ofEpochSecond( - Math.floorDiv(timestampMicros, 1_000_000), - Math.floorMod(timestampMicros + 1, 1_000_000) * 1000) - .atOffset(ZoneOffset.UTC); - return (int) granularity.between(EPOCH, timestamp) - 1; - } + @Override + public SerializableFunction bind(Type type) { + Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type); + return apply; } @Override @@ -117,12 +139,12 @@ public UnboundPredicate project(String fieldName, BoundPredicate } else if (pred.isLiteralPredicate()) { UnboundPredicate projected = - ProjectionUtil.truncateLong(fieldName, pred.asLiteralPredicate(), this); + ProjectionUtil.truncateLong(fieldName, pred.asLiteralPredicate(), apply); return ProjectionUtil.fixInclusiveTimeProjection(projected); } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) { UnboundPredicate projected = - ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply); return ProjectionUtil.fixInclusiveTimeProjection(projected); } @@ -140,12 +162,12 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< } else if (pred.isLiteralPredicate()) { UnboundPredicate projected = - ProjectionUtil.truncateLongStrict(fieldName, pred.asLiteralPredicate(), this); + ProjectionUtil.truncateLongStrict(fieldName, pred.asLiteralPredicate(), apply); return ProjectionUtil.fixStrictTimeProjection(projected); } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) { UnboundPredicate projected = - ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this); + ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply); return ProjectionUtil.fixStrictTimeProjection(projected); } @@ -153,7 +175,7 @@ public UnboundPredicate projectStrict(String fieldName, BoundPredicate< } @Override - public String toHumanString(Integer value) { + public String toHumanString(Type outputType, Integer value) { if (value == null) { return "null"; } diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transform.java b/api/src/main/java/org/apache/iceberg/transforms/Transform.java index 9e61bf377e72..6905eddc6596 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Transform.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Transform.java @@ -19,9 +19,13 @@ package org.apache.iceberg.transforms; import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.function.Function; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.SerializableFunction; /** * A transform function used for partitioning. @@ -38,8 +42,23 @@ public interface Transform extends Serializable { * * @param value a source value * @return a transformed partition value + * @deprecated use {@link #bind(Type)} instead; will be removed in 2.0.0 */ - T apply(S value); + @Deprecated + default T apply(S value) { + throw new UnsupportedOperationException( + "apply(value) is deprecated, use bind(Type).apply(value)"); + } + + /** + * Returns a function that applies this transform to values of the given {@link Type type}. + * + * @param type an Iceberg {@link Type} + * @return a {@link Function} that applies this transform to values of the given type. + */ + default SerializableFunction bind(Type type) { + throw new UnsupportedOperationException("bind is not implemented"); + } /** * Checks whether this function can be applied to the given {@link Type}. @@ -84,7 +103,7 @@ default boolean satisfiesOrderOf(Transform other) { /** * Transforms a {@link BoundPredicate predicate} to an inclusive predicate on the partition values - * produced by {@link #apply(Object)}. + * produced by the transform. * *

This inclusive transform guarantees that if pred(v) is true, then projected(apply(v)) is * true. @@ -97,7 +116,7 @@ default boolean satisfiesOrderOf(Transform other) { /** * Transforms a {@link BoundPredicate predicate} to a strict predicate on the partition values - * produced by {@link #apply(Object)}. + * produced by the transform. * *

This strict transform guarantees that if strict(apply(v)) is true, then pred(v) is also * true. @@ -124,9 +143,47 @@ default boolean isIdentity() { * * @param value a transformed value * @return a human-readable String representation of the value + * @deprecated use {@link #toHumanString(Type, Object)} instead; will be removed in 2.0.0 */ + @Deprecated default String toHumanString(T value) { - return String.valueOf(value); + if (value instanceof ByteBuffer) { + return TransformUtil.base64encode(((ByteBuffer) value).duplicate()); + } else if (value instanceof byte[]) { + return TransformUtil.base64encode(ByteBuffer.wrap((byte[]) value)); + } else { + return String.valueOf(value); + } + } + + default String toHumanString(Type type, T value) { + if (value == null) { + return "null"; + } + + switch (type.typeId()) { + case DATE: + return TransformUtil.humanDay((Integer) value); + case TIME: + return TransformUtil.humanTime((Long) value); + case TIMESTAMP: + if (((Types.TimestampType) type).shouldAdjustToUTC()) { + return TransformUtil.humanTimestampWithZone((Long) value); + } else { + return TransformUtil.humanTimestampWithoutZone((Long) value); + } + case FIXED: + case BINARY: + if (value instanceof ByteBuffer) { + return TransformUtil.base64encode(((ByteBuffer) value).duplicate()); + } else if (value instanceof byte[]) { + return TransformUtil.base64encode(ByteBuffer.wrap((byte[]) value)); + } else { + throw new UnsupportedOperationException("Unsupported binary type: " + value.getClass()); + } + default: + return value.toString(); + } } /** diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java index 35ace7a80079..0a0c70b74604 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java @@ -39,20 +39,49 @@ private Transforms() {} private static final Pattern HAS_WIDTH = Pattern.compile("(\\w+)\\[(\\d+)\\]"); + public static Transform fromString(String transform) { + Matcher widthMatcher = HAS_WIDTH.matcher(transform); + if (widthMatcher.matches()) { + String name = widthMatcher.group(1); + int parsedWidth = Integer.parseInt(widthMatcher.group(2)); + if (name.equalsIgnoreCase("truncate")) { + return Truncate.get(parsedWidth); + } else if (name.equals("bucket")) { + return Bucket.get(parsedWidth); + } + } + + if (transform.equalsIgnoreCase("identity")) { + return Identity.get(); + } else if (transform.equalsIgnoreCase("year")) { + return Years.get(); + } else if (transform.equalsIgnoreCase("month")) { + return Months.get(); + } else if (transform.equalsIgnoreCase("day")) { + return Days.get(); + } else if (transform.equalsIgnoreCase("hour")) { + return Hours.get(); + } else if (transform.equalsIgnoreCase("void")) { + return VoidTransform.get(); + } + + return new UnknownTransform<>(transform); + } + public static Transform fromString(Type type, String transform) { Matcher widthMatcher = HAS_WIDTH.matcher(transform); if (widthMatcher.matches()) { String name = widthMatcher.group(1); int parsedWidth = Integer.parseInt(widthMatcher.group(2)); if (name.equalsIgnoreCase("truncate")) { - return Truncate.get(type, parsedWidth); + return (Transform) Truncate.get(type, parsedWidth); } else if (name.equals("bucket")) { - return Bucket.get(type, parsedWidth); + return (Transform) Bucket.get(type, parsedWidth); } } if (transform.equalsIgnoreCase("identity")) { - return Identity.get(type); + return Identity.get(); } try { @@ -69,7 +98,7 @@ private Transforms() {} return VoidTransform.get(); } - return new UnknownTransform<>(type, transform); + return new UnknownTransform<>(transform); } /** @@ -78,9 +107,11 @@ private Transforms() {} * @param type the {@link Type source type} for the transform * @param Java type passed to this transform * @return an identity transform + * @deprecated use {@link #identity()} instead; will be removed in 2.0.0 */ + @Deprecated public static Transform identity(Type type) { - return Identity.get(type); + return Identity.get(); } /** @@ -89,7 +120,9 @@ public static Transform identity(Type type) { * @param type the {@link Type source type} for the transform * @param Java type passed to this transform * @return a year transform + * @deprecated use {@link #year()} instead; will be removed in 2.0.0 */ + @Deprecated @SuppressWarnings("unchecked") public static Transform year(Type type) { switch (type.typeId()) { @@ -108,7 +141,9 @@ public static Transform year(Type type) { * @param type the {@link Type source type} for the transform * @param Java type passed to this transform * @return a month transform + * @deprecated use {@link #month()} instead; will be removed in 2.0.0 */ + @Deprecated @SuppressWarnings("unchecked") public static Transform month(Type type) { switch (type.typeId()) { @@ -127,7 +162,9 @@ public static Transform month(Type type) { * @param type the {@link Type source type} for the transform * @param Java type passed to this transform * @return a day transform + * @deprecated use {@link #day()} instead; will be removed in 2.0.0 */ + @Deprecated @SuppressWarnings("unchecked") public static Transform day(Type type) { switch (type.typeId()) { @@ -146,7 +183,9 @@ public static Transform day(Type type) { * @param type the {@link Type source type} for the transform * @param Java type passed to this transform * @return a hour transform + * @deprecated use {@link #hour()} instead; will be removed in 2.0.0 */ + @Deprecated @SuppressWarnings("unchecked") public static Transform hour(Type type) { Preconditions.checkArgument( @@ -161,7 +200,9 @@ public static Transform hour(Type type) { * @param numBuckets the number of buckets for the transform to produce * @param Java type passed to this transform * @return a transform that buckets values into numBuckets + * @deprecated use {@link #bucket(int)} instead; will be removed in 2.0.0 */ + @Deprecated public static Transform bucket(Type type, int numBuckets) { return Bucket.get(type, numBuckets); } @@ -173,9 +214,83 @@ public static Transform bucket(Type type, int numBuckets) { * @param width the width to truncate data values * @param Java type passed to this transform * @return a transform that truncates the given type to width + * @deprecated use {@link #truncate(int)} instead; will be removed in 2.0.0 */ + @Deprecated public static Transform truncate(Type type, int width) { - return Truncate.get(type, width); + return (Transform) Truncate.get(type, width); + } + + /** + * Returns an identity {@link Transform} that can be used for any type. + * + * @param Java type passed to this transform + * @return an identity transform + */ + public static Transform identity() { + return Identity.get(); + } + + /** + * Returns a year {@link Transform} for date or timestamp types. + * + * @param Java type passed to this transform + * @return a year transform + */ + public static Transform year() { + return Years.get(); + } + + /** + * Returns a year {@link Transform} for date or timestamp types. + * + * @param Java type passed to this transform + * @return a year transform + */ + public static Transform month() { + return Months.get(); + } + + /** + * Returns a year {@link Transform} for date or timestamp types. + * + * @param Java type passed to this transform + * @return a year transform + */ + public static Transform day() { + return Days.get(); + } + + /** + * Returns a year {@link Transform} for date or timestamp types. + * + * @param Java type passed to this transform + * @return a year transform + */ + public static Transform hour() { + return Hours.get(); + } + + /** + * Returns a bucket {@link Transform} for the given number of buckets. + * + * @param numBuckets the number of buckets for the transform to produce + * @param Java type passed to this transform + * @return a transform that buckets values into numBuckets + */ + public static Transform bucket(int numBuckets) { + return Bucket.get(numBuckets); + } + + /** + * Returns a truncate {@link Transform} for the given width. + * + * @param width the width to truncate data values + * @param Java type passed to this transform + * @return a transform that truncates the given type to width + */ + public static Transform truncate(int width) { + return Truncate.get(width); } /** diff --git a/api/src/main/java/org/apache/iceberg/transforms/Truncate.java b/api/src/main/java/org/apache/iceberg/transforms/Truncate.java index 22958fe42b94..a6c0427d8149 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Truncate.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Truncate.java @@ -21,6 +21,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; +import java.util.function.Function; import org.apache.iceberg.expressions.BoundLiteralPredicate; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.BoundTransform; @@ -32,34 +33,84 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; import org.apache.iceberg.util.BinaryUtil; +import org.apache.iceberg.util.SerializableFunction; import org.apache.iceberg.util.TruncateUtil; import org.apache.iceberg.util.UnicodeUtil; -abstract class Truncate implements Transform { +class Truncate implements Transform, Function { + static Truncate get(int width) { + Preconditions.checkArgument(width > 0, "Invalid truncate width: %s (must be > 0)", width); + return new Truncate<>(width); + } + + @Deprecated @SuppressWarnings("unchecked") - static Truncate get(Type type, int width) { + static & SerializableFunction> R get(Type type, int width) { Preconditions.checkArgument(width > 0, "Invalid truncate width: %s (must be > 0)", width); switch (type.typeId()) { case INTEGER: - return (Truncate) new TruncateInteger(width); + return (R) new TruncateInteger(width); case LONG: - return (Truncate) new TruncateLong(width); + return (R) new TruncateLong(width); case DECIMAL: - return (Truncate) new TruncateDecimal(width); + return (R) new TruncateDecimal(width); case STRING: - return (Truncate) new TruncateString(width); + return (R) new TruncateString(width); case BINARY: - return (Truncate) new TruncateByteBuffer(width); + return (R) new TruncateByteBuffer(width); default: throw new UnsupportedOperationException("Cannot truncate type: " + type); } } - public abstract Integer width(); + @SuppressWarnings("checkstyle:VisibilityModifier") + protected final int width; + + Truncate(int width) { + this.width = width; + } + + public Integer width() { + return width; + } + + @Override + public T apply(T value) { + throw new UnsupportedOperationException( + "apply(value) is deprecated, use bind(Type).apply(value)"); + } + + @Override + public SerializableFunction bind(Type type) { + Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type); + return (SerializableFunction) get(type, width); + } + + @Override + public boolean canTransform(Type type) { + switch (type.typeId()) { + case INTEGER: + case LONG: + case STRING: + case BINARY: + case DECIMAL: + return true; + } + return false; + } + + @Override + public UnboundPredicate project(String name, BoundPredicate predicate) { + Truncate bound = (Truncate) get(predicate.term().type(), width); + return bound.project(name, predicate); + } @Override - public abstract T apply(T value); + public UnboundPredicate projectStrict(String name, BoundPredicate predicate) { + Truncate bound = (Truncate) get(predicate.term().type(), width); + return bound.projectStrict(name, predicate); + } @Override public Type getResultType(Type sourceType) { @@ -71,16 +122,56 @@ public boolean preservesOrder() { return true; } - private static class TruncateInteger extends Truncate { - private final int width; + @Override + public boolean satisfiesOrderOf(Transform other) { + if (this == other) { + return true; + } + + if (!(other instanceof Truncate)) { + return false; + } + + Truncate otherTrunc = (Truncate) other; + return otherTrunc.width <= width; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (!(o instanceof Truncate)) { + return false; + } + + Truncate that = (Truncate) o; + return width == that.width; + } + + @Override + public int hashCode() { + return Objects.hashCode(width); + } + + @Override + public String toString() { + return "truncate[" + width + "]"; + } + + private static class TruncateInteger extends Truncate + implements SerializableFunction { private TruncateInteger(int width) { - this.width = width; + super(width); } @Override - public Integer width() { - return width; + public SerializableFunction bind(Type type) { + Preconditions.checkArgument( + type.typeId() == Type.TypeID.INTEGER, + "Cannot bind truncate to a different type: %s", + type); + return this; } @Override @@ -92,11 +183,6 @@ public Integer apply(Integer value) { return TruncateUtil.truncateInt(width, value); } - @Override - public boolean canTransform(Type type) { - return type.typeId() == Type.TypeID.INTEGER; - } - @Override public UnboundPredicate project(String name, BoundPredicate pred) { if (pred.term() instanceof BoundTransform) { @@ -131,40 +217,20 @@ public UnboundPredicate projectStrict(String name, BoundPredicate { - private final int width; + private static class TruncateLong extends Truncate + implements SerializableFunction { private TruncateLong(int width) { - this.width = width; + super(width); } @Override - public Integer width() { - return width; + public SerializableFunction bind(Type type) { + Preconditions.checkArgument( + type.typeId() == Type.TypeID.LONG, "Cannot bind truncate to a different type: %s", type); + return this; } @Override @@ -176,11 +242,6 @@ public Long apply(Long value) { return TruncateUtil.truncateLong(width, value); } - @Override - public boolean canTransform(Type type) { - return type.typeId() == Type.TypeID.LONG; - } - @Override public UnboundPredicate project(String name, BoundPredicate pred) { if (pred.term() instanceof BoundTransform) { @@ -212,40 +273,22 @@ public UnboundPredicate projectStrict(String name, BoundPredicate pr } return null; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } else if (!(o instanceof TruncateLong)) { - return false; - } - - TruncateLong that = (TruncateLong) o; - return width == that.width; - } - - @Override - public int hashCode() { - return Objects.hashCode(width); - } - - @Override - public String toString() { - return "truncate[" + width + "]"; - } } - private static class TruncateString extends Truncate { - private final int length; + private static class TruncateString extends Truncate + implements SerializableFunction { private TruncateString(int length) { - this.length = length; + super(length); } @Override - public Integer width() { - return length; + public SerializableFunction bind(Type type) { + Preconditions.checkArgument( + type.typeId() == Type.TypeID.STRING, + "Cannot bind truncate to a different type: %s", + type); + return this; } @Override @@ -254,12 +297,7 @@ public CharSequence apply(CharSequence value) { return null; } - return UnicodeUtil.truncateString(value, length); - } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == Type.TypeID.STRING; + return UnicodeUtil.truncateString(value, width); } @Override @@ -351,40 +389,22 @@ public UnboundPredicate projectStrict( } return null; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } else if (!(o instanceof TruncateString)) { - return false; - } - - TruncateString that = (TruncateString) o; - return length == that.length; - } - - @Override - public int hashCode() { - return Objects.hashCode(length); - } - - @Override - public String toString() { - return "truncate[" + length + "]"; - } } - private static class TruncateByteBuffer extends Truncate { - private final int length; + private static class TruncateByteBuffer extends Truncate + implements SerializableFunction { private TruncateByteBuffer(int length) { - this.length = length; + super(length); } @Override - public Integer width() { - return length; + public SerializableFunction bind(Type type) { + Preconditions.checkArgument( + type.typeId() == Type.TypeID.BINARY || type.typeId() == Type.TypeID.FIXED, + "Cannot bind truncate to a different type: %s", + type); + return this; } @Override @@ -393,12 +413,7 @@ public ByteBuffer apply(ByteBuffer value) { return null; } - return BinaryUtil.truncateBinaryUnsafe(value, length); - } - - @Override - public boolean canTransform(Type type) { - return type.typeId() == Type.TypeID.BINARY; + return BinaryUtil.truncateBinaryUnsafe(value, width); } @Override @@ -433,45 +448,25 @@ public UnboundPredicate projectStrict( } return null; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } else if (!(o instanceof TruncateByteBuffer)) { - return false; - } - - TruncateByteBuffer that = (TruncateByteBuffer) o; - return length == that.length; - } - - @Override - public int hashCode() { - return Objects.hashCode(length); - } - - @Override - public String toHumanString(ByteBuffer value) { - return value == null ? "null" : TransformUtil.base64encode(value); - } - - @Override - public String toString() { - return "truncate[" + length + "]"; - } } - private static class TruncateDecimal extends Truncate { + private static class TruncateDecimal extends Truncate + implements SerializableFunction { + private final BigInteger unscaledWidth; private TruncateDecimal(int unscaledWidth) { + super(unscaledWidth); this.unscaledWidth = BigInteger.valueOf(unscaledWidth); } @Override - public Integer width() { - return unscaledWidth.intValue(); + public SerializableFunction bind(Type type) { + Preconditions.checkArgument( + type.typeId() == Type.TypeID.DECIMAL, + "Cannot bind truncate to a different type: %s", + type); + return this; } @Override @@ -483,11 +478,6 @@ public BigDecimal apply(BigDecimal value) { return TruncateUtil.truncateDecimal(unscaledWidth, value); } - @Override - public boolean canTransform(Type type) { - return type.typeId() == Type.TypeID.DECIMAL; - } - @Override public UnboundPredicate project(String name, BoundPredicate pred) { if (pred.term() instanceof BoundTransform) { @@ -520,27 +510,5 @@ public UnboundPredicate projectStrict( } return null; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } else if (!(o instanceof TruncateDecimal)) { - return false; - } - - TruncateDecimal that = (TruncateDecimal) o; - return unscaledWidth.equals(that.unscaledWidth); - } - - @Override - public int hashCode() { - return Objects.hashCode(unscaledWidth); - } - - @Override - public String toString() { - return "truncate[" + unscaledWidth + "]"; - } } } diff --git a/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java b/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java index 95a30beac23e..457bd0630475 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java +++ b/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java @@ -23,14 +23,13 @@ import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.SerializableFunction; public class UnknownTransform implements Transform { - private final Type sourceType; private final String transform; - UnknownTransform(Type sourceType, String transform) { - this.sourceType = sourceType; + UnknownTransform(String transform) { this.transform = transform; } @@ -40,12 +39,16 @@ public T apply(S value) { String.format("Cannot apply unsupported transform: %s", transform)); } + @Override + public SerializableFunction bind(Type type) { + throw new UnsupportedOperationException( + String.format("Cannot bind unsupported transform: %s", transform)); + } + @Override public boolean canTransform(Type type) { - // assume the transform function can be applied for this type because unknown transform is only - // used when parsing - // a transform in an existing table. a different Iceberg version must have already validated it. - return this.sourceType.equals(type); + // assume the transform function can be applied for any type + return true; } @Override @@ -78,11 +81,11 @@ public boolean equals(Object other) { } UnknownTransform that = (UnknownTransform) other; - return sourceType.equals(that.sourceType) && transform.equals(that.transform); + return transform.equals(that.transform); } @Override public int hashCode() { - return Objects.hash(sourceType, transform); + return Objects.hash(transform); } } diff --git a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java b/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java index 83f7f76bf014..cbab2879ffe8 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java +++ b/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java @@ -19,9 +19,11 @@ package org.apache.iceberg.transforms; import java.io.ObjectStreamException; +import java.io.Serializable; import org.apache.iceberg.expressions.BoundPredicate; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.types.Type; +import org.apache.iceberg.util.SerializableFunction; class VoidTransform implements Transform { private static final VoidTransform INSTANCE = new VoidTransform<>(); @@ -31,6 +33,20 @@ static VoidTransform get() { return (VoidTransform) INSTANCE; } + private static class Apply implements SerializableFunction, Serializable { + private static final Apply APPLY_INSTANCE = new Apply<>(); + + @SuppressWarnings("unchecked") + private static Apply get() { + return (Apply) APPLY_INSTANCE; + } + + @Override + public Void apply(S t) { + return null; + } + } + private VoidTransform() {} @Override @@ -38,6 +54,11 @@ public Void apply(Object value) { return null; } + @Override + public SerializableFunction bind(Type type) { + return Apply.get(); + } + @Override public boolean canTransform(Type type) { return true; diff --git a/api/src/main/java/org/apache/iceberg/transforms/Years.java b/api/src/main/java/org/apache/iceberg/transforms/Years.java new file mode 100644 index 000000000000..6c1eee578506 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/transforms/Years.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.transforms; + +import java.io.ObjectStreamException; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +class Years extends TimeTransform { + private static final Years INSTANCE = new Years<>(); + + @SuppressWarnings("unchecked") + static Years get() { + return (Years) INSTANCE; + } + + @Override + @SuppressWarnings("unchecked") + protected Transform toEnum(Type type) { + switch (type.typeId()) { + case DATE: + return (Transform) Dates.YEAR; + case TIMESTAMP: + return (Transform) Timestamps.YEAR; + default: + throw new IllegalArgumentException("Unsupported type: " + type); + } + } + + @Override + public Type getResultType(Type sourceType) { + return Types.IntegerType.get(); + } + + @Override + public boolean satisfiesOrderOf(Transform other) { + if (this == other) { + return true; + } + + if (other instanceof Timestamps) { + return Timestamps.YEAR.satisfiesOrderOf(other); + } else if (other instanceof Dates) { + return Dates.YEAR.satisfiesOrderOf(other); + } else if (other instanceof Years) { + return true; + } + + return false; + } + + @Override + public String toHumanString(Type alwaysInt, Integer value) { + return value != null ? TransformUtil.humanYear(value) : "null"; + } + + @Override + public String toString() { + return "year"; + } + + Object writeReplace() throws ObjectStreamException { + return SerializationProxies.YearsTransformProxy.get(); + } +} diff --git a/api/src/main/java/org/apache/iceberg/util/SerializableFunction.java b/api/src/main/java/org/apache/iceberg/util/SerializableFunction.java new file mode 100644 index 000000000000..0c191d905dac --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/util/SerializableFunction.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.util; + +import java.io.Serializable; +import java.util.function.Function; + +/** + * A concrete transform function that applies a transform to values of a certain type. + * + * @param Java class of source values + * @param Java class of transformed values + */ +public interface SerializableFunction extends Function, Serializable {} diff --git a/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java b/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java index f6d076a4465a..5e4ca1fb11be 100644 --- a/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java +++ b/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; @SuppressWarnings("checkstyle:HideUtilityClassConstructor") @@ -69,8 +70,12 @@ public class PartitionSpecTestBase { PartitionSpec.builderFor(SCHEMA).truncate("l", 10).build(), PartitionSpec.builderFor(SCHEMA).truncate("dec", 10).build(), PartitionSpec.builderFor(SCHEMA).truncate("s", 10).build(), - PartitionSpec.builderFor(SCHEMA).add(6, "dec_unsupported", "unsupported").build(), - PartitionSpec.builderFor(SCHEMA).add(6, 1111, "dec_unsupported", "unsupported").build(), + PartitionSpec.builderFor(SCHEMA) + .add(6, "dec_unsupported", Transforms.fromString("unsupported")) + .build(), + PartitionSpec.builderFor(SCHEMA) + .add(6, 1111, "dec_unsupported", Transforms.fromString("unsupported")) + .build(), PartitionSpec.builderFor(SCHEMA).alwaysNull("ts").build(), }; } diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java index d5a8767b2640..886a03415c8e 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java @@ -21,6 +21,7 @@ import org.apache.iceberg.TestHelpers.Row; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Test; @@ -33,17 +34,16 @@ public class TestPartitionPaths { Types.NestedField.optional(3, "ts", Types.TimestampType.withoutZone())); @Test - @SuppressWarnings("unchecked") public void testPartitionPath() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("ts").bucket("id", 10).build(); - Transform hour = spec.getFieldsBySourceId(3).get(0).transform(); - Transform bucket = spec.getFieldsBySourceId(1).get(0).transform(); + Transform hour = Transforms.hour(); + Transform bucket = Transforms.bucket(10); Literal ts = Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.withoutZone()); - Object tsHour = hour.apply(ts.value()); - Object idBucket = bucket.apply(1); + Object tsHour = hour.bind(Types.TimestampType.withoutZone()).apply(ts.value()); + Object idBucket = bucket.bind(Types.IntegerType.get()).apply(1); Row partition = Row.of(tsHour, idBucket); diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index 14c2a9ab1b6e..ed1d035cdc4c 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.junit.Assert; @@ -301,7 +302,7 @@ public void testAutoSettingPartitionFieldIds() { PartitionSpec.builderFor(SCHEMA) .year("ts", "custom_year") .bucket("ts", 4, "custom_bucket") - .add(1, "id_partition2", "bucket[4]") + .add(1, "id_partition2", Transforms.bucket(4)) .truncate("s", 1, "custom_truncate") .build(); @@ -316,9 +317,9 @@ public void testAutoSettingPartitionFieldIds() { public void testAddPartitionFieldsWithFieldIds() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA) - .add(1, 1005, "id_partition1", "bucket[4]") - .add(1, 1006, "id_partition2", "bucket[5]") - .add(1, 1002, "id_partition3", "bucket[6]") + .add(1, 1005, "id_partition1", Transforms.bucket(4)) + .add(1, 1006, "id_partition2", Transforms.bucket(5)) + .add(1, 1002, "id_partition3", Transforms.bucket(6)) .build(); Assert.assertEquals(1005, spec.fields().get(0).fieldId()); @@ -331,8 +332,8 @@ public void testAddPartitionFieldsWithFieldIds() { public void testAddPartitionFieldsWithAndWithoutFieldIds() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA) - .add(1, "id_partition2", "bucket[5]") - .add(1, 1005, "id_partition1", "bucket[4]") + .add(1, "id_partition2", Transforms.bucket(5)) + .add(1, 1005, "id_partition1", Transforms.bucket(4)) .truncate("s", 1, "custom_truncate") .build(); diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java index 62f6ff9b21a0..54e275080b2f 100644 --- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java +++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java @@ -237,6 +237,6 @@ private void assertInvalidateNaNThrows(Callable> callab } private UnboundTerm self(String name) { - return new UnboundTransform<>(ref(name), Transforms.identity(Types.DoubleType.get())); + return new UnboundTransform<>(ref(name), Transforms.identity()); } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java b/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java index fecf8ca97eca..04d8207be87d 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java @@ -32,6 +32,7 @@ import org.apache.iceberg.relocated.com.google.common.hash.HashFunction; import org.apache.iceberg.relocated.com.google.common.hash.Hashing; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.BucketUtil; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -61,86 +62,65 @@ public void initRandom() { @Test public void testSpecValues() { + Assert.assertEquals("Spec example: hash(true) = 1392991556", 1392991556, BucketUtil.hash(1)); + Assert.assertEquals("Spec example: hash(34) = 2017239379", 2017239379, BucketUtil.hash(34)); + Assert.assertEquals("Spec example: hash(34L) = 2017239379", 2017239379, BucketUtil.hash(34L)); Assert.assertEquals( - "Spec example: hash(true) = 1392991556", - 1392991556, - Bucket.get(Types.IntegerType.get(), 100).hash(1)); + "Spec example: hash(17.11F) = -142385009", -142385009, BucketUtil.hash(1.0F)); Assert.assertEquals( - "Spec example: hash(34) = 2017239379", - 2017239379, - Bucket.get(Types.IntegerType.get(), 100).hash(34)); - Assert.assertEquals( - "Spec example: hash(34L) = 2017239379", - 2017239379, - Bucket.get(Types.LongType.get(), 100).hash(34L)); - Assert.assertEquals( - "Spec example: hash(17.11F) = -142385009", - -142385009, - new Bucket.BucketFloat(100).hash(1.0F)); - Assert.assertEquals( - "Spec example: hash(17.11D) = -142385009", - -142385009, - new Bucket.BucketDouble(100).hash(1.0D)); + "Spec example: hash(17.11D) = -142385009", -142385009, BucketUtil.hash(1.0D)); Assert.assertEquals( "Spec example: hash(decimal2(14.20)) = -500754589", -500754589, - Bucket.get(Types.DecimalType.of(9, 2), 100).hash(new BigDecimal("14.20"))); + BucketUtil.hash(new BigDecimal("14.20"))); Assert.assertEquals( "Spec example: hash(decimal2(14.20)) = -500754589", -500754589, - Bucket.get(Types.DecimalType.of(9, 2), 100).hash(new BigDecimal("14.20"))); + BucketUtil.hash(new BigDecimal("14.20"))); Literal date = Literal.of("2017-11-16").to(Types.DateType.get()); Assert.assertEquals( - "Spec example: hash(2017-11-16) = -653330422", - -653330422, - Bucket.get(Types.DateType.get(), 100).hash(date.value())); + "Spec example: hash(2017-11-16) = -653330422", -653330422, BucketUtil.hash(date.value())); Literal timeValue = Literal.of("22:31:08").to(Types.TimeType.get()); Assert.assertEquals( "Spec example: hash(22:31:08) = -662762989", -662762989, - Bucket.get(Types.TimeType.get(), 100).hash(timeValue.value())); + BucketUtil.hash(timeValue.value())); Literal timestampVal = Literal.of("2017-11-16T22:31:08").to(Types.TimestampType.withoutZone()); Assert.assertEquals( "Spec example: hash(2017-11-16T22:31:08) = -2047944441", -2047944441, - Bucket.get(Types.TimestampType.withoutZone(), 100).hash(timestampVal.value())); + BucketUtil.hash(timestampVal.value())); Literal timestamptzVal = Literal.of("2017-11-16T14:31:08-08:00").to(Types.TimestampType.withZone()); Assert.assertEquals( "Spec example: hash(2017-11-16T14:31:08-08:00) = -2047944441", -2047944441, - Bucket.get(Types.TimestampType.withZone(), 100).hash(timestamptzVal.value())); + BucketUtil.hash(timestamptzVal.value())); Assert.assertEquals( - "Spec example: hash(\"iceberg\") = 1210000089", - 1210000089, - Bucket.get(Types.StringType.get(), 100).hash("iceberg")); + "Spec example: hash(\"iceberg\") = 1210000089", 1210000089, BucketUtil.hash("iceberg")); Assert.assertEquals( "Spec example: hash(\"iceberg\") = 1210000089", 1210000089, - Bucket.get(Types.StringType.get(), 100).hash(new Utf8("iceberg"))); + BucketUtil.hash(new Utf8("iceberg"))); Literal uuid = Literal.of("f79c3e09-677c-4bbd-a479-3f349cb785e7").to(Types.UUIDType.get()); Assert.assertEquals( "Spec example: hash(f79c3e09-677c-4bbd-a479-3f349cb785e7) = 1488055340", 1488055340, - Bucket.get(Types.UUIDType.get(), 100).hash(uuid.value())); + BucketUtil.hash(uuid.value())); ByteBuffer bytes = ByteBuffer.wrap(new byte[] {0, 1, 2, 3}); Assert.assertEquals( - "Spec example: hash([00 01 02 03]) = -188683207", - -188683207, - Bucket.get(Types.BinaryType.get(), 100).hash(bytes)); + "Spec example: hash([00 01 02 03]) = -188683207", -188683207, BucketUtil.hash(bytes)); Assert.assertEquals( - "Spec example: hash([00 01 02 03]) = -188683207", - -188683207, - Bucket.get(Types.BinaryType.get(), 100).hash(bytes)); + "Spec example: hash([00 01 02 03]) = -188683207", -188683207, BucketUtil.hash(bytes)); } @Test @@ -148,14 +128,12 @@ public void testInteger() { int num = testRandom.nextInt(); ByteBuffer buffer = ByteBuffer.allocate(8); buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.putLong((long) num); - - Bucket bucketFunc = Bucket.get(Types.IntegerType.get(), 100); + buffer.putLong(num); Assert.assertEquals( "Integer hash should match hash of little-endian bytes", hashBytes(buffer.array()), - bucketFunc.hash(num)); + BucketUtil.hash(num)); } @Test @@ -165,68 +143,30 @@ public void testLong() { buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putLong(num); - Bucket bucketFunc = Bucket.get(Types.LongType.get(), 100); - Assert.assertEquals( "Long hash should match hash of little-endian bytes", hashBytes(buffer.array()), - bucketFunc.hash(num)); + BucketUtil.hash(num)); } @Test public void testIntegerTypePromotion() { - Bucket bucketInts = Bucket.get(Types.IntegerType.get(), 100); - Bucket bucketLongs = Bucket.get(Types.LongType.get(), 100); - int randomInt = testRandom.nextInt(); Assert.assertEquals( "Integer and Long bucket results should match", - bucketInts.apply(randomInt), - bucketLongs.apply((long) randomInt)); - } - - @Test - public void testFloat() { - float num = testRandom.nextFloat(); - ByteBuffer buffer = ByteBuffer.allocate(8); - buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.putDouble((double) num); - - Bucket bucketFunc = new Bucket.BucketFloat(100); - - Assert.assertEquals( - "Float hash should match hash of little-endian bytes", - hashBytes(buffer.array()), - bucketFunc.hash(num)); - } - - @Test - public void testDouble() { - double num = testRandom.nextDouble(); - ByteBuffer buffer = ByteBuffer.allocate(8); - buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.putDouble(num); - - Bucket bucketFunc = new Bucket.BucketDouble(100); - - Assert.assertEquals( - "Double hash should match hash of little-endian bytes", - hashBytes(buffer.array()), - bucketFunc.hash(num)); + BucketUtil.hash(randomInt), + BucketUtil.hash((long) randomInt)); } @Test public void testFloatTypePromotion() { - Bucket bucketFloats = new Bucket.BucketFloat(100); - Bucket bucketDoubles = new Bucket.BucketDouble(100); - float randomFloat = testRandom.nextFloat(); Assert.assertEquals( "Float and Double bucket results should match", - bucketFloats.apply(randomFloat), - bucketDoubles.apply((double) randomFloat)); + BucketUtil.hash(randomFloat), + BucketUtil.hash((double) randomFloat)); } @Test @@ -235,12 +175,10 @@ public void testDecimal() { BigDecimal decimal = BigDecimal.valueOf(num); byte[] unscaledBytes = decimal.unscaledValue().toByteArray(); - Bucket bucketFunc = Bucket.get(Types.DecimalType.of(9, 2), 100); - Assert.assertEquals( "Decimal hash should match hash of backing bytes", hashBytes(unscaledBytes), - bucketFunc.hash(decimal)); + BucketUtil.hash(decimal)); } @Test @@ -248,12 +186,10 @@ public void testString() { String string = "string to test murmur3 hash"; byte[] asBytes = string.getBytes(StandardCharsets.UTF_8); - Bucket bucketFunc = Bucket.get(Types.StringType.get(), 100); - Assert.assertEquals( "String hash should match hash of UTF-8 bytes", hashBytes(asBytes), - bucketFunc.hash(string)); + BucketUtil.hash(string)); } @Test @@ -263,12 +199,10 @@ public void testStringWithSurrogatePair() { "string has no surrogate pairs", string.length(), string.codePoints().count()); byte[] asBytes = string.getBytes(StandardCharsets.UTF_8); - Bucket bucketFunc = Bucket.get(Types.StringType.get(), 100); - Assert.assertEquals( "String hash should match hash of UTF-8 bytes", hashBytes(asBytes), - bucketFunc.hash(string)); + BucketUtil.hash(string)); } @Test @@ -276,10 +210,8 @@ public void testUtf8() { Utf8 utf8 = new Utf8("string to test murmur3 hash"); byte[] asBytes = utf8.toString().getBytes(StandardCharsets.UTF_8); - Bucket bucketFunc = Bucket.get(Types.StringType.get(), 100); - Assert.assertEquals( - "String hash should match hash of UTF-8 bytes", hashBytes(asBytes), bucketFunc.hash(utf8)); + "String hash should match hash of UTF-8 bytes", hashBytes(asBytes), BucketUtil.hash(utf8)); } @Test @@ -287,12 +219,10 @@ public void testByteBufferOnHeap() { byte[] bytes = randomBytes(128); ByteBuffer buffer = ByteBuffer.wrap(bytes, 5, 100); - Bucket bucketFunc = Bucket.get(Types.BinaryType.get(), 100); - Assert.assertEquals( "HeapByteBuffer hash should match hash for correct slice", hashBytes(bytes, 5, 100), - bucketFunc.hash(buffer)); + BucketUtil.hash(buffer)); // verify that the buffer was not modified Assert.assertEquals("Buffer position should not change", 5, buffer.position()); @@ -306,12 +236,10 @@ public void testByteBufferOnHeapArrayOffset() { ByteBuffer buffer = raw.slice(); Assert.assertEquals("Buffer arrayOffset should be 5", 5, buffer.arrayOffset()); - Bucket bucketFunc = Bucket.get(Types.BinaryType.get(), 100); - Assert.assertEquals( "HeapByteBuffer hash should match hash for correct slice", hashBytes(bytes, 5, 100), - bucketFunc.hash(buffer)); + BucketUtil.hash(buffer)); // verify that the buffer was not modified Assert.assertEquals("Buffer position should be 0", 0, buffer.position()); @@ -330,12 +258,10 @@ public void testByteBufferOffHeap() { buffer.put(bytes, 5, 100); buffer.reset(); - Bucket bucketFunc = Bucket.get(Types.BinaryType.get(), 100); - Assert.assertEquals( "DirectByteBuffer hash should match hash for correct slice", hashBytes(bytes, 5, 100), - bucketFunc.hash(buffer)); + BucketUtil.hash(buffer)); // verify that the buffer was not modified Assert.assertEquals("Buffer position should not change", 5, buffer.position()); @@ -347,12 +273,10 @@ public void testUUIDHash() { byte[] uuidBytes = randomBytes(16); UUID uuid = newUUID(uuidBytes); - Bucket bucketFunc = Bucket.get(Types.UUIDType.get(), 100); - Assert.assertEquals( "UUID hash should match hash of backing bytes", hashBytes(uuidBytes), - bucketFunc.hash(uuid)); + BucketUtil.hash(uuid)); } @Test @@ -361,7 +285,7 @@ public void testVerifiedIllegalNumBuckets() { "Should fail if numBucket is less than or equal to zero", IllegalArgumentException.class, "Invalid number of buckets: 0 (must be > 0)", - () -> Bucket.get(Types.IntegerType.get(), 0)); + () -> Bucket.get(0)); } private byte[] randomBytes(int length) { diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java index 25f10f06ecaf..34b5a19dfdad 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java @@ -61,19 +61,18 @@ public void assertProjectionStrict( Assert.assertNotEquals( "Strict projection never runs for IN", Expression.Operation.IN, predicate.op()); - Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform(); if (predicate.op() == Expression.Operation.NOT_IN) { Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString(v)) + .map(String::valueOf) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString(literal.value()); + Literal literal = predicate.literal(); + String output = String.valueOf(literal.value()); Assert.assertEquals(expectedLiteral, output); } } @@ -103,19 +102,18 @@ public void assertProjectionInclusive( Assert.assertNotEquals( "Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); - Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform(); if (predicate.op() == Expression.Operation.IN) { Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString(v)) + .map(String::valueOf) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString(literal.value()); + Literal literal = predicate.literal(); + String output = String.valueOf(literal.value()); Assert.assertEquals(expectedLiteral, output); } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java index 39829221d6b5..1df5beeaa79c 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java @@ -26,7 +26,8 @@ public class TestDates { @Test - public void testDateTransform() { + @SuppressWarnings("deprecation") + public void testDeprecatedDateTransform() { Types.DateType type = Types.DateType.get(); Literal date = Literal.of("2017-12-01").to(type); Literal pd = Literal.of("1970-01-01").to(type); @@ -48,28 +49,57 @@ public void testDateTransform() { Assert.assertEquals("Should produce -1", -1, (int) days.apply(nd.value())); } + @Test + public void testDateTransform() { + Types.DateType type = Types.DateType.get(); + Literal date = Literal.of("2017-12-01").to(type); + Literal pd = Literal.of("1970-01-01").to(type); + Literal nd = Literal.of("1969-12-31").to(type); + + Transform years = Transforms.year(); + Assert.assertEquals( + "Should produce 2017 - 1970 = 47", 47, (int) years.bind(type).apply(date.value())); + Assert.assertEquals( + "Should produce 1970 - 1970 = 0", 0, (int) years.bind(type).apply(pd.value())); + Assert.assertEquals( + "Should produce 1969 - 1970 = -1", -1, (int) years.bind(type).apply(nd.value())); + + Transform months = Transforms.month(); + Assert.assertEquals( + "Should produce 47 * 12 + 11 = 575", 575, (int) months.bind(type).apply(date.value())); + Assert.assertEquals( + "Should produce 0 * 12 + 0 = 0", 0, (int) months.bind(type).apply(pd.value())); + Assert.assertEquals("Should produce -1", -1, (int) months.bind(type).apply(nd.value())); + + Transform days = Transforms.day(); + Assert.assertEquals("Should produce 17501", 17501, (int) days.bind(type).apply(date.value())); + Assert.assertEquals( + "Should produce 0 * 365 + 0 = 0", 0, (int) days.bind(type).apply(pd.value())); + Assert.assertEquals("Should produce -1", -1, (int) days.bind(type).apply(nd.value())); + } + @Test public void testDateToHumanString() { Types.DateType type = Types.DateType.get(); Literal date = Literal.of("2017-12-01").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "2017", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "2017-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "2017-12-01", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); } @Test @@ -77,23 +107,23 @@ public void testNegativeDateToHumanString() { Types.DateType type = Types.DateType.get(); Literal date = Literal.of("1969-12-30").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1969", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1969-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-30", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); } @Test @@ -101,23 +131,23 @@ public void testDateToHumanStringLowerBound() { Types.DateType type = Types.DateType.get(); Literal date = Literal.of("1970-01-01").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1970", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1970-01", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1970-01-01", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); } @Test @@ -125,23 +155,23 @@ public void testNegativeDateToHumanStringLowerBound() { Types.DateType type = Types.DateType.get(); Literal date = Literal.of("1969-01-01").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1969", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1969-01", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1969-01-01", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); } @Test @@ -149,49 +179,49 @@ public void testNegativeDateToHumanStringUpperBound() { Types.DateType type = Types.DateType.get(); Literal date = Literal.of("1969-12-31").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1969", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1969-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-31", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); } @Test public void testNullHumanString() { Types.DateType type = Types.DateType.get(); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.year(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.year().toHumanString(type, null)); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.month(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.month().toHumanString(type, null)); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.day(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.day().toHumanString(type, null)); } @Test public void testDatesReturnType() { Types.DateType type = Types.DateType.get(); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Type yearResultType = year.getResultType(type); Assert.assertEquals(Types.IntegerType.get(), yearResultType); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Type monthResultType = month.getResultType(type); Assert.assertEquals(Types.IntegerType.get(), monthResultType); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Type dayResultType = day.getResultType(type); Assert.assertEquals(Types.DateType.get(), dayResultType); } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java index a6abdc8f93eb..5106be0ca648 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java @@ -38,6 +38,7 @@ import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Test; @@ -46,6 +47,7 @@ public class TestDatesProjection { private static final Types.DateType TYPE = Types.DateType.get(); private static final Schema SCHEMA = new Schema(optional(1, "date", TYPE)); + @SuppressWarnings("unchecked") public void assertProjectionStrict( PartitionSpec spec, UnboundPredicate filter, @@ -53,26 +55,28 @@ public void assertProjectionStrict( String expectedLiteral) { Expression projection = Projections.strict(spec).project(filter); - UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals( "Strict projection never runs for IN", Expression.Operation.IN, predicate.op()); - Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); + Transform transform = + (Transform) spec.getFieldsBySourceId(1).get(0).transform(); + Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type(); if (predicate.op() == Expression.Operation.NOT_IN) { - Iterable values = Iterables.transform(predicate.literals(), Literal::value); + Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString((Integer) v)) + .map(v -> transform.toHumanString(type, v)) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString((int) literal.value()); + Literal literal = predicate.literal(); + String output = transform.toHumanString(type, literal.value()); Assert.assertEquals(expectedLiteral, output); } } @@ -91,32 +95,35 @@ public void assertProjectionInclusiveValue( Assert.assertEquals(expectedOp, projection.op()); } + @SuppressWarnings("unchecked") public void assertProjectionInclusive( PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.inclusive(spec).project(filter); - UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals( "Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); - Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); + Transform transform = + (Transform) spec.getFieldsBySourceId(1).get(0).transform(); + Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type(); if (predicate.op() == Expression.Operation.IN) { - Iterable values = Iterables.transform(predicate.literals(), Literal::value); + Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString((Integer) v)) + .map(v -> transform.toHumanString(type, v)) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString((int) literal.value()); + Literal literal = predicate.literal(); + String output = transform.toHumanString(type, literal.value()); Assert.assertEquals(expectedLiteral, output); } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java b/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java index e2e3680c7b5d..a14b7c861810 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java @@ -29,59 +29,62 @@ public class TestIdentity { @Test public void testNullHumanString() { Types.LongType longType = Types.LongType.get(); - Transform identity = Transforms.identity(longType); + Transform identity = Transforms.identity(); - Assert.assertEquals("Should produce \"null\" for null", "null", identity.toHumanString(null)); + Assert.assertEquals( + "Should produce \"null\" for null", "null", identity.toHumanString(longType, null)); } @Test public void testBinaryHumanString() { Types.BinaryType binary = Types.BinaryType.get(); - Transform identity = Transforms.identity(binary); + Transform identity = Transforms.identity(); Assert.assertEquals( "Should base64-encode binary", "AQID", - identity.toHumanString(ByteBuffer.wrap(new byte[] {1, 2, 3}))); + identity.toHumanString(binary, ByteBuffer.wrap(new byte[] {1, 2, 3}))); } @Test public void testFixedHumanString() { Types.FixedType fixed3 = Types.FixedType.ofLength(3); - Transform identity = Transforms.identity(fixed3); + Transform identity = Transforms.identity(); Assert.assertEquals( - "Should base64-encode binary", "AQID", identity.toHumanString(new byte[] {1, 2, 3})); + "Should base64-encode binary", + "AQID", + identity.toHumanString(fixed3, new byte[] {1, 2, 3})); } @Test public void testDateHumanString() { Types.DateType date = Types.DateType.get(); - Transform identity = Transforms.identity(date); + Transform identity = Transforms.identity(); String dateString = "2017-12-01"; Literal dateLit = Literal.of(dateString).to(date); Assert.assertEquals( - "Should produce identical date", dateString, identity.toHumanString(dateLit.value())); + "Should produce identical date", dateString, identity.toHumanString(date, dateLit.value())); } @Test public void testTimeHumanString() { Types.TimeType time = Types.TimeType.get(); - Transform identity = Transforms.identity(time); + Transform identity = Transforms.identity(); String timeString = "10:12:55.038194"; Literal timeLit = Literal.of(timeString).to(time); Assert.assertEquals( - "Should produce identical time", timeString, identity.toHumanString(timeLit.value())); + "Should produce identical time", timeString, identity.toHumanString(time, timeLit.value())); } @Test public void testTimestampWithZoneHumanString() { Types.TimestampType timestamptz = Types.TimestampType.withZone(); - Transform identity = Transforms.identity(timestamptz); + Transform identity = Transforms.identity(); Literal ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(timestamptz); @@ -89,13 +92,13 @@ public void testTimestampWithZoneHumanString() { Assert.assertEquals( "Should produce timestamp with time zone adjusted to UTC", "2017-12-01T18:12:55.038194Z", - identity.toHumanString(ts.value())); + identity.toHumanString(timestamptz, ts.value())); } @Test public void testTimestampWithoutZoneHumanString() { Types.TimestampType timestamp = Types.TimestampType.withoutZone(); - Transform identity = Transforms.identity(timestamp); + Transform identity = Transforms.identity(); String tsString = "2017-12-01T10:12:55.038194"; Literal ts = Literal.of(tsString).to(timestamp); @@ -104,35 +107,38 @@ public void testTimestampWithoutZoneHumanString() { Assert.assertEquals( "Should produce identical timestamp without time zone", tsString, - identity.toHumanString(ts.value())); + identity.toHumanString(timestamp, ts.value())); } @Test public void testLongToHumanString() { Types.LongType longType = Types.LongType.get(); - Transform identity = Transforms.identity(longType); + Transform identity = Transforms.identity(); Assert.assertEquals( - "Should use Long toString", "-1234567890000", identity.toHumanString(-1234567890000L)); + "Should use Long toString", + "-1234567890000", + identity.toHumanString(longType, -1234567890000L)); } @Test public void testStringToHumanString() { Types.StringType string = Types.StringType.get(); - Transform identity = Transforms.identity(string); + Transform identity = Transforms.identity(); String withSlash = "a/b/c=d"; - Assert.assertEquals("Should not modify Strings", withSlash, identity.toHumanString(withSlash)); + Assert.assertEquals( + "Should not modify Strings", withSlash, identity.toHumanString(string, withSlash)); } @Test public void testBigDecimalToHumanString() { Types.DecimalType decimal = Types.DecimalType.of(9, 2); - Transform identity = Transforms.identity(decimal); + Transform identity = Transforms.identity(); String decimalString = "-1.50"; BigDecimal bigDecimal = new BigDecimal(decimalString); Assert.assertEquals( - "Should not modify Strings", decimalString, identity.toHumanString(bigDecimal)); + "Should not modify Strings", decimalString, identity.toHumanString(decimal, bigDecimal)); } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java b/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java index 54a362a9d337..88c762797bcb 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java @@ -93,7 +93,7 @@ public void testTruncateProjections() { @Test public void testTruncateStringWhenProjectedPredicateTermIsLongerThanWidth() { - Truncate trunc = Truncate.get(Types.StringType.get(), 2); + Truncate trunc = Truncate.get(2); UnboundPredicate expr = notStartsWith(COLUMN, "abcde"); BoundPredicate boundExpr = (BoundPredicate) Binder.bind(SCHEMA.asStruct(), expr, false); @@ -127,7 +127,7 @@ public void testTruncateStringWhenProjectedPredicateTermIsLongerThanWidth() { @Test public void testTruncateStringWhenProjectedPredicateTermIsShorterThanWidth() { - Truncate trunc = Truncate.get(Types.StringType.get(), 16); + Truncate trunc = Truncate.get(16); UnboundPredicate expr = notStartsWith(COLUMN, "ab"); BoundPredicate boundExpr = (BoundPredicate) Binder.bind(SCHEMA.asStruct(), expr, false); @@ -153,7 +153,7 @@ public void testTruncateStringWhenProjectedPredicateTermIsShorterThanWidth() { @Test public void testTruncateStringWhenProjectedPredicateTermIsEqualToWidth() { - Truncate trunc = Truncate.get(Types.StringType.get(), 7); + Truncate trunc = Truncate.get(7); UnboundPredicate expr = notStartsWith(COLUMN, "abcdefg"); BoundPredicate boundExpr = (BoundPredicate) Binder.bind(SCHEMA.asStruct(), expr, false); @@ -223,6 +223,7 @@ private void assertProjectionStrict( assertProjection(spec, expectedLiteral, projection, expectedOp); } + @SuppressWarnings("unchecked") private void assertProjection( PartitionSpec spec, String expectedLiteral, @@ -232,7 +233,7 @@ private void assertProjection( Literal literal = predicate.literal(); Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform(); - String output = transform.toHumanString((String) literal.value()); + String output = transform.toHumanString(Types.StringType.get(), (String) literal.value()); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertEquals(expectedLiteral, output); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java b/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java index dc46964c4686..c2bb855cfd0c 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java @@ -33,6 +33,7 @@ import static org.apache.iceberg.expressions.Expressions.notNaN; import static org.apache.iceberg.expressions.Expressions.or; +import java.util.function.Function; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.TestHelpers.Row; @@ -40,7 +41,6 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.Literal; -import org.apache.iceberg.expressions.Predicate; import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.types.Types; @@ -210,10 +210,10 @@ public void testInTimestamp() { PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build(); - Transform day = spec.getFieldsBySourceId(50).get(0).transform(); - Integer tsDay = (Integer) day.apply(date20191201); + Function day = Transforms.day().bind(Types.TimestampType.withoutZone()); + Integer tsDay = day.apply(date20191201); - Predicate pred = in("ts", date20191201, date20191202); + Expression pred = in("ts", date20191201, date20191202); ResidualEvaluator resEval = ResidualEvaluator.of(spec, pred, true); Expression residual = resEval.residualFor(Row.of(tsDay)); @@ -318,10 +318,10 @@ public void testNotInTimestamp() { PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build(); - Transform day = spec.getFieldsBySourceId(50).get(0).transform(); - Integer tsDay = (Integer) day.apply(date20191201); + Function day = Transforms.day().bind(Types.TimestampType.withoutZone()); + Integer tsDay = day.apply(date20191201); - Predicate pred = notIn("ts", date20191201, date20191202); + Expression pred = notIn("ts", date20191201, date20191202); ResidualEvaluator resEval = ResidualEvaluator.of(spec, pred, true); Expression residual = resEval.residualFor(Row.of(tsDay)); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java b/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java index 90143aa7e856..8ceeb9195253 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java @@ -61,8 +61,9 @@ public void testTruncateProjections() { } @Test + @SuppressWarnings("unchecked") public void testTruncateString() { - Truncate trunc = Truncate.get(Types.StringType.get(), 2); + Truncate trunc = Truncate.get(2); Expression expr = startsWith(COLUMN, "abcde"); BoundPredicate boundExpr = (BoundPredicate) Binder.bind(SCHEMA.asStruct(), expr, false); @@ -93,16 +94,17 @@ private void assertProjectionStrict( assertProjection(spec, expectedLiteral, projection, expectedOp); } + @SuppressWarnings("unchecked") private void assertProjection( PartitionSpec spec, String expectedLiteral, Expression projection, Expression.Operation expectedOp) { UnboundPredicate predicate = assertAndUnwrapUnbound(projection); - Literal literal = predicate.literal(); + Literal literal = predicate.literal(); Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform(); - String output = transform.toHumanString((String) literal.value()); + String output = transform.toHumanString(Types.StringType.get(), (String) literal.value()); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertEquals(expectedLiteral, output); diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java index c5ff6788dbd2..aefa98ed260c 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java @@ -26,7 +26,8 @@ public class TestTimestamps { @Test - public void testTimestampTransform() { + @SuppressWarnings("deprecation") + public void testDeprecatedTimestampTransform() { Types.TimestampType type = Types.TimestampType.withoutZone(); Literal ts = Literal.of("2017-12-01T10:12:55.038194").to(type); Literal pts = Literal.of("1970-01-01T00:00:01.000001").to(type); @@ -53,34 +54,70 @@ public void testTimestampTransform() { Assert.assertEquals("Should produce -1", -1, (int) hours.apply(nts.value())); } + @Test + public void testTimestampTransform() { + Types.TimestampType type = Types.TimestampType.withoutZone(); + Literal ts = Literal.of("2017-12-01T10:12:55.038194").to(type); + Literal pts = Literal.of("1970-01-01T00:00:01.000001").to(type); + Literal nts = Literal.of("1969-12-31T23:59:58.999999").to(type); + + Transform years = Transforms.year(); + Assert.assertEquals( + "Should produce 2017 - 1970 = 47", 47, (int) years.bind(type).apply(ts.value())); + Assert.assertEquals( + "Should produce 1970 - 1970 = 0", 0, (int) years.bind(type).apply(pts.value())); + Assert.assertEquals( + "Should produce 1969 - 1970 = -1", -1, (int) years.bind(type).apply(nts.value())); + + Transform months = Transforms.month(); + Assert.assertEquals( + "Should produce 47 * 12 + 11 = 575", 575, (int) months.bind(type).apply(ts.value())); + Assert.assertEquals( + "Should produce 0 * 12 + 0 = 0", 0, (int) months.bind(type).apply(pts.value())); + Assert.assertEquals("Should produce -1", -1, (int) months.bind(type).apply(nts.value())); + + Transform days = Transforms.day(); + Assert.assertEquals("Should produce 17501", 17501, (int) days.bind(type).apply(ts.value())); + Assert.assertEquals( + "Should produce 0 * 365 + 0 = 0", 0, (int) days.bind(type).apply(pts.value())); + Assert.assertEquals("Should produce -1", -1, (int) days.bind(type).apply(nts.value())); + + Transform hours = Transforms.hour(); + Assert.assertEquals( + "Should produce 17501 * 24 + 10", 420034, (int) hours.bind(type).apply(ts.value())); + Assert.assertEquals( + "Should produce 0 * 24 + 0 = 0", 0, (int) hours.bind(type).apply(pts.value())); + Assert.assertEquals("Should produce -1", -1, (int) hours.bind(type).apply(nts.value())); + } + @Test public void testTimestampWithoutZoneToHumanString() { Types.TimestampType type = Types.TimestampType.withoutZone(); Literal date = Literal.of("2017-12-01T10:12:55.038194").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "2017", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "2017-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "2017-12-01", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); - Transform hour = Transforms.hour(type); + Transform hour = Transforms.hour(); Assert.assertEquals( "Should produce the correct Human string", "2017-12-01-10", - hour.toHumanString(hour.apply(date.value()))); + hour.toHumanString(type, hour.bind(type).apply(date.value()))); } @Test @@ -88,29 +125,29 @@ public void testNegativeTimestampWithoutZoneToHumanString() { Types.TimestampType type = Types.TimestampType.withoutZone(); Literal date = Literal.of("1969-12-30T10:12:55.038194").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1969", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1969-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-30", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); - Transform hour = Transforms.hour(type); + Transform hour = Transforms.hour(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-30-10", - hour.toHumanString(hour.apply(date.value()))); + hour.toHumanString(type, hour.bind(type).apply(date.value()))); } @Test @@ -118,29 +155,29 @@ public void testNegativeTimestampWithoutZoneToHumanStringLowerBound() { Types.TimestampType type = Types.TimestampType.withoutZone(); Literal date = Literal.of("1969-12-30T00:00:00.000000").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1969", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1969-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-30", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); - Transform hour = Transforms.hour(type); + Transform hour = Transforms.hour(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-30-00", - hour.toHumanString(hour.apply(date.value()))); + hour.toHumanString(type, hour.bind(type).apply(date.value()))); } @Test @@ -148,29 +185,29 @@ public void testNegativeTimestampWithoutZoneToHumanStringUpperBound() { Types.TimestampType type = Types.TimestampType.withoutZone(); Literal date = Literal.of("1969-12-31T23:59:59.999999").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "1969", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "1969-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-31", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); - Transform hour = Transforms.hour(type); + Transform hour = Transforms.hour(); Assert.assertEquals( "Should produce the correct Human string", "1969-12-31-23", - hour.toHumanString(hour.apply(date.value()))); + hour.toHumanString(type, hour.bind(type).apply(date.value()))); } @Test @@ -178,62 +215,62 @@ public void testTimestampWithZoneToHumanString() { Types.TimestampType type = Types.TimestampType.withZone(); Literal date = Literal.of("2017-12-01T10:12:55.038194-08:00").to(type); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Assert.assertEquals( "Should produce the correct Human string", "2017", - year.toHumanString(year.apply(date.value()))); + year.toHumanString(type, year.bind(type).apply(date.value()))); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Assert.assertEquals( "Should produce the correct Human string", "2017-12", - month.toHumanString(month.apply(date.value()))); + month.toHumanString(type, month.bind(type).apply(date.value()))); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Assert.assertEquals( "Should produce the correct Human string", "2017-12-01", - day.toHumanString(day.apply(date.value()))); + day.toHumanString(type, day.bind(type).apply(date.value()))); // the hour is 18 because the value is always UTC - Transform hour = Transforms.hour(type); + Transform hour = Transforms.hour(); Assert.assertEquals( "Should produce the correct Human string", "2017-12-01-18", - hour.toHumanString(hour.apply(date.value()))); + hour.toHumanString(type, hour.bind(type).apply(date.value()))); } @Test public void testNullHumanString() { Types.TimestampType type = Types.TimestampType.withZone(); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.year(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.year().toHumanString(type, null)); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.month(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.month().toHumanString(type, null)); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.day(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.day().toHumanString(type, null)); Assert.assertEquals( - "Should produce \"null\" for null", "null", Transforms.hour(type).toHumanString(null)); + "Should produce \"null\" for null", "null", Transforms.hour().toHumanString(type, null)); } @Test public void testTimestampsReturnType() { Types.TimestampType type = Types.TimestampType.withZone(); - Transform year = Transforms.year(type); + Transform year = Transforms.year(); Type yearResultType = year.getResultType(type); Assert.assertEquals(Types.IntegerType.get(), yearResultType); - Transform month = Transforms.month(type); + Transform month = Transforms.month(); Type monthResultType = month.getResultType(type); Assert.assertEquals(Types.IntegerType.get(), monthResultType); - Transform day = Transforms.day(type); + Transform day = Transforms.day(); Type dayResultType = day.getResultType(type); Assert.assertEquals(Types.DateType.get(), dayResultType); - Transform hour = Transforms.hour(type); + Transform hour = Transforms.hour(); Type hourResultType = hour.getResultType(type); Assert.assertEquals(Types.IntegerType.get(), hourResultType); } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java index 3ed5f9bac84a..3da034d50162 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java @@ -38,6 +38,7 @@ import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Test; @@ -46,6 +47,7 @@ public class TestTimestampsProjection { private static final Types.TimestampType TYPE = Types.TimestampType.withoutZone(); private static final Schema SCHEMA = new Schema(optional(1, "timestamp", TYPE)); + @SuppressWarnings("unchecked") public void assertProjectionStrict( PartitionSpec spec, UnboundPredicate filter, @@ -53,26 +55,28 @@ public void assertProjectionStrict( String expectedLiteral) { Expression projection = Projections.strict(spec).project(filter); - UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals( "Strict projection never runs for IN", Expression.Operation.IN, predicate.op()); - Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform(); + Transform transform = + (Transform) spec.getFieldsBySourceId(1).get(0).transform(); + Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type(); if (predicate.op() == Expression.Operation.NOT_IN) { - Iterable values = Iterables.transform(predicate.literals(), Literal::value); + Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString((Integer) v)) + .map(v -> transform.toHumanString(type, v)) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString((int) literal.value()); + Literal literal = predicate.literal(); + String output = transform.toHumanString(type, literal.value()); Assert.assertEquals(expectedLiteral, output); } } @@ -91,32 +95,35 @@ public void assertProjectionInclusiveValue( Assert.assertEquals(expectedOp, projection.op()); } + @SuppressWarnings("unchecked") public void assertProjectionInclusive( PartitionSpec spec, UnboundPredicate filter, Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.inclusive(spec).project(filter); - UnboundPredicate predicate = assertAndUnwrapUnbound(projection); + UnboundPredicate predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals( "Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); - Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform(); + Transform transform = + (Transform) spec.getFieldsBySourceId(1).get(0).transform(); + Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type(); if (predicate.op() == Expression.Operation.IN) { - Iterable values = Iterables.transform(predicate.literals(), Literal::value); + Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString((Integer) v)) + .map(v -> transform.toHumanString(type, v)) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString((int) literal.value()); + Literal literal = predicate.literal(); + String output = transform.toHumanString(type, literal.value()); Assert.assertEquals(expectedLiteral, output); } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java b/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java new file mode 100644 index 000000000000..2c04779c676b --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.transforms; + +import org.apache.iceberg.TestHelpers; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.SerializableFunction; +import org.junit.Assert; +import org.junit.Test; + +public class TestTransformSerialization { + @Test + public void testFunctionSerialization() throws Exception { + Type[] types = + new Type[] { + Types.BooleanType.get(), + Types.IntegerType.get(), + Types.LongType.get(), + Types.FloatType.get(), + Types.DoubleType.get(), + Types.StringType.get(), + Types.DateType.get(), + Types.TimeType.get(), + Types.TimestampType.withoutZone(), + Types.TimestampType.withoutZone(), + Types.BinaryType.get(), + Types.FixedType.ofLength(4), + Types.DecimalType.of(9, 4), + Types.UUIDType.get(), + }; + + Transform[] transforms = + new Transform[] { + Transforms.identity(), + Transforms.bucket(1024), + Transforms.year(), + Transforms.month(), + Transforms.day(), + Transforms.hour(), + Transforms.truncate(16) + }; + + for (Type type : types) { + for (Transform transform : transforms) { + Assert.assertEquals(transform, TestHelpers.roundTripSerialize(transform)); + + if (transform.canTransform(type)) { + SerializableFunction func = transform.bind(type); + Assert.assertTrue(func.getClass().isInstance(TestHelpers.roundTripSerialize(func))); + } + } + } + } +} diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java b/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java index 6682086a6e1f..4324998bd476 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java @@ -20,15 +20,32 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.function.Function; import org.apache.iceberg.AssertHelpers; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Test; public class TestTruncate { + @Test + public void testDeprecatedTruncateInteger() { + Truncate trunc = Truncate.get(Types.IntegerType.get(), 10); + Assert.assertEquals(0, (int) trunc.apply(0)); + Assert.assertEquals(0, (int) trunc.apply(1)); + Assert.assertEquals(0, (int) trunc.apply(5)); + Assert.assertEquals(0, (int) trunc.apply(9)); + Assert.assertEquals(10, (int) trunc.apply(10)); + Assert.assertEquals(10, (int) trunc.apply(11)); + Assert.assertEquals(-10, (int) trunc.apply(-1)); + Assert.assertEquals(-10, (int) trunc.apply(-5)); + Assert.assertEquals(-10, (int) trunc.apply(-10)); + Assert.assertEquals(-20, (int) trunc.apply(-11)); + } + @Test public void testTruncateInteger() { - Truncate trunc = Truncate.get(Types.IntegerType.get(), 10); + Function trunc = Truncate.get(10).bind(Types.IntegerType.get()); Assert.assertEquals(0, (int) trunc.apply(0)); Assert.assertEquals(0, (int) trunc.apply(1)); Assert.assertEquals(0, (int) trunc.apply(5)); @@ -43,7 +60,7 @@ public void testTruncateInteger() { @Test public void testTruncateLong() { - Truncate trunc = Truncate.get(Types.LongType.get(), 10); + Function trunc = Truncate.get(10).bind(Types.LongType.get()); Assert.assertEquals(0L, (long) trunc.apply(0L)); Assert.assertEquals(0L, (long) trunc.apply(1L)); Assert.assertEquals(0L, (long) trunc.apply(5L)); @@ -59,7 +76,7 @@ public void testTruncateLong() { @Test public void testTruncateDecimal() { // decimal truncation works by applying the decimal scale to the width: 10 scale 2 = 0.10 - Truncate trunc = Truncate.get(Types.DecimalType.of(9, 2), 10); + Function trunc = Truncate.get(10).bind(Types.DecimalType.of(9, 2)); Assert.assertEquals(new BigDecimal("12.30"), trunc.apply(new BigDecimal("12.34"))); Assert.assertEquals(new BigDecimal("12.30"), trunc.apply(new BigDecimal("12.30"))); Assert.assertEquals(new BigDecimal("12.20"), trunc.apply(new BigDecimal("12.29"))); @@ -69,7 +86,7 @@ public void testTruncateDecimal() { @Test public void testTruncateString() { - Truncate trunc = Truncate.get(Types.StringType.get(), 5); + Function trunc = Truncate.get(5).bind(Types.StringType.get()); Assert.assertEquals( "Should truncate strings longer than length", "abcde", trunc.apply("abcdefg")); Assert.assertEquals("Should not pad strings shorter than length", "abc", trunc.apply("abc")); @@ -77,16 +94,16 @@ public void testTruncateString() { } @Test - public void testTruncateByteBuffer() throws Exception { - Truncate trunc = Truncate.get(Types.BinaryType.get(), 4); + public void testTruncateByteBuffer() { + Function trunc = Truncate.get(4).bind(Types.BinaryType.get()); Assert.assertEquals( "Should truncate binary longer than length", - ByteBuffer.wrap("abcd".getBytes("UTF-8")), - trunc.apply(ByteBuffer.wrap("abcdefg".getBytes("UTF-8")))); + ByteBuffer.wrap("abcd".getBytes(StandardCharsets.UTF_8)), + trunc.apply(ByteBuffer.wrap("abcdefg".getBytes(StandardCharsets.UTF_8)))); Assert.assertEquals( "Should not pad binary shorter than length", - ByteBuffer.wrap("abc".getBytes("UTF-8")), - trunc.apply(ByteBuffer.wrap("abc".getBytes("UTF-8")))); + ByteBuffer.wrap("abc".getBytes(StandardCharsets.UTF_8)), + trunc.apply(ByteBuffer.wrap("abc".getBytes(StandardCharsets.UTF_8)))); } @Test @@ -95,6 +112,6 @@ public void testVerifiedIllegalWidth() { "Should fail if width is less than or equal to zero", IllegalArgumentException.class, "Invalid truncate width: 0 (must be > 0)", - () -> Truncate.get(Types.IntegerType.get(), 0)); + () -> Truncate.get(0)); } } diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java index 4cbabd4213d7..cc0ae0040a5a 100644 --- a/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java +++ b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java @@ -40,12 +40,14 @@ import org.apache.iceberg.expressions.UnboundPredicate; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Test; public class TestTruncatesProjection { + @SuppressWarnings("unchecked") public void assertProjectionStrict( PartitionSpec spec, UnboundPredicate filter, @@ -60,19 +62,21 @@ public void assertProjectionStrict( Assert.assertNotEquals( "Strict projection never runs for IN", Expression.Operation.IN, predicate.op()); - Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform(); + Transform transform = + (Transform) spec.getFieldsBySourceId(1).get(0).transform(); + Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type(); if (predicate.op() == Expression.Operation.NOT_IN) { Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString(v)) + .map(v -> transform.toHumanString(type, v)) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString(literal.value()); + Literal literal = predicate.literal(); + String output = transform.toHumanString(type, literal.value()); Assert.assertEquals(expectedLiteral, output); } } @@ -91,6 +95,7 @@ public void assertProjectionInclusiveValue( Assert.assertEquals(projection.op(), expectedOp); } + @SuppressWarnings("unchecked") public void assertProjectionInclusive( PartitionSpec spec, UnboundPredicate filter, @@ -104,19 +109,21 @@ public void assertProjectionInclusive( Assert.assertNotEquals( "Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); - Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform(); + Transform transform = + (Transform) spec.getFieldsBySourceId(1).get(0).transform(); + Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type(); if (predicate.op() == Expression.Operation.IN) { Iterable values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream() .sorted() - .map(v -> transform.toHumanString(v)) + .map(v -> transform.toHumanString(type, v)) .collect(Collectors.toList()) .toString(); Assert.assertEquals(expectedLiteral, actual); } else { - Literal literal = predicate.literal(); - String output = transform.toHumanString(literal.value()); + Literal literal = predicate.literal(); + String output = transform.toHumanString(type, literal.value()); Assert.assertEquals(expectedLiteral, output); } } diff --git a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java index 5bf8215dc986..31890843f238 100644 --- a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java +++ b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java @@ -25,6 +25,7 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.transforms.Transforms; /** * Base class for metadata tables. @@ -63,7 +64,9 @@ protected BaseMetadataTable(TableOperations ops, Table table, String name) { static PartitionSpec transformSpec(Schema metadataTableSchema, PartitionSpec spec) { PartitionSpec.Builder identitySpecBuilder = PartitionSpec.builderFor(metadataTableSchema).checkConflicts(false); - spec.fields().forEach(pf -> identitySpecBuilder.add(pf.fieldId(), pf.name(), "identity")); + for (PartitionField field : spec.fields()) { + identitySpecBuilder.add(field.fieldId(), field.name(), Transforms.identity()); + } return identitySpecBuilder.build(); } diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java index 6f688848f761..11c74a045edd 100644 --- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java +++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java @@ -343,7 +343,7 @@ public void commit() { private Transform toTransform(BoundTerm term) { if (term instanceof BoundReference) { - return Transforms.identity(term.type()); + return Transforms.identity(); } else if (term instanceof BoundTransform) { return ((BoundTransform) term).transform(); } else { diff --git a/core/src/main/java/org/apache/iceberg/LocationProviders.java b/core/src/main/java/org/apache/iceberg/LocationProviders.java index f22060a52a22..61ad1c2a5727 100644 --- a/core/src/main/java/org/apache/iceberg/LocationProviders.java +++ b/core/src/main/java/org/apache/iceberg/LocationProviders.java @@ -19,11 +19,11 @@ package org.apache.iceberg; import java.util.Map; +import java.util.function.Function; import org.apache.hadoop.fs.Path; import org.apache.iceberg.common.DynConstructors; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.LocationUtil; @@ -104,8 +104,8 @@ public String newDataLocation(String filename) { } static class ObjectStoreLocationProvider implements LocationProvider { - private static final Transform HASH_FUNC = - Transforms.bucket(Types.StringType.get(), Integer.MAX_VALUE); + private static final Function HASH_FUNC = + Transforms.bucket(Integer.MAX_VALUE).bind(Types.StringType.get()); private final String storageLocation; private final String context; diff --git a/core/src/main/java/org/apache/iceberg/ManifestsTable.java b/core/src/main/java/org/apache/iceberg/ManifestsTable.java index 88f9943ddfd4..2168a5b1ff18 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestsTable.java +++ b/core/src/main/java/org/apache/iceberg/ManifestsTable.java @@ -130,12 +130,14 @@ static List partitionSummariesToRows( .get(i) .transform() .toHumanString( + spec.partitionType().fields().get(i).type(), Conversions.fromByteBuffer( spec.partitionType().fields().get(i).type(), summary.lowerBound())), spec.fields() .get(i) .transform() .toHumanString( + spec.partitionType().fields().get(i).type(), Conversions.fromByteBuffer( spec.partitionType().fields().get(i).type(), summary.upperBound())))); } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index e449a26e421b..0a566581abe5 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -109,8 +109,7 @@ static TableMetadata newTableMetadata( // look up the name of the source field in the old schema to get the new schema's id String sourceName = schema.findColumnName(field.sourceId()); // reassign all partition fields with fresh partition field Ids to ensure consistency - specBuilder.add( - freshSchema.findField(sourceName).fieldId(), field.name(), field.transform().toString()); + specBuilder.add(freshSchema.findField(sourceName).fieldId(), field.name(), field.transform()); } PartitionSpec freshSpec = specBuilder.build(); diff --git a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java index fa9b8649ce7d..6b5c3bb9e02e 100644 --- a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java +++ b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java @@ -35,7 +35,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.transforms.Transforms; -import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; @@ -316,7 +315,7 @@ private static Expression.Operation fromType(String type) { @SuppressWarnings("unchecked") private static UnboundPredicate predicateFromJson( Expression.Operation op, JsonNode node, Schema schema) { - UnboundTerm term = term(JsonUtil.get(TERM, node), schema); + UnboundTerm term = term(JsonUtil.get(TERM, node)); Function convertValue; if (schema != null) { @@ -399,7 +398,7 @@ private static Object asObject(JsonNode node) { } @SuppressWarnings("unchecked") - private static UnboundTerm term(JsonNode node, Schema schema) { + private static UnboundTerm term(JsonNode node) { if (node.isTextual()) { return Expressions.ref(node.asText()); } else if (node.isObject()) { @@ -408,11 +407,10 @@ private static UnboundTerm term(JsonNode node, Schema schema) { case REFERENCE: return Expressions.ref(JsonUtil.getString(TERM, node)); case TRANSFORM: - UnboundTerm child = term(JsonUtil.get(TERM, node), schema); - Type termType = schema.findType(child.ref().name()); + UnboundTerm child = term(JsonUtil.get(TERM, node)); String transform = JsonUtil.getString(TRANSFORM, node); return (UnboundTerm) - Expressions.transform(child.ref().name(), Transforms.fromString(termType, transform)); + Expressions.transform(child.ref().name(), Transforms.fromString(transform)); default: throw new IllegalArgumentException("Cannot parse type as a reference: " + type); } diff --git a/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java b/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java index db131134861f..fd01b5f7db5d 100644 --- a/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java +++ b/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java @@ -96,7 +96,7 @@ public void testReplaceTransactionWithCustomSortOrder() { Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction()); Assert.assertEquals( "Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder()); - Transform transform = Transforms.identity(Types.IntegerType.get()); + Transform transform = Transforms.identity(); Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform()); } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index e6c7109f6a69..ab42ebea4f90 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -53,6 +53,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; import org.assertj.core.api.Assertions; @@ -1069,8 +1070,8 @@ public void testNewTableMetadataReassignmentAllIds() throws Exception { PartitionSpec spec = PartitionSpec.builderFor(schema) .withSpecId(5) - .add(3, 1005, "x_partition", "bucket[4]") - .add(5, 1003, "z_partition", "bucket[8]") + .add(3, 1005, "x_partition", Transforms.bucket(4)) + .add(5, 1003, "z_partition", Transforms.bucket(8)) .build(); String location = "file://tmp/db/table"; TableMetadata metadata = @@ -1080,8 +1081,8 @@ public void testNewTableMetadataReassignmentAllIds() throws Exception { PartitionSpec expected = PartitionSpec.builderFor(metadata.schema()) .withSpecId(0) - .add(1, 1000, "x_partition", "bucket[4]") - .add(3, 1001, "z_partition", "bucket[8]") + .add(1, 1000, "x_partition", Transforms.bucket(4)) + .add(3, 1001, "z_partition", Transforms.bucket(8)) .build(); Assert.assertEquals(expected, metadata.spec()); @@ -1094,7 +1095,7 @@ public void testInvalidUpdatePartitionSpecForV1Table() throws Exception { PartitionSpec spec = PartitionSpec.builderFor(schema) .withSpecId(5) - .add(1, 1005, "x_partition", "bucket[4]") + .add(1, 1005, "x_partition", Transforms.bucket(4)) .build(); String location = "file://tmp/db/table"; TableMetadata metadata = @@ -1135,9 +1136,9 @@ public void testBuildReplacementForV1Table() { PartitionSpec expected = PartitionSpec.builderFor(updated.schema()) .withSpecId(1) - .add(1, 1000, "x", "identity") - .add(2, 1001, "y", "void") - .add(3, 1002, "z_bucket", "bucket[8]") + .add(1, 1000, "x", Transforms.identity()) + .add(2, 1001, "y", Transforms.alwaysNull()) + .add(3, 1002, "z_bucket", Transforms.bucket(8)) .build(); Assert.assertEquals( "Should reassign the partition field IDs and reuse any existing IDs for equivalent fields", @@ -1171,8 +1172,8 @@ public void testBuildReplacementForV2Table() { PartitionSpec expected = PartitionSpec.builderFor(updated.schema()) .withSpecId(1) - .add(3, 1002, "z_bucket", "bucket[8]") - .add(1, 1000, "x", "identity") + .add(3, 1002, "z_bucket", Transforms.bucket(8)) + .add(1, 1000, "x", Transforms.identity()) .build(); Assert.assertEquals( "Should reassign the partition field IDs and reuse any existing IDs for equivalent fields", diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java index 6c3414748a68..f770cd279287 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java @@ -21,6 +21,7 @@ import static org.apache.iceberg.expressions.Expressions.bucket; import static org.apache.iceberg.expressions.Expressions.truncate; +import org.apache.iceberg.transforms.Transforms; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -83,7 +84,7 @@ public void testCommitUpdatedSpec() { "Should hard delete id and data buckets", PartitionSpec.builderFor(table.schema()) .withSpecId(2) - .add(2, 1002, "data_trunc_8", "truncate[8]") + .add(2, 1002, "data_trunc_8", Transforms.truncate(8)) .build(), table.spec()); @@ -179,7 +180,7 @@ public void testRemoveAndAddField() { "Should hard delete data bucket", PartitionSpec.builderFor(table.schema()) .withSpecId(1) - .add(1, 1001, "id_bucket_8", "bucket[8]") + .add(1, 1001, "id_bucket_8", Transforms.bucket(8)) .build(), table.spec()); @@ -202,7 +203,7 @@ public void testAddAndRemoveField() { "Should remove and then add a bucket field", PartitionSpec.builderFor(table.schema()) .withSpecId(1) - .add(2, 1001, "data_bucket_6", "bucket[6]") + .add(2, 1001, "data_bucket_6", Transforms.bucket(6)) .build(), table.spec()); Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); @@ -243,7 +244,7 @@ public void testAddAfterLastFieldRemoved() { "Should add a new id bucket", PartitionSpec.builderFor(table.schema()) .withSpecId(2) - .add(1, 1001, "id_bucket_8", "bucket[8]") + .add(1, 1001, "id_bucket_8", Transforms.bucket(8)) .build(), table.spec()); Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); diff --git a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java index a4f6f8518ccf..97210e51b5ba 100644 --- a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java @@ -213,9 +213,8 @@ public void testAddHourToDay() { Assert.assertEquals( "Should have a day and an hour time field", ImmutableList.of( - new PartitionField(2, 1000, "ts_day", Transforms.day(Types.TimestampType.withZone())), - new PartitionField( - 2, 1001, "ts_hour", Transforms.hour(Types.TimestampType.withZone()))), + new PartitionField(2, 1000, "ts_day", Transforms.day()), + new PartitionField(2, 1001, "ts_hour", Transforms.hour())), byHour.fields()); } @@ -254,8 +253,8 @@ public void testRemoveIdentityByName() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("ts"), 1001, "ts_day", Transforms.day(Types.TimestampType.withZone())) - .add(id("id"), 1002, "shard", Transforms.bucket(Types.LongType.get(), 16)) + .add(id("ts"), 1001, "ts_day", Transforms.day()) + .add(id("id"), 1002, "shard", Transforms.bucket(16)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -277,8 +276,8 @@ public void testRemoveBucketByName() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("category"), 1000, "category", Transforms.identity(Types.StringType.get())) - .add(id("ts"), 1001, "ts_day", Transforms.day(Types.TimestampType.withZone())) + .add(id("category"), 1000, "category", Transforms.identity()) + .add(id("ts"), 1001, "ts_day", Transforms.day()) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -302,8 +301,8 @@ public void testRemoveIdentityByEquivalent() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("ts"), 1001, "ts_day", Transforms.day(Types.TimestampType.withZone())) - .add(id("id"), 1002, "shard", Transforms.bucket(Types.LongType.get(), 16)) + .add(id("ts"), 1001, "ts_day", Transforms.day()) + .add(id("id"), 1002, "shard", Transforms.bucket(16)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -325,8 +324,8 @@ public void testRemoveDayByEquivalent() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("category"), 1000, "category", Transforms.identity(Types.StringType.get())) - .add(id("id"), 1002, "shard", Transforms.bucket(Types.LongType.get(), 16)) + .add(id("category"), 1000, "category", Transforms.identity()) + .add(id("id"), 1002, "shard", Transforms.bucket(16)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -388,9 +387,9 @@ public void testMultipleChanges() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("category"), 1000, "category", Transforms.identity(Types.StringType.get())) - .add(id("id"), 1002, "id_bucket", Transforms.bucket(Types.LongType.get(), 16)) - .add(id("data"), 1003, "prefix", Transforms.truncate(Types.StringType.get(), 4)) + .add(id("category"), 1000, "category", Transforms.identity()) + .add(id("id"), 1002, "id_bucket", Transforms.bucket(16)) + .add(id("data"), 1003, "prefix", Transforms.truncate(4)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java index 7d5599919d21..2f89f6875d91 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java @@ -51,7 +51,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; -import org.apache.iceberg.types.Types; import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Test; @@ -181,7 +180,7 @@ public void testCreateTableCustomSortOrder() throws Exception { Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction()); Assert.assertEquals( "Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder()); - Transform transform = Transforms.identity(Types.IntegerType.get()); + Transform transform = Transforms.identity(); Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform()); } diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java index ef7c8252a6f5..e1cd27567442 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java @@ -143,7 +143,7 @@ public void testCustomSortOrder() { Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction()); Assert.assertEquals( "Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder()); - Transform transform = Transforms.identity(Types.IntegerType.get()); + Transform transform = Transforms.identity(); Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform()); } diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java index 68016af8ed04..87e97832a0ac 100644 --- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java +++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java @@ -250,7 +250,7 @@ public void testCreateTableCustomSortOrder() { Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction()); Assert.assertEquals( "Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder()); - Transform transform = Transforms.identity(Types.IntegerType.get()); + Transform transform = Transforms.identity(); Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform()); } diff --git a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java index 8d0b94262aee..13ff034e4bf5 100644 --- a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java +++ b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java @@ -87,8 +87,10 @@ public void setupBenchmark() { setupSpark(); List data = Lists.newArrayList(RandomData.generateSpark(SCHEMA, NUM_ROWS, 0L)); - Transform transform = Transforms.bucket(Types.IntegerType.get(), 32); - data.sort(Comparator.comparingInt(row -> transform.apply(row.getInt(1)))); + Transform transform = Transforms.bucket(32); + data.sort( + Comparator.comparingInt( + row -> transform.bind(Types.IntegerType.get()).apply(row.getInt(1)))); this.rows = data; this.positionDeleteRows = diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java index 094364d229b3..eb2420c0b254 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark; -import org.apache.iceberg.transforms.Transform; +import java.util.function.Function; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Type; import org.apache.spark.sql.SparkSession; @@ -32,7 +32,7 @@ public static void registerBucketUDF( SparkSession session, String funcName, DataType sourceType, int numBuckets) { SparkTypeToType typeConverter = new SparkTypeToType(); Type sourceIcebergType = typeConverter.atomic(sourceType); - Transform bucket = Transforms.bucket(sourceIcebergType, numBuckets); + Function bucket = Transforms.bucket(numBuckets).bind(sourceIcebergType); session .udf() .register( @@ -45,7 +45,7 @@ public static void registerTruncateUDF( SparkSession session, String funcName, DataType sourceType, int width) { SparkTypeToType typeConverter = new SparkTypeToType(); Type sourceIcebergType = typeConverter.atomic(sourceType); - Transform truncate = Transforms.truncate(sourceIcebergType, width); + Function truncate = Transforms.truncate(width).bind(sourceIcebergType); session .udf() .register( diff --git a/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala b/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala index 2d2c2fb9aae5..dffac82af791 100644 --- a/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala +++ b/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala @@ -22,6 +22,7 @@ package org.apache.spark.sql.catalyst.expressions import java.nio.ByteBuffer import java.nio.CharBuffer import java.nio.charset.StandardCharsets +import java.util.function import org.apache.iceberg.spark.SparkSchemaUtil import org.apache.iceberg.transforms.Transform import org.apache.iceberg.transforms.Transforms @@ -48,7 +49,7 @@ abstract class IcebergTransformExpression abstract class IcebergTimeTransform extends IcebergTransformExpression with ImplicitCastInputTypes { - def transform: Transform[Any, Integer] + def transform: function.Function[Any, Integer] override protected def nullSafeEval(value: Any): Any = { transform(value).toInt @@ -62,7 +63,7 @@ abstract class IcebergTimeTransform case class IcebergYearTransform(child: Expression) extends IcebergTimeTransform { - @transient lazy val transform: Transform[Any, Integer] = Transforms.year[Any](icebergInputType) + @transient lazy val transform: function.Function[Any, Integer] = Transforms.year[Any]().bind(icebergInputType) override protected def withNewChildInternal(newChild: Expression): Expression = { copy(child = newChild) @@ -72,7 +73,7 @@ case class IcebergYearTransform(child: Expression) case class IcebergMonthTransform(child: Expression) extends IcebergTimeTransform { - @transient lazy val transform: Transform[Any, Integer] = Transforms.month[Any](icebergInputType) + @transient lazy val transform: function.Function[Any, Integer] = Transforms.month[Any]().bind(icebergInputType) override protected def withNewChildInternal(newChild: Expression): Expression = { copy(child = newChild) @@ -82,7 +83,7 @@ case class IcebergMonthTransform(child: Expression) case class IcebergDayTransform(child: Expression) extends IcebergTimeTransform { - @transient lazy val transform: Transform[Any, Integer] = Transforms.day[Any](icebergInputType) + @transient lazy val transform: function.Function[Any, Integer] = Transforms.day[Any]().bind(icebergInputType) override protected def withNewChildInternal(newChild: Expression): Expression = { copy(child = newChild) @@ -92,7 +93,7 @@ case class IcebergDayTransform(child: Expression) case class IcebergHourTransform(child: Expression) extends IcebergTimeTransform { - @transient lazy val transform: Transform[Any, Integer] = Transforms.hour[Any](icebergInputType) + @transient lazy val transform: function.Function[Any, Integer] = Transforms.hour[Any]().bind(icebergInputType) override protected def withNewChildInternal(newChild: Expression): Expression = { copy(child = newChild) @@ -103,15 +104,15 @@ case class IcebergBucketTransform(numBuckets: Int, child: Expression) extends Ic @transient lazy val bucketFunc: Any => Int = child.dataType match { case _: DecimalType => - val t = Transforms.bucket[Any](icebergInputType, numBuckets) + val t = Transforms.bucket[Any](numBuckets).bind(icebergInputType) d: Any => t(d.asInstanceOf[Decimal].toJavaBigDecimal).toInt case _: StringType => // the spec requires that the hash of a string is equal to the hash of its UTF-8 encoded bytes // TODO: pass bytes without the copy out of the InternalRow - val t = Transforms.bucket[ByteBuffer](Types.BinaryType.get(), numBuckets) + val t = Transforms.bucket[ByteBuffer](numBuckets).bind(Types.BinaryType.get()) s: Any => t(ByteBuffer.wrap(s.asInstanceOf[UTF8String].getBytes)).toInt case _ => - val t = Transforms.bucket[Any](icebergInputType, numBuckets) + val t = Transforms.bucket[Any](numBuckets).bind(icebergInputType) a: Any => t(a).toInt } @@ -130,20 +131,20 @@ case class IcebergTruncateTransform(child: Expression, width: Int) extends Icebe @transient lazy val truncateFunc: Any => Any = child.dataType match { case _: DecimalType => - val t = Transforms.truncate[java.math.BigDecimal](icebergInputType, width) + val t = Transforms.truncate[java.math.BigDecimal](width).bind(icebergInputType) d: Any => Decimal.apply(t(d.asInstanceOf[Decimal].toJavaBigDecimal)) case _: StringType => - val t = Transforms.truncate[CharSequence](icebergInputType, width) + val t = Transforms.truncate[CharSequence](width).bind(icebergInputType) s: Any => { val charSequence = t(StandardCharsets.UTF_8.decode(ByteBuffer.wrap(s.asInstanceOf[UTF8String].getBytes))) val bb = StandardCharsets.UTF_8.encode(CharBuffer.wrap(charSequence)); UTF8String.fromBytes(ByteBuffers.toByteArray(bb)) } case _: BinaryType => - val t = Transforms.truncate[ByteBuffer](icebergInputType, width) + val t = Transforms.truncate[ByteBuffer](width).bind(icebergInputType) s: Any => ByteBuffers.toByteArray(t(ByteBuffer.wrap(s.asInstanceOf[Array[Byte]]))) case _ => - val t = Transforms.truncate[Any](icebergInputType, width) + val t = Transforms.truncate[Any](width).bind(icebergInputType) a: Any => t(a) } diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java index b30bbf145f23..a2b4837ec2df 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Locale; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.DataFile; @@ -48,7 +49,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.data.GenericsHelpers; -import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Dataset; @@ -116,23 +116,23 @@ public static void startSpark() { TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate(); // define UDFs used by partition tests - Transform bucket4 = Transforms.bucket(Types.LongType.get(), 4); + Function bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); spark.udf().register("bucket4", (UDF1) bucket4::apply, IntegerType$.MODULE$); - Transform day = Transforms.day(Types.TimestampType.withZone()); + Function day = Transforms.day().bind(Types.TimestampType.withZone()); spark .udf() .register( "ts_day", - (UDF1) timestamp -> day.apply((Long) fromJavaTimestamp(timestamp)), + (UDF1) timestamp -> day.apply(fromJavaTimestamp(timestamp)), IntegerType$.MODULE$); - Transform hour = Transforms.hour(Types.TimestampType.withZone()); + Function hour = Transforms.hour().bind(Types.TimestampType.withZone()); spark .udf() .register( "ts_hour", - (UDF1) timestamp -> hour.apply((Long) fromJavaTimestamp(timestamp)), + (UDF1) timestamp -> hour.apply(fromJavaTimestamp(timestamp)), IntegerType$.MODULE$); spark.udf().register("data_ident", (UDF1) data -> data, StringType$.MODULE$); diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java index 0be1e0b1bd05..37e329a8b97b 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java @@ -61,7 +61,8 @@ public void testRegisterIntegerBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_int_16(1)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.IntegerType.get(), 16).apply(1), results.get(0).getInt(0)); + (int) Transforms.bucket(16).bind(Types.IntegerType.get()).apply(1), + results.get(0).getInt(0)); } @Test @@ -70,7 +71,8 @@ public void testRegisterShortBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_short_16(1S)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.IntegerType.get(), 16).apply(1), results.get(0).getInt(0)); + (int) Transforms.bucket(16).bind(Types.IntegerType.get()).apply(1), + results.get(0).getInt(0)); } @Test @@ -79,7 +81,8 @@ public void testRegisterByteBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_byte_16(1Y)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.IntegerType.get(), 16).apply(1), results.get(0).getInt(0)); + (int) Transforms.bucket(16).bind(Types.IntegerType.get()).apply(1), + results.get(0).getInt(0)); } @Test @@ -88,7 +91,7 @@ public void testRegisterLongBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_long_16(1L)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.LongType.get(), 16).apply(1L), results.get(0).getInt(0)); + (int) Transforms.bucket(16).bind(Types.LongType.get()).apply(1L), results.get(0).getInt(0)); } @Test @@ -97,7 +100,7 @@ public void testRegisterStringBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_string_16('hello')").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.StringType.get(), 16).apply("hello"), + (int) Transforms.bucket(16).bind(Types.StringType.get()).apply("hello"), results.get(0).getInt(0)); } @@ -107,7 +110,7 @@ public void testRegisterCharBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_char_16('hello')").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.StringType.get(), 16).apply("hello"), + (int) Transforms.bucket(16).bind(Types.StringType.get()).apply("hello"), results.get(0).getInt(0)); } @@ -117,7 +120,7 @@ public void testRegisterVarCharBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_varchar_16('hello')").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.StringType.get(), 16).apply("hello"), + (int) Transforms.bucket(16).bind(Types.StringType.get()).apply("hello"), results.get(0).getInt(0)); } @@ -129,7 +132,8 @@ public void testRegisterDateBucketUDF() { Assert.assertEquals(1, results.size()); Assert.assertEquals( (int) - Transforms.bucket(Types.DateType.get(), 16) + Transforms.bucket(16) + .bind(Types.DateType.get()) .apply(DateTimeUtils.fromJavaDate(Date.valueOf("2021-06-30"))), results.get(0).getInt(0)); } @@ -145,7 +149,8 @@ public void testRegisterTimestampBucketUDF() { Assert.assertEquals(1, results.size()); Assert.assertEquals( (int) - Transforms.bucket(Types.TimestampType.withZone(), 16) + Transforms.bucket(16) + .bind(Types.TimestampType.withZone()) .apply( DateTimeUtils.fromJavaTimestamp(Timestamp.valueOf("2021-06-30 00:00:00.000"))), results.get(0).getInt(0)); @@ -158,7 +163,8 @@ public void testRegisterBinaryBucketUDF() { Assert.assertEquals(1, results.size()); Assert.assertEquals( (int) - Transforms.bucket(Types.BinaryType.get(), 16) + Transforms.bucket(16) + .bind(Types.BinaryType.get()) .apply(ByteBuffer.wrap(new byte[] {0x00, 0x20, 0x00, 0x1F})), results.get(0).getInt(0)); } @@ -169,7 +175,7 @@ public void testRegisterDecimalBucketUDF() { List results = spark.sql("SELECT iceberg_bucket_decimal_16(11.11)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - (int) Transforms.bucket(Types.DecimalType.of(4, 2), 16).apply(new BigDecimal("11.11")), + (int) Transforms.bucket(16).bind(Types.DecimalType.of(4, 2)).apply(new BigDecimal("11.11")), results.get(0).getInt(0)); } @@ -209,7 +215,7 @@ public void testRegisterIntegerTruncateUDF() { List results = spark.sql("SELECT iceberg_truncate_int_4(1)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - Transforms.truncate(Types.IntegerType.get(), 4).apply(1), results.get(0).getInt(0)); + Transforms.truncate(4).bind(Types.IntegerType.get()).apply(1), results.get(0).getInt(0)); } @Test @@ -218,7 +224,7 @@ public void testRegisterLongTruncateUDF() { List results = spark.sql("SELECT iceberg_truncate_long_4(1L)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - Transforms.truncate(Types.LongType.get(), 4).apply(1L), results.get(0).getLong(0)); + Transforms.truncate(4).bind(Types.LongType.get()).apply(1L), results.get(0).getLong(0)); } @Test @@ -227,7 +233,7 @@ public void testRegisterDecimalTruncateUDF() { List results = spark.sql("SELECT iceberg_truncate_decimal_4(11.11)").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - Transforms.truncate(Types.DecimalType.of(4, 2), 4).apply(new BigDecimal("11.11")), + Transforms.truncate(4).bind(Types.DecimalType.of(4, 2)).apply(new BigDecimal("11.11")), results.get(0).getDecimal(0)); } @@ -237,6 +243,7 @@ public void testRegisterStringTruncateUDF() { List results = spark.sql("SELECT iceberg_truncate_string_4('hello')").collectAsList(); Assert.assertEquals(1, results.size()); Assert.assertEquals( - Transforms.truncate(Types.StringType.get(), 4).apply("hello"), results.get(0).getString(0)); + Transforms.truncate(4).bind(Types.StringType.get()).apply("hello"), + results.get(0).getString(0)); } } diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java index ffe21432f00c..4ef022c50c59 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java @@ -29,6 +29,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; @@ -47,7 +48,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.SparkSchemaUtil; -import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.spark.api.java.JavaRDD; @@ -96,12 +96,12 @@ public TestPartitionPruning(String format, boolean vectorized) { private static SparkSession spark = null; private static JavaSparkContext sparkContext = null; - private static Transform bucketTransform = - Transforms.bucket(Types.IntegerType.get(), 3); - private static Transform truncateTransform = - Transforms.truncate(Types.StringType.get(), 5); - private static Transform hourTransform = - Transforms.hour(Types.TimestampType.withoutZone()); + private static final Function BUCKET_FUNC = + Transforms.bucket(3).bind(Types.IntegerType.get()); + private static final Function TRUNCATE_FUNC = + Transforms.truncate(5).bind(Types.StringType.get()); + private static final Function HOUR_FUNC = + Transforms.hour().bind(Types.TimestampType.withoutZone()); @BeforeClass public static void startSpark() { @@ -112,19 +112,17 @@ public static void startSpark() { CONF.set(optionKey, CountOpenLocalFileSystem.class.getName()); spark.conf().set(optionKey, CountOpenLocalFileSystem.class.getName()); spark.conf().set("spark.sql.session.timeZone", "UTC"); + spark.udf().register("bucket3", (Integer num) -> BUCKET_FUNC.apply(num), DataTypes.IntegerType); spark .udf() - .register("bucket3", (Integer num) -> bucketTransform.apply(num), DataTypes.IntegerType); - spark - .udf() - .register("truncate5", (String str) -> truncateTransform.apply(str), DataTypes.StringType); + .register("truncate5", (String str) -> TRUNCATE_FUNC.apply(str), DataTypes.StringType); // NOTE: date transforms take the type long, not Timestamp spark .udf() .register( "hour", (Timestamp ts) -> - hourTransform.apply( + HOUR_FUNC.apply( org.apache.spark.sql.catalyst.util.DateTimeUtils.fromJavaTimestamp(ts)), DataTypes.IntegerType); } @@ -197,7 +195,7 @@ public void testPartitionPruningBucketingInteger() { (Row r) -> { int bucketId = r.getInt(2); Set buckets = - Arrays.stream(ids).map(bucketTransform::apply).boxed().collect(Collectors.toSet()); + Arrays.stream(ids).map(BUCKET_FUNC::apply).boxed().collect(Collectors.toSet()); return buckets.contains(bucketId); }; @@ -243,7 +241,7 @@ public void testPartitionPruningHourlyPartition() { int hourValue = r.getInt(4); Instant instant = getInstant("2020-02-03T01:00:00"); Integer hourValueToFilter = - hourTransform.apply(TimeUnit.MILLISECONDS.toMicros(instant.toEpochMilli())); + HOUR_FUNC.apply(TimeUnit.MILLISECONDS.toMicros(instant.toEpochMilli())); return hourValue >= hourValueToFilter; };