From a96f950a9800f7efe6717e02d437c8c155758ac7 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 8 Jun 2020 16:21:02 +0100 Subject: [PATCH 01/14] Adding serde classes --- build.gradle | 1 + .../IcebergObjectInspectorGenerator.java | 86 +++++++++++++ .../mr/mapred/IcebergSchemaToTypeInfo.java | 119 ++++++++++++++++++ .../iceberg/mr/mapred/IcebergSerDe.java | 109 ++++++++++++++++ versions.props | 1 + 5 files changed, 316 insertions(+) create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java diff --git a/build.gradle b/build.gradle index c5bed3291b45..5ca9537d1503 100644 --- a/build.gradle +++ b/build.gradle @@ -309,6 +309,7 @@ project(':iceberg-mr') { compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' } + compileOnly("org.apache.hive:hive-serde") testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java new file mode 100644 index 000000000000..f6838dbae9d2 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Types; + +class IcebergObjectInspectorGenerator { + + protected ObjectInspector createObjectInspector(Schema schema) throws Exception { + List columnNames = setColumnNames(schema); + List columnTypes = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + List columnOIs = new ArrayList<>(columnTypes.size()); + for (int i = 0; i < columnTypes.size(); i++) { + columnOIs.add(createObjectInspectorWorker(columnTypes.get(i))); + } + return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs, null); + } + + protected ObjectInspector createObjectInspectorWorker(TypeInfo typeInfo) throws Exception { + ObjectInspector.Category typeCategory = typeInfo.getCategory(); + + switch (typeCategory) { + case PRIMITIVE: + PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo; + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti); + case LIST: + ListTypeInfo ati = (ListTypeInfo) typeInfo; + return ObjectInspectorFactory + .getStandardListObjectInspector(createObjectInspectorWorker(ati.getListElementTypeInfo())); + case MAP: + MapTypeInfo mti = (MapTypeInfo) typeInfo; + return ObjectInspectorFactory.getStandardMapObjectInspector( + createObjectInspectorWorker(mti.getMapKeyTypeInfo()), + createObjectInspectorWorker(mti.getMapValueTypeInfo())); + case STRUCT: + StructTypeInfo sti = (StructTypeInfo) typeInfo; + List ois = new ArrayList<>(sti.getAllStructFieldTypeInfos().size()); + for (TypeInfo structTypeInfos : sti.getAllStructFieldTypeInfos()) { + ois.add(createObjectInspectorWorker(structTypeInfos)); + } + return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois); + default: + throw new SerDeException("Couldn't create Object Inspector for category: '" + typeCategory + "'"); + } + } + + protected List setColumnNames(Schema schema) { + List fields = schema.columns(); + List fieldsList = new ArrayList<>(fields.size()); + for (Types.NestedField field : fields) { + fieldsList.add(field.name()); + } + return fieldsList; + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java new file mode 100644 index 000000000000..6190c6d22938 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.Schema; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +/** + * Class to convert Iceberg types to Hive TypeInfo + */ +final class IcebergSchemaToTypeInfo { + + private IcebergSchemaToTypeInfo() { + + } + + private static final ImmutableMap primitiveTypeToTypeInfo = ImmutableMap.builder() + .put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME)) + .put(Types.IntegerType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME)) + .put(Types.LongType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) + .put(Types.FloatType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME)) + .put(Types.DoubleType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)) + .put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME)) + .put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)) + .put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME)) + .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) + .put(Types.TimestampType.withZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) + .build(); + + public static List getColumnTypes(Schema schema) throws Exception { + List fields = schema.columns(); + List types = new ArrayList<>(fields.size()); + for (Types.NestedField field : fields) { + types.add(generateTypeInfo(field.type())); + } + return types; + } + + private static TypeInfo generateTypeInfo(Type type) throws Exception { + if (primitiveTypeToTypeInfo.containsKey(type)) { + return (TypeInfo) primitiveTypeToTypeInfo.get(type); + } + switch (type.typeId()) { + case UUID: + return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); + case FIXED: + return TypeInfoFactory.getPrimitiveTypeInfo("binary"); + case TIME: + return TypeInfoFactory.getPrimitiveTypeInfo("long"); + case DECIMAL: + Types.DecimalType dec = (Types.DecimalType) type; + int scale = dec.scale(); + int precision = dec.precision(); + try { + HiveDecimalUtils.validateParameter(precision, scale); + } catch (Exception e) { + //TODO Log that precision / scale isn't valid + throw e; + } + return TypeInfoFactory.getDecimalTypeInfo(precision, scale); + case STRUCT: + return generateStructTypeInfo((Types.StructType) type); + case LIST: + return generateListTypeInfo((Types.ListType) type); + case MAP: + return generateMapTypeInfo((Types.MapType) type); + default: + throw new SerDeException("Can't map Iceberg type to Hive TypeInfo: '" + type.typeId() + "'"); + } + } + + private static TypeInfo generateMapTypeInfo(Types.MapType type) throws Exception { + Type keyType = type.keyType(); + Type valueType = type.valueType(); + return TypeInfoFactory.getMapTypeInfo(generateTypeInfo(keyType), generateTypeInfo(valueType)); + } + + private static TypeInfo generateStructTypeInfo(Types.StructType type) throws Exception { + List fields = type.fields(); + List fieldNames = new ArrayList<>(fields.size()); + List typeInfos = new ArrayList<>(fields.size()); + + for (Types.NestedField field : fields) { + fieldNames.add(field.name()); + typeInfos.add(generateTypeInfo(field.type())); + } + return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); + } + + private static TypeInfo generateListTypeInfo(Types.ListType type) throws Exception { + return TypeInfoFactory.getListTypeInfo(generateTypeInfo(type.elementType())); + } +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java new file mode 100644 index 000000000000..f9d117fe0aae --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import javax.annotation.Nullable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Writable; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotsTable; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; + +import static org.apache.iceberg.mr.mapred.SystemTableUtil.getVirtualColumnName; +import static org.apache.iceberg.mr.mapred.TableResolverUtil.resolveTableFromConfiguration; + +public class IcebergSerDe extends AbstractSerDe { + + private Schema schema; + private ObjectInspector inspector; + + @Override + public void initialize(@Nullable Configuration configuration, Properties serDeProperties) throws SerDeException { + Table table = null; + try { + table = resolveTableFromConfiguration(configuration, serDeProperties); + } catch (IOException e) { + throw new UncheckedIOException("Unable to resolve table from configuration: ", e); + } + this.schema = table.schema(); + if (table instanceof SnapshotsTable) { + try { + this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(schema); + } catch (Exception e) { + throw new SerDeException(e); + } + } else { + List columns = new ArrayList<>(schema.columns()); + columns.add(Types.NestedField.optional(Integer.MAX_VALUE, getVirtualColumnName(serDeProperties), Types.LongType.get())); + Schema withVirtualColumn = new Schema(columns); + + try { + this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(withVirtualColumn); + } catch (Exception e) { + throw new SerDeException(e); + } + } + } + + @Override + public Class getSerializedClass() { + return null; + } + + @Override + public Writable serialize(Object o, ObjectInspector objectInspector) { + return null; + } + + @Override + public SerDeStats getSerDeStats() { + return null; + } + + @Override + public Object deserialize(Writable writable) { + IcebergWritable icebergWritable = (IcebergWritable) writable; + Schema schema = icebergWritable.getSchema(); + List fields = schema.columns(); + List row = new ArrayList<>(); + + for (Types.NestedField field : fields) { + Object obj = ((IcebergWritable) writable).getRecord().getField(field.name()); + row.add(obj); + } + return Collections.unmodifiableList(row); + } + + @Override + public ObjectInspector getObjectInspector() { + return inspector; + } +} diff --git a/versions.props b/versions.props index 45a9be439e49..86309aad8842 100644 --- a/versions.props +++ b/versions.props @@ -2,6 +2,7 @@ org.slf4j:* = 1.7.25 org.apache.avro:avro = 1.9.2 org.apache.hadoop:* = 2.7.3 org.apache.hive:hive-metastore = 2.3.7 +org.apache.hive:hive-serde = 2.3.7 org.apache.orc:* = 1.6.3 org.apache.parquet:* = 1.11.0 org.apache.spark:spark-hive_2.11 = 2.4.5 From 8ec7a4bf44fea63dd17a95e31f283b2541ee6dfd Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 8 Jun 2020 16:50:10 +0100 Subject: [PATCH 02/14] added some required classes --- .../apache/iceberg/mr/InputFormatConfig.java | 130 ++++++++++++++++++ .../iceberg/mr/mapred/IcebergSerDe.java | 11 +- .../iceberg/mr/mapred/IcebergWritable.java | 61 ++++++++ .../iceberg/mr/mapred/SystemTableUtil.java | 74 ++++++++++ .../iceberg/mr/mapred/TableResolver.java | 121 ++++++++++++++++ .../iceberg/mr/mapred/TestTableResolver.java | 62 +++++++++ 6 files changed, 452 insertions(+), 7 deletions(-) create mode 100644 mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java new file mode 100644 index 000000000000..e462704efd95 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.expressions.Expression; + +public class InputFormatConfig { + + private InputFormatConfig() {} + + // configuration values for Iceberg input formats + public static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; + public static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; + public static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; + public static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; + public static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; + public static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; + public static final String READ_SCHEMA = "iceberg.mr.read.schema"; + public static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; + public static final String SPLIT_SIZE = "iceberg.mr.split.size"; + public static final String TABLE_PATH = "iceberg.mr.table.path"; + public static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; + public static final String LOCALITY = "iceberg.mr.locality"; + public static final String CATALOG = "iceberg.mr.catalog"; + + public static final String CATALOG_NAME = "iceberg.catalog"; + public static final String HADOOP_CATALOG = "hadoop.catalog"; + public static final String HADOOP_TABLES = "hadoop.tables"; + public static final String HIVE_CATALOG = "hive.catalog"; + public static final String ICEBERG_SNAPSHOTS_TABLE_SUFFIX = ".snapshots"; + public static final String SNAPSHOT_TABLE = "iceberg.snapshots.table"; + public static final String SNAPSHOT_TABLE_SUFFIX = "__snapshots"; + public static final String TABLE_LOCATION = "location"; + public static final String TABLE_NAME = "name"; + + public static class ConfigBuilder { + private final Configuration conf; + + public ConfigBuilder(Configuration conf) { + this.conf = conf; + // defaults + conf.setBoolean(SKIP_RESIDUAL_FILTERING, false); + conf.setBoolean(CASE_SENSITIVE, true); + conf.setBoolean(REUSE_CONTAINERS, false); + conf.setBoolean(LOCALITY, false); + } + + public ConfigBuilder filter(Expression expression) { + conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); + return this; + } + + public ConfigBuilder project(Schema schema) { + conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder reuseContainers(boolean reuse) { + conf.setBoolean(InputFormatConfig.REUSE_CONTAINERS, reuse); + return this; + } + + public ConfigBuilder caseSensitive(boolean caseSensitive) { + conf.setBoolean(InputFormatConfig.CASE_SENSITIVE, caseSensitive); + return this; + } + + public ConfigBuilder snapshotId(long snapshotId) { + conf.setLong(SNAPSHOT_ID, snapshotId); + return this; + } + + public ConfigBuilder asOfTime(long asOfTime) { + conf.setLong(AS_OF_TIMESTAMP, asOfTime); + return this; + } + + public ConfigBuilder splitSize(long splitSize) { + conf.setLong(SPLIT_SIZE, splitSize); + return this; + } + + /** + * If this API is called. The input splits constructed will have host location information + */ + public ConfigBuilder preferLocality() { + conf.setBoolean(LOCALITY, true); + return this; + } + + public ConfigBuilder catalogFunc(Class> catalogFuncClass) { + conf.setClass(CATALOG, catalogFuncClass, Function.class); + return this; + } + + /** + * Compute platforms pass down filters to data sources. If the data source cannot apply some filters, or only + * partially applies the filter, it will return the residual filter back. If the platform can correctly apply the + * residual filters, then it should call this api. Otherwise the current api will throw an exception if the passed + * in filter is not completely satisfied. + */ + public ConfigBuilder skipResidualFiltering() { + conf.setBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, true); + return this; + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java index f9d117fe0aae..8fe331a1293b 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -37,9 +37,6 @@ import org.apache.iceberg.Table; import org.apache.iceberg.types.Types; -import static org.apache.iceberg.mr.mapred.SystemTableUtil.getVirtualColumnName; -import static org.apache.iceberg.mr.mapred.TableResolverUtil.resolveTableFromConfiguration; - public class IcebergSerDe extends AbstractSerDe { private Schema schema; @@ -49,7 +46,7 @@ public class IcebergSerDe extends AbstractSerDe { public void initialize(@Nullable Configuration configuration, Properties serDeProperties) throws SerDeException { Table table = null; try { - table = resolveTableFromConfiguration(configuration, serDeProperties); + table = TableResolver.resolveTableFromConfiguration(configuration, serDeProperties); } catch (IOException e) { throw new UncheckedIOException("Unable to resolve table from configuration: ", e); } @@ -62,7 +59,8 @@ public void initialize(@Nullable Configuration configuration, Properties serDePr } } else { List columns = new ArrayList<>(schema.columns()); - columns.add(Types.NestedField.optional(Integer.MAX_VALUE, getVirtualColumnName(serDeProperties), Types.LongType.get())); + columns.add(Types.NestedField.optional(Integer.MAX_VALUE, SystemTableUtil.getVirtualColumnName(serDeProperties), + Types.LongType.get())); Schema withVirtualColumn = new Schema(columns); try { @@ -91,8 +89,7 @@ public SerDeStats getSerDeStats() { @Override public Object deserialize(Writable writable) { IcebergWritable icebergWritable = (IcebergWritable) writable; - Schema schema = icebergWritable.getSchema(); - List fields = schema.columns(); + List fields = icebergWritable.getSchema().columns(); List row = new ArrayList<>(); for (Types.NestedField field : fields) { diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java new file mode 100644 index 000000000000..8b0eb79fbe73 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import org.apache.hadoop.io.Writable; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.Record; + +public class IcebergWritable implements Writable { + + private Record record; + private Schema schema; + + public IcebergWritable() {} + + public void setRecord(Record record) { + this.record = record; + } + + public Record getRecord() { + return record; + } + + public Schema getSchema() { + return schema; + } + + public void setSchema(Schema schema) { + this.schema = schema; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + + } +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java new file mode 100644 index 000000000000..17f5b9fb07b2 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Types; + +public class SystemTableUtil { + + static final String VIRTUAL_COLUMN_NAME = "iceberg.hive.snapshot.virtual.column.name"; + + private static final String DEFAULT_SNAPSHOT_ID_COLUMN_NAME = "snapshot__id"; + + private SystemTableUtil() {} + + protected static Schema schemaWithVirtualColumn(Schema schema, String columnName) { + List columns = new ArrayList<>(schema.columns()); + columns.add(Types.NestedField.optional(Integer.MAX_VALUE, columnName, Types.LongType.get())); + return new Schema(columns); + } + + protected static Record recordWithVirtualColumn(Record record, long snapshotId, Schema oldSchema, + String columnName) { + Schema newSchema = schemaWithVirtualColumn(oldSchema, columnName); + Record newRecord = GenericRecord.create(newSchema); + for (Types.NestedField field : oldSchema.columns()) { + newRecord.setField(field.name(), record.getField(field.name())); + } + newRecord.setField(columnName, snapshotId); + return newRecord; + } + + protected static String getVirtualColumnName(Configuration conf) { + String virtualColumnName = conf.get(VIRTUAL_COLUMN_NAME); + if (virtualColumnName == null) { + return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; + } else { + return virtualColumnName; + } + } + + protected static String getVirtualColumnName(Properties properties) { + String virtualColumnName = properties.getProperty(VIRTUAL_COLUMN_NAME); + if (virtualColumnName == null) { + return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; + } else { + return virtualColumnName; + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java new file mode 100644 index 000000000000..b6e2d879cbe7 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Properties; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; + +final class TableResolver { + + private TableResolver() { + } + + static Table resolveTableFromJob(JobConf conf) throws IOException { + Properties properties = new Properties(); + properties.setProperty(InputFormatConfig.CATALOG_NAME, extractProperty(conf, InputFormatConfig.CATALOG_NAME)); + if (conf.get(InputFormatConfig.CATALOG_NAME).equals(InputFormatConfig.HADOOP_CATALOG)) { + properties.setProperty(InputFormatConfig.SNAPSHOT_TABLE, conf.get(InputFormatConfig.SNAPSHOT_TABLE, "true")); + } + properties.setProperty(InputFormatConfig.TABLE_LOCATION, extractProperty(conf, InputFormatConfig.TABLE_LOCATION)); + properties.setProperty(InputFormatConfig.TABLE_NAME, extractProperty(conf, InputFormatConfig.TABLE_NAME)); + return resolveTableFromConfiguration(conf, properties); + } + + static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { + String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME); + URI tableLocation = pathAsURI(properties.getProperty(InputFormatConfig.TABLE_LOCATION)); + if (catalogName == null) { + throw new IllegalArgumentException("Catalog property: 'iceberg.catalog' not set in JobConf"); + } + switch (catalogName) { + case InputFormatConfig.HADOOP_TABLES: + HadoopTables tables = new HadoopTables(conf); + return tables.load(tableLocation.getPath()); + case InputFormatConfig.HADOOP_CATALOG: + String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); + TableIdentifier id = TableIdentifier.parse(tableName); + if (tableName.endsWith(InputFormatConfig.SNAPSHOT_TABLE_SUFFIX)) { + if (!Boolean.parseBoolean(properties.getProperty(InputFormatConfig.SNAPSHOT_TABLE, + Boolean.TRUE.toString()))) { + String tablePath = id.toString().replaceAll("\\.", "/"); + URI warehouseLocation = pathAsURI(tableLocation.getPath().replaceAll(tablePath, "")); + HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); + return catalog.loadTable(id); + } else { + return resolveMetadataTable(conf, tableLocation.getPath(), tableName); + } + } else { + URI warehouseLocation = pathAsURI(extractWarehousePath(tableLocation.getPath(), tableName)); + HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); + return catalog.loadTable(id); + } + case InputFormatConfig.HIVE_CATALOG: + //TODO Implement HiveCatalog + return null; + } + return null; + } + + static Table resolveMetadataTable(Configuration conf, String location, String tableName) throws IOException { + URI warehouseLocation = pathAsURI(extractWarehousePath(location, tableName)); + HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); + String baseTableName = StringUtils.removeEnd(tableName, InputFormatConfig.SNAPSHOT_TABLE_SUFFIX); + + TableIdentifier snapshotsId = TableIdentifier.parse(baseTableName + + InputFormatConfig.ICEBERG_SNAPSHOTS_TABLE_SUFFIX); + return catalog.loadTable(snapshotsId); + } + + static URI pathAsURI(String path) throws IOException { + if (path == null) { + throw new IllegalArgumentException("Path is null."); + } + try { + return new URI(path); + } catch (URISyntaxException e) { + throw new IOException("Unable to create URI for table location: '" + path + "'", e); + } + } + + protected static String extractProperty(JobConf conf, String key) { + String value = conf.get(key); + if (value == null) { + throw new IllegalArgumentException("Property not set in JobConf: " + key); + } + return value; + } + + protected static String extractWarehousePath(String location, String tableName) { + String tablePath = tableName.replaceAll("\\.", "/").replaceAll( + InputFormatConfig.SNAPSHOT_TABLE_SUFFIX, ""); + return location.replaceAll(tablePath, ""); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java new file mode 100644 index 000000000000..e4b9c87a51c9 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import org.apache.hadoop.mapred.JobConf; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestTableResolver { + + @Test + public void extractWarehouseLocationRegularTable() { + // This is the style of input expected from HiveConf + String testLocation = "some/folder/database/table_a"; + String testTableName = "database.table_a"; + + String expected = "some/folder/"; + String result = TableResolver.extractWarehousePath(testLocation, testTableName); + + assertEquals(expected, result); + } + + @Test + public void extractPropertyFromJobConf() { + JobConf conf = new JobConf(); + String key = "iceberg.catalog"; + String value = "hadoop.tables"; + + conf.set(key, value); + + String result = TableResolver.extractProperty(conf, key); + + assertEquals(value, result); + } + + @Test(expected = IllegalArgumentException.class) + public void extractNonExistentProperty() { + JobConf conf = new JobConf(); + String key = "iceberg.catalog"; + + TableResolver.extractProperty(conf, key); + } + +} From 90311cc8ba9d11ace4e8c4237f5c908d95d91ce6 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 8 Jun 2020 17:24:21 +0100 Subject: [PATCH 03/14] Add tests --- mr/dependencies.lock | 3272 ++++++++++++++++- .../apache/iceberg/mr/mapred/TestHelpers.java | 123 + .../TestIcebergObjectInspectorGenerator.java | 41 + .../mapred/TestIcebergSchemaToTypeInfo.java | 162 + .../iceberg/mr/mapred/TestIcebergSerDe.java | 163 + 5 files changed, 3558 insertions(+), 203 deletions(-) create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java diff --git a/mr/dependencies.lock b/mr/dependencies.lock index 555afd070dcc..9fbd7551d7a1 100644 --- a/mr/dependencies.lock +++ b/mr/dependencies.lock @@ -1,4 +1,365 @@ { + "allProcessors": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "annotationProcessor": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "checkstyle": { + "antlr:antlr": { + "locked": "2.7.7", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.1.3", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "26.0-jre", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.puppycrawl.tools:checkstyle": { + "locked": "8.13" + }, + "commons-beanutils:commons-beanutils": { + "locked": "1.9.3", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "commons-cli:commons-cli": { + "locked": "1.4", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "commons-collections:commons-collections": { + "locked": "3.2.2", + "transitive": [ + "commons-beanutils:commons-beanutils" + ] + }, + "net.sf.saxon:Saxon-HE": { + "locked": "9.8.0-14", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "org.antlr:antlr4-runtime": { + "locked": "4.7.1", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.2", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.14", + "transitive": [ + "com.google.guava:guava" + ] + } + }, "compile": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", @@ -225,6 +586,12 @@ } }, "compileClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -234,10 +601,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -255,7 +635,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -265,6 +647,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -279,7 +667,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -294,21 +683,6 @@ "com.github.ben-manes.caffeine:caffeine" ] }, - "com.google.guava:guava": { - "locked": "16.0.1", - "transitive": [ - "org.apache.curator:curator-client", - "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" - ] - }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ @@ -330,7 +704,20 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -353,6 +740,7 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -360,6 +748,7 @@ "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -368,6 +757,7 @@ "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -377,6 +767,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -395,18 +791,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -429,7 +830,13 @@ "commons-configuration:commons-configuration" ] }, - "commons-httpclient:commons-httpclient": { + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, + "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ "org.apache.hadoop:hadoop-common" @@ -452,17 +859,24 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -491,6 +905,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -504,6 +939,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -517,11 +959,26 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -534,11 +991,18 @@ ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ @@ -549,9 +1013,35 @@ "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -560,7 +1050,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -580,7 +1077,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -613,6 +1111,24 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ @@ -632,6 +1148,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -715,6 +1232,40 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -723,15 +1274,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -773,9 +1328,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -825,12 +1416,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -845,7 +1457,8 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -898,6 +1511,18 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.fusesource.leveldbjni:leveldbjni-all": { "locked": "1.8", "transitive": [ @@ -916,6 +1541,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -931,16 +1560,24 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper" ] }, @@ -962,6 +1599,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -983,6 +1632,12 @@ } }, "compileOnly": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -992,14 +1647,53 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "com.fasterxml.jackson.core:jackson-annotations": { + "locked": "2.6.0", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind" + ] + }, + "com.fasterxml.jackson.core:jackson-core": { + "locked": "2.6.5", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind" + ] + }, + "com.fasterxml.jackson.core:jackson-databind": { + "locked": "2.6.5", + "transitive": [ + "io.dropwizard.metrics:metrics-json", + "org.apache.hive:hive-common" + ] + }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -1019,16 +1713,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -1043,22 +1753,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -1067,21 +1796,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -1091,6 +1827,18 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "com.thoughtworks.paranamer:paranamer": { + "locked": "2.3", + "transitive": [ + "org.apache.avro:avro" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -1109,18 +1857,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -1128,7 +1881,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -1143,6 +1897,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -1154,7 +1914,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -1166,24 +1927,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -1193,6 +1966,39 @@ "org.apache.hadoop:hadoop-common" ] }, + "io.airlift:aircompressor": { + "locked": "0.8", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "io.airlift:slice": { + "locked": "0.29", + "transitive": [ + "io.airlift:aircompressor" + ] + }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -1209,7 +2015,9 @@ "javax.activation:activation": { "locked": "1.1", "transitive": [ - "javax.xml.bind:jaxb-api" + "javax.mail:mail", + "javax.xml.bind:jaxb-api", + "org.eclipse.jetty.aggregate:jetty-all" ] }, "javax.inject:javax.inject": { @@ -1219,17 +2027,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -1237,7 +2060,9 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "javax.xml.stream:stax-api": { @@ -1247,11 +2072,18 @@ ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ @@ -1260,14 +2092,54 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, + "org.apache.avro:avro": { + "locked": "1.7.7", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" + ] + }, "org.apache.commons:commons-compress": { - "locked": "1.4.1", + "locked": "1.9", "transitive": [ + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -1287,7 +2159,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -1320,11 +2193,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -1339,6 +2235,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -1372,73 +2269,214 @@ "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hadoop:hadoop-client" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + ] + }, + "org.apache.hadoop:hadoop-yarn-api": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-client": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common" + ] + }, + "org.apache.hadoop:hadoop-yarn-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-nodemanager": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.htrace:htrace-core": { + "locked": "3.1.0-incubating", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.4.1", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.httpcomponents:httpcore": { + "locked": "4.4.1", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-api": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" ] }, - "org.apache.hadoop:hadoop-yarn-client": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" ] }, - "org.apache.hadoop:hadoop-yarn-common": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-common": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-nodemanager": { - "locked": "2.7.3", + "org.apache.orc:orc-core": { + "locked": "1.3.4", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.hive:hive-common" ] }, - "org.apache.htrace:htrace-core": { - "locked": "3.1.0-incubating", + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive:hive-serde" ] }, - "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "org.apache.thrift:libfb303": { + "locked": "0.9.3", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hive:hive-service-rpc" ] }, - "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "org.apache.thrift:libthrift": { + "locked": "0.9.3", "transitive": [ - "org.apache.httpcomponents:httpclient" + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" ] }, "org.apache.zookeeper:zookeeper": { @@ -1449,13 +2487,16 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.codehaus.jackson:jackson-core-asl": { "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", @@ -1475,6 +2516,7 @@ "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", @@ -1493,7 +2535,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -1501,13 +2557,26 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.openjdk.jol:jol-core": { + "locked": "0.2", + "transitive": [ + "io.airlift:slice" ] }, "org.slf4j:slf4j-api": { - "locked": "1.7.10", + "locked": "1.7.21", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", "org.apache.directory.api:api-util", @@ -1522,6 +2591,18 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.orc:orc-core", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper" ] }, @@ -1531,10 +2612,22 @@ "com.google.inject:guice" ] }, - "org.tukaani:xz": { - "locked": "1.0", + "org.xerial.snappy:snappy-java": { + "locked": "1.0.5", + "transitive": [ + "org.apache.avro:avro" + ] + }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", "transitive": [ - "org.apache.commons:commons-compress" + "org.apache.hive:hive-service-rpc" ] }, "xerces:xercesImpl": { @@ -1782,6 +2875,154 @@ ] } }, + "errorprone": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "errorproneJavac": { + "com.google.errorprone:javac": { + "locked": "9+181-r4173-1", + "requested": "9+181-r4173-1" + } + }, "runtime": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", @@ -2232,7 +3473,155 @@ ] } }, + "testAnnotationProcessor": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, "testCompile": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -2242,10 +3631,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -2263,7 +3665,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -2273,6 +3677,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -2287,7 +3697,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -2313,16 +3724,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -2337,22 +3764,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -2361,21 +3807,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -2385,6 +3838,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -2403,18 +3862,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -2422,7 +3886,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -2437,6 +3902,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -2448,7 +3919,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -2460,24 +3932,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -2499,6 +3983,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -2512,6 +4017,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -2525,17 +4037,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -2543,16 +4070,25 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -2564,12 +4100,39 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -2578,7 +4141,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -2598,7 +4168,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -2631,11 +4202,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -2650,6 +4244,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -2699,8 +4294,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -2715,8 +4313,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -2724,7 +4331,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -2733,6 +4343,66 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -2741,15 +4411,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -2791,9 +4465,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -2843,12 +4553,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -2863,7 +4594,9 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -2913,7 +4646,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -2921,8 +4668,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -2950,6 +4699,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -2965,16 +4718,27 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -3000,6 +4764,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -3021,6 +4797,12 @@ } }, "testCompileClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -3030,10 +4812,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -3051,7 +4846,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -3061,6 +4858,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -3075,7 +4878,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -3102,7 +4906,8 @@ "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "com.google.inject:guice": { @@ -3126,7 +4931,20 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -3149,6 +4967,7 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -3156,6 +4975,7 @@ "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -3164,6 +4984,7 @@ "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -3173,6 +4994,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -3191,18 +5018,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -3225,6 +5057,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -3248,17 +5086,24 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -3287,6 +5132,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -3300,6 +5166,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -3313,11 +5186,26 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -3330,11 +5218,18 @@ ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -3348,9 +5243,35 @@ "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -3359,7 +5280,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -3379,7 +5307,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -3412,6 +5341,24 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ @@ -3431,6 +5378,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -3514,6 +5462,40 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -3522,15 +5504,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -3572,9 +5558,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -3624,12 +5646,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -3644,7 +5687,8 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -3697,6 +5741,18 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.fusesource.leveldbjni:leveldbjni-all": { "locked": "1.8", "transitive": [ @@ -3724,6 +5780,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -3739,16 +5799,24 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -3774,6 +5842,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -3795,6 +5875,12 @@ } }, "testRuntime": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -3804,10 +5890,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -3825,7 +5924,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -3835,6 +5936,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -3849,7 +5956,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -3875,16 +5983,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -3899,22 +6023,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -3923,21 +6066,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -3947,6 +6097,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -3965,18 +6121,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -3984,7 +6145,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -3999,6 +6161,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -4010,7 +6178,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -4022,24 +6191,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -4061,6 +6242,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -4074,6 +6276,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -4087,17 +6296,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -4105,16 +6329,25 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -4126,12 +6359,39 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -4140,7 +6400,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -4160,7 +6427,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -4193,11 +6461,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -4212,6 +6503,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -4261,8 +6553,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -4277,8 +6572,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -4286,7 +6590,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -4295,6 +6602,66 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -4303,15 +6670,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -4353,9 +6724,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -4405,12 +6812,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -4425,7 +6853,9 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -4475,7 +6905,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -4483,8 +6927,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -4512,6 +6958,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -4527,16 +6977,27 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -4562,6 +7023,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -4583,6 +7056,12 @@ } }, "testRuntimeClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -4592,10 +7071,23 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -4613,7 +7105,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -4623,6 +7117,12 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ @@ -4637,7 +7137,8 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde" ] }, "com.google.code.gson:gson": { @@ -4663,16 +7164,32 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hive.shims:hive-shims-common" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -4687,22 +7204,41 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -4711,21 +7247,28 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -4735,6 +7278,12 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -4753,18 +7302,23 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -4772,7 +7326,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -4787,6 +7342,12 @@ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -4798,7 +7359,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "commons-lang:commons-lang": { @@ -4810,24 +7372,36 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", + "org.apache.orc:orc-core" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -4849,6 +7423,27 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ @@ -4862,6 +7457,13 @@ "org.apache.hadoop:hadoop-hdfs" ] }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.annotation:javax.annotation-api": { "locked": "1.3.2", "transitive": [ @@ -4875,17 +7477,32 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" ] }, "javax.xml.bind:jaxb-api": { @@ -4893,16 +7510,25 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { "locked": "4.12" }, @@ -4914,12 +7540,39 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -4928,7 +7581,14 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.commons:commons-math3": { @@ -4948,7 +7608,8 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common" ] }, "org.apache.curator:curator-recipes": { @@ -4981,11 +7642,34 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -5000,6 +7684,7 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", "org.apache.hadoop:hadoop-client" ] }, @@ -5049,8 +7734,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -5065,8 +7753,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -5074,7 +7771,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -5083,6 +7783,66 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -5091,15 +7851,19 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -5141,9 +7905,45 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -5193,12 +7993,33 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -5213,7 +8034,9 @@ "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common" ] }, "org.checkerframework:checker-qual": { @@ -5263,7 +8086,21 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -5271,8 +8108,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -5300,6 +8139,10 @@ "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -5315,16 +8158,27 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -5350,6 +8204,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java new file mode 100644 index 000000000000..7f96979fa7ed --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataWriter; +import org.apache.iceberg.data.orc.GenericOrcWriter; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.types.Types; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestHelpers { + + private TestHelpers() {} + + public static DataFile writeFile(File targetFile, Table table, StructLike partitionData, FileFormat fileFormat, + List records) throws IOException { + if (targetFile.exists()) { + if (!targetFile.delete()) { + throw new IOException("Unable to delete " + targetFile.getAbsolutePath()); + } + } + FileAppender appender; + switch (fileFormat) { + case AVRO: + appender = Avro.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(DataWriter::create) + .named(fileFormat.name()) + .build(); + break; + case PARQUET: + appender = Parquet.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(GenericParquetWriter::buildWriter) + .named(fileFormat.name()) + .build(); + break; + case ORC: + appender = ORC.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(GenericOrcWriter::buildWriter) + .build(); + break; + default: + throw new UnsupportedOperationException("Cannot write format: " + fileFormat); + } + + try { + appender.addAll(records); + } finally { + appender.close(); + } + + DataFiles.Builder builder = DataFiles.builder(table.spec()) + .withPath(targetFile.toString()) + .withFormat(fileFormat) + .withFileSizeInBytes(targetFile.length()) + .withMetrics(appender.metrics()); + if (partitionData != null) { + builder.withPartition(partitionData); + } + return builder.build(); + } + + /** + * Based on: https://github.com/apache/incubator-iceberg/blob/master/ + * spark/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java + */ + public static Record createSimpleRecord(long id, String data) { + Schema schema = new Schema(required(1, "id", Types.StringType.get()), + optional(2, "data", Types.LongType.get())); + GenericRecord record = GenericRecord.create(schema); + record.setField("id", id); + record.setField("data", data); + return record; + } + + public static Record createCustomRecord(Schema schema, List dataValues) { + GenericRecord record = GenericRecord.create(schema); + List fields = schema.columns(); + for (int i = 0; i < fields.size(); i++) { + record.setField(fields.get(i).name(), dataValues.get(i)); + } + return record; + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java new file mode 100644 index 000000000000..ab05672746cc --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.List; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Types; +import org.junit.Test; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.junit.Assert.assertEquals; + +public class TestIcebergObjectInspectorGenerator { + + @Test + public void testGetColumnNames() throws Exception { + Schema schema = new Schema(optional(1, "name", Types.StringType.get()), + optional(2, "salary", Types.LongType.get())); + IcebergObjectInspectorGenerator oi = new IcebergObjectInspectorGenerator(); + + List fieldsNames = oi.setColumnNames(schema); + assertEquals(fieldsNames.size(), 2); + } +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java new file mode 100644 index 000000000000..22865b84b91d --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Types; +import org.junit.Test; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertEquals; + +public class TestIcebergSchemaToTypeInfo { + + @Test + public void testGeneratePrimitiveTypeInfo() throws Exception { + Schema schema = new Schema( + required(1, "id", Types.IntegerType.get()), + optional(2, "data", Types.StringType.get()), + required(8, "feature1", Types.BooleanType.get()), + required(12, "lat", Types.FloatType.get()), + required(15, "x", Types.LongType.get()), + required(16, "date", Types.DateType.get()), + required(17, "double", Types.DoubleType.get()), + required(18, "binary", Types.BinaryType.get())); + List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + assertEquals(8, types.size()); + } + + @Test + public void testGenerateMapTypeInfo() throws Exception { + TypeInfo expected = TypeInfoFactory.getMapTypeInfo( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); + + Schema schema = new Schema( + optional(7, "properties", Types.MapType.ofOptional(18, 19, + Types.StringType.get(), + Types.StringType.get() + ), "string map of properties")); + + List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + assertEquals(1, types.size()); + assertEquals(expected, types.get(0)); + } + + @Test + public void testGenerateListTypeInfo() throws Exception { + TypeInfo expected = TypeInfoFactory + .getListTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)); + Schema schema = new Schema( + required(6, "doubles", Types.ListType.ofRequired(17, + Types.DoubleType.get() + ))); + List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + assertEquals(1, types.size()); + assertEquals(expected, types.get(0)); + } + + @Test + public void testGenerateMapAndStructTypeInfo() throws Exception { + List names1 = new ArrayList<>(Arrays.asList("address", "city", "state", "zip")); + List typeInfo1 = new ArrayList<>(Arrays.asList( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME) + )); + TypeInfo mapKeyStructExpected = TypeInfoFactory.getStructTypeInfo(names1, typeInfo1); + + List names2 = new ArrayList<>(Arrays.asList("lat", "long")); + List typeInfo2 = new ArrayList<>(Arrays.asList( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME) + )); + TypeInfo mapValueStructExpected = TypeInfoFactory.getStructTypeInfo(names2, typeInfo2); + + TypeInfo expected = TypeInfoFactory.getMapTypeInfo(mapKeyStructExpected, mapValueStructExpected); + + Schema schema = new Schema( + required(4, "locations", Types.MapType.ofRequired(10, 11, + Types.StructType.of( + required(20, "address", Types.StringType.get()), + required(21, "city", Types.StringType.get()), + required(22, "state", Types.StringType.get()), + required(23, "zip", Types.IntegerType.get()) + ), + Types.StructType.of( + required(12, "lat", Types.FloatType.get()), + required(13, "long", Types.FloatType.get()) + )), "map of address to coordinate")); + List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + assertEquals(1, types.size()); + assertEquals(expected, types.get(0)); + } + + @Test + public void testComplexSchema() throws Exception { + Schema schema = new Schema( + required(1, "id", Types.IntegerType.get()), + optional(2, "data", Types.StringType.get()), + optional(3, "preferences", Types.StructType.of( + required(8, "feature1", Types.BooleanType.get()), + optional(9, "feature2", Types.BooleanType.get()) + ), "struct of named boolean options"), + required(6, "doubles", Types.ListType.ofRequired(17, + Types.DoubleType.get() + )), + optional(7, "properties", Types.MapType.ofOptional(18, 19, + Types.StringType.get(), + Types.StringType.get() + ), "string map of properties") + ); + List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + assertEquals(5, types.size()); + assertEquals(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME), types.get(0)); + assertEquals(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), types.get(1)); + + List preferencesNames = new ArrayList<>(Arrays.asList("feature1", "feature2")); + List preferencesTypeInfo = new ArrayList<>(Arrays.asList( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME) + )); + TypeInfo preferencesTypeExpected = TypeInfoFactory.getStructTypeInfo(preferencesNames, preferencesTypeInfo); + assertEquals(preferencesTypeExpected, types.get(2)); + assertEquals(TypeInfoFactory.getListTypeInfo( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)), types.get(3)); + + TypeInfo propertiesExpected = TypeInfoFactory.getMapTypeInfo( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); + assertEquals(propertiesExpected, types.get(4)); + } +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java new file mode 100644 index 000000000000..88d7043ccbbe --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestIcebergSerDe { + + private File tableLocation; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Before + public void before() throws IOException { + tableLocation = temp.newFolder(); + Schema schema = new Schema(optional(1, "name", Types.StringType.get()), + optional(2, "salary", Types.LongType.get())); + PartitionSpec spec = PartitionSpec.unpartitioned(); + + Configuration conf = new Configuration(); + HadoopCatalog catalog = new HadoopCatalog(conf, tableLocation.getAbsolutePath()); + TableIdentifier id = TableIdentifier.parse("source_db.table_a"); + Table table = catalog.createTable(id, schema, spec); + + List data = new ArrayList<>(); + data.add(TestHelpers.createCustomRecord(schema, Arrays.asList("Michael", 3000L))); + data.add(TestHelpers.createCustomRecord(schema, Arrays.asList("Andy", 3000L))); + data.add(TestHelpers.createCustomRecord(schema, Arrays.asList("Berta", 4000L))); + + DataFile fileA = TestHelpers.writeFile(temp.newFile(), table, null, FileFormat.PARQUET, data); + + table.newAppend().appendFile(fileA).commit(); + } + + @Test + public void testDeserializeMap() { + Schema schema = new Schema(required(1, "map_type", Types.MapType + .ofRequired(18, 19, Types.StringType.get(), Types.StringType.get()))); + Map expected = ImmutableMap.of("foo", "bar"); + List data = new ArrayList<>(); + data.add(expected); + + Record record = TestHelpers.createCustomRecord(schema, data); + IcebergWritable writable = new IcebergWritable(); + writable.setRecord(record); + writable.setSchema(schema); + + IcebergSerDe serDe = new IcebergSerDe(); + List deserialized = (List) serDe.deserialize(writable); + Map result = (Map) deserialized.get(0); + + assertEquals(expected, result); + assertTrue(result.containsKey("foo")); + assertTrue(result.containsValue("bar")); + } + + @Test + public void testDeserializeList() { + Schema schema = new Schema(required(1, "list_type", Types.ListType.ofRequired(17, Types.LongType.get()))); + List expected = Arrays.asList(1000L, 2000L, 3000L); + List data = new ArrayList<>(); + data.add(expected); + + Record record = TestHelpers.createCustomRecord(schema, data); + IcebergWritable writable = new IcebergWritable(); + writable.setRecord(record); + writable.setSchema(schema); + + IcebergSerDe serDe = new IcebergSerDe(); + List deserialized = (List) serDe.deserialize(writable); + List result = (List) deserialized.get(0); + + assertEquals(expected, result); + } + + @Test + public void testDeserializePrimitives() { + Schema schema = new Schema(required(1, "string_type", Types.StringType.get()), + required(2, "int_type", Types.IntegerType.get()), + required(3, "long_type", Types.LongType.get()), + required(4, "boolean_type", Types.BooleanType.get()), + required(5, "float_type", Types.FloatType.get()), + required(6, "double_type", Types.DoubleType.get()), + required(7, "date_type", Types.DateType.get())); + + List expected = Arrays.asList("foo", 12, 3000L, true, 3.01F, 3.0D, "1998-11-13"); + + Record record = TestHelpers.createCustomRecord(schema, expected); + IcebergWritable writable = new IcebergWritable(); + writable.setRecord(record); + writable.setSchema(schema); + + IcebergSerDe serDe = new IcebergSerDe(); + List result = (List) serDe.deserialize(writable); + + assertEquals(expected, result); + } + + @Test + public void testDeserializeNestedList() { + Schema schema = new Schema(required(1, "map_type", Types.MapType + .ofRequired(18, 19, Types.StringType.get(), Types.ListType.ofRequired(17, Types.LongType.get())))); + Map expected = ImmutableMap.of("foo", Arrays.asList(1000L, 2000L, 3000L)); + List data = new ArrayList<>(); + data.add(expected); + + Record record = TestHelpers.createCustomRecord(schema, data); + IcebergWritable writable = new IcebergWritable(); + writable.setRecord(record); + writable.setSchema(schema); + + IcebergSerDe serDe = new IcebergSerDe(); + List deserialized = (List) serDe.deserialize(writable); + Map result = (Map) deserialized.get(0); + + assertEquals(expected, result); + assertTrue(result.containsKey("foo")); + assertTrue(result.containsValue(Arrays.asList(1000L, 2000L, 3000L))); + } +} From c0cec1786f33ae4dafe46e0aa8be61e5acebea4e Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Tue, 9 Jun 2020 09:45:12 +0100 Subject: [PATCH 04/14] Remove try/catch --- .../apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java index 6190c6d22938..38e4068292ea 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -77,12 +77,7 @@ private static TypeInfo generateTypeInfo(Type type) throws Exception { Types.DecimalType dec = (Types.DecimalType) type; int scale = dec.scale(); int precision = dec.precision(); - try { - HiveDecimalUtils.validateParameter(precision, scale); - } catch (Exception e) { - //TODO Log that precision / scale isn't valid - throw e; - } + HiveDecimalUtils.validateParameter(precision, scale); return TypeInfoFactory.getDecimalTypeInfo(precision, scale); case STRUCT: return generateStructTypeInfo((Types.StructType) type); From e4ccfd7b996c68fb4ad82d7351c244baa43b4856 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Thu, 11 Jun 2020 17:52:08 +0100 Subject: [PATCH 05/14] Fix date/time types, address format comments --- .../mr/mapred/IcebergSchemaToTypeInfo.java | 15 ++- .../iceberg/mr/mapred/IcebergSerDe.java | 39 ++++++-- .../iceberg/mr/mapred/IcebergWritable.java | 24 ++--- .../iceberg/mr/mapred/SystemTableUtil.java | 22 ++--- .../apache/iceberg/mr/mapred/TestHelpers.java | 8 +- .../iceberg/mr/mapred/TestIcebergSerDe.java | 99 ++++++++----------- 6 files changed, 103 insertions(+), 104 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java index 38e4068292ea..edc622f73a5f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -37,7 +37,6 @@ final class IcebergSchemaToTypeInfo { private IcebergSchemaToTypeInfo() { - } private static final ImmutableMap primitiveTypeToTypeInfo = ImmutableMap.builder() @@ -49,8 +48,8 @@ private IcebergSchemaToTypeInfo() { .put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME)) .put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)) .put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME)) - .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) - .put(Types.TimestampType.withZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) + .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)) + .put(Types.TimestampType.withZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)) .build(); public static List getColumnTypes(Schema schema) throws Exception { @@ -70,15 +69,13 @@ private static TypeInfo generateTypeInfo(Type type) throws Exception { case UUID: return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); case FIXED: - return TypeInfoFactory.getPrimitiveTypeInfo("binary"); + return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); case TIME: - return TypeInfoFactory.getPrimitiveTypeInfo("long"); + return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); case DECIMAL: Types.DecimalType dec = (Types.DecimalType) type; - int scale = dec.scale(); - int precision = dec.precision(); - HiveDecimalUtils.validateParameter(precision, scale); - return TypeInfoFactory.getDecimalTypeInfo(precision, scale); + HiveDecimalUtils.validateParameter(dec.precision(), dec.scale()); + return TypeInfoFactory.getDecimalTypeInfo(dec.precision(), dec.scale()); case STRUCT: return generateStructTypeInfo((Types.StructType) type); case LIST: diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java index 8fe331a1293b..56b842e53866 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -21,6 +21,12 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -35,12 +41,14 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.SnapshotsTable; import org.apache.iceberg.Table; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; public class IcebergSerDe extends AbstractSerDe { private Schema schema; private ObjectInspector inspector; + private List row; @Override public void initialize(@Nullable Configuration configuration, Properties serDeProperties) throws SerDeException { @@ -59,10 +67,9 @@ public void initialize(@Nullable Configuration configuration, Properties serDePr } } else { List columns = new ArrayList<>(schema.columns()); - columns.add(Types.NestedField.optional(Integer.MAX_VALUE, SystemTableUtil.getVirtualColumnName(serDeProperties), - Types.LongType.get())); + columns.add(Types.NestedField.optional(Integer.MAX_VALUE, + SystemTableUtil.snapshotIdVirtualColumnName(serDeProperties), Types.LongType.get())); Schema withVirtualColumn = new Schema(columns); - try { this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(withVirtualColumn); } catch (Exception e) { @@ -89,12 +96,28 @@ public SerDeStats getSerDeStats() { @Override public Object deserialize(Writable writable) { IcebergWritable icebergWritable = (IcebergWritable) writable; - List fields = icebergWritable.getSchema().columns(); - List row = new ArrayList<>(); + List fields = icebergWritable.schema().columns(); - for (Types.NestedField field : fields) { - Object obj = ((IcebergWritable) writable).getRecord().getField(field.name()); - row.add(obj); + if (row == null || row.size() != fields.size()) { + row = new ArrayList(fields.size()); + } else { + row.clear(); + } + for (int i = 0; i < fields.size(); i++) { + Object obj = ((IcebergWritable) writable).record().get(i); + Type fieldType = fields.get(i).type(); + if (fieldType.equals(Types.DateType.get())) { + row.add(Date.valueOf((LocalDate) obj)); + } else if (fieldType.equals(Types.TimestampType.withoutZone())) { + row.add(Timestamp.valueOf((LocalDateTime) obj)); + } else if (fieldType.equals(Types.TimestampType.withZone())) { + LocalDateTime timestamp = ((OffsetDateTime) obj).toLocalDateTime(); + row.add(Timestamp.valueOf(timestamp)); + } else if (fieldType.equals(Types.TimeType.get())) { + row.add(((LocalTime) obj).toString()); + } else { + row.add(obj); + } } return Collections.unmodifiableList(row); } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java index 8b0eb79fbe73..3f3772e27e9e 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java @@ -21,41 +21,43 @@ import java.io.DataInput; import java.io.DataOutput; -import java.io.IOException; import org.apache.hadoop.io.Writable; import org.apache.iceberg.Schema; import org.apache.iceberg.data.Record; +/** + * Wraps an Iceberg Record in a Writable which Hive can use in the SerDe. + */ public class IcebergWritable implements Writable { private Record record; private Schema schema; - public IcebergWritable() {} - - public void setRecord(Record record) { + @SuppressWarnings("checkstyle:HiddenField") + public void wrapRecord(Record record) { this.record = record; } - public Record getRecord() { + public Record record() { return record; } - public Schema getSchema() { + public Schema schema() { return schema; } - public void setSchema(Schema schema) { + @SuppressWarnings("checkstyle:HiddenField") + public void wrapSchema(Schema schema) { this.schema = schema; } @Override - public void write(DataOutput dataOutput) throws IOException { - + public void write(DataOutput dataOutput) { + throw new UnsupportedOperationException("write is not supported."); } @Override - public void readFields(DataInput dataInput) throws IOException { - + public void readFields(DataInput dataInput) { + throw new UnsupportedOperationException("readFields is not supported."); } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java index 17f5b9fb07b2..105a4ea13a98 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java @@ -19,13 +19,13 @@ package org.apache.iceberg.mr.mapred; -import java.util.ArrayList; import java.util.List; import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; public class SystemTableUtil { @@ -36,24 +36,24 @@ public class SystemTableUtil { private SystemTableUtil() {} - protected static Schema schemaWithVirtualColumn(Schema schema, String columnName) { - List columns = new ArrayList<>(schema.columns()); + protected static Schema schemaWithSnapshotIdVirtualColumn(Schema schema, String columnName) { + List columns = Lists.newArrayList(schema.columns()); columns.add(Types.NestedField.optional(Integer.MAX_VALUE, columnName, Types.LongType.get())); return new Schema(columns); } - protected static Record recordWithVirtualColumn(Record record, long snapshotId, Schema oldSchema, - String columnName) { - Schema newSchema = schemaWithVirtualColumn(oldSchema, columnName); + protected static Record recordWithSnapshotIdVirtualColumn(Record record, long snapshotId, Schema oldSchema, + String virtualColumnName) { + Schema newSchema = schemaWithSnapshotIdVirtualColumn(oldSchema, virtualColumnName); Record newRecord = GenericRecord.create(newSchema); - for (Types.NestedField field : oldSchema.columns()) { - newRecord.setField(field.name(), record.getField(field.name())); + for (int i = 0; i < oldSchema.columns().size(); i++) { + newRecord.set(i, record.get(i)); } - newRecord.setField(columnName, snapshotId); + newRecord.setField(virtualColumnName, snapshotId); return newRecord; } - protected static String getVirtualColumnName(Configuration conf) { + protected static String snapshotIdVirtualColumnName(Configuration conf) { String virtualColumnName = conf.get(VIRTUAL_COLUMN_NAME); if (virtualColumnName == null) { return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; @@ -62,7 +62,7 @@ protected static String getVirtualColumnName(Configuration conf) { } } - protected static String getVirtualColumnName(Properties properties) { + protected static String snapshotIdVirtualColumnName(Properties properties) { String virtualColumnName = properties.getProperty(VIRTUAL_COLUMN_NAME); if (virtualColumnName == null) { return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java index 7f96979fa7ed..0d8cdc036e7f 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHelpers.java @@ -98,10 +98,6 @@ public static DataFile writeFile(File targetFile, Table table, StructLike partit return builder.build(); } - /** - * Based on: https://github.com/apache/incubator-iceberg/blob/master/ - * spark/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java - */ public static Record createSimpleRecord(long id, String data) { Schema schema = new Schema(required(1, "id", Types.StringType.get()), optional(2, "data", Types.LongType.get())); @@ -111,11 +107,11 @@ public static Record createSimpleRecord(long id, String data) { return record; } - public static Record createCustomRecord(Schema schema, List dataValues) { + public static Record createCustomRecord(Schema schema, Object... dataValues) { GenericRecord record = GenericRecord.create(schema); List fields = schema.columns(); for (int i = 0; i < fields.size(); i++) { - record.setField(fields.get(i).name(), dataValues.get(i)); + record.set(i, dataValues[i]); } return record; } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java index 88d7043ccbbe..b397376e6b70 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java @@ -19,95 +19,78 @@ package org.apache.iceberg.mr.mapred; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import static org.apache.iceberg.types.Types.NestedField.optional; import static org.apache.iceberg.types.Types.NestedField.required; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestIcebergSerDe { - private File tableLocation; - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Before - public void before() throws IOException { - tableLocation = temp.newFolder(); - Schema schema = new Schema(optional(1, "name", Types.StringType.get()), - optional(2, "salary", Types.LongType.get())); - PartitionSpec spec = PartitionSpec.unpartitioned(); - - Configuration conf = new Configuration(); - HadoopCatalog catalog = new HadoopCatalog(conf, tableLocation.getAbsolutePath()); - TableIdentifier id = TableIdentifier.parse("source_db.table_a"); - Table table = catalog.createTable(id, schema, spec); + @Test + public void testDeserializeWritable() { + Schema schema = new Schema(required(1, "string_type", Types.StringType.get()), + required(2, "int_type", Types.IntegerType.get()), + required(3, "long_type", Types.LongType.get()), + required(4, "boolean_type", Types.BooleanType.get()), + required(5, "float_type", Types.FloatType.get()), + required(6, "double_type", Types.DoubleType.get()), + required(7, "binary_type", Types.BinaryType.get()), + required(8, "date_type", Types.DateType.get()), + required(9, "timestamp_with_zone_type", Types.TimestampType.withZone()), + required(10, "timestamp_without_zone_type", Types.TimestampType.withoutZone()), + required(11, "map_type", Types.MapType + .ofRequired(12, 13, Types.IntegerType.get(), Types.StringType.get())), + required(14, "list_type", Types.ListType.ofRequired(15, Types.LongType.get())) + ); + Map expected = ImmutableMap.of("foo", "bar"); - List data = new ArrayList<>(); - data.add(TestHelpers.createCustomRecord(schema, Arrays.asList("Michael", 3000L))); - data.add(TestHelpers.createCustomRecord(schema, Arrays.asList("Andy", 3000L))); - data.add(TestHelpers.createCustomRecord(schema, Arrays.asList("Berta", 4000L))); + Record record = TestHelpers.createCustomRecord(schema, expected); + IcebergWritable writable = new IcebergWritable(); + writable.wrapRecord(record); + writable.wrapSchema(schema); - DataFile fileA = TestHelpers.writeFile(temp.newFile(), table, null, FileFormat.PARQUET, data); + IcebergSerDe serDe = new IcebergSerDe(); + List deserialized = (List) serDe.deserialize(writable); + Map result = (Map) deserialized.get(0); - table.newAppend().appendFile(fileA).commit(); + assertEquals(expected, result); } @Test - public void testDeserializeMap() { + public void testDeserializeMapWithIntKeyType() { Schema schema = new Schema(required(1, "map_type", Types.MapType - .ofRequired(18, 19, Types.StringType.get(), Types.StringType.get()))); - Map expected = ImmutableMap.of("foo", "bar"); - List data = new ArrayList<>(); - data.add(expected); + .ofRequired(18, 19, Types.IntegerType.get(), Types.StringType.get()))); + Map expected = ImmutableMap.of(22, "bar"); - Record record = TestHelpers.createCustomRecord(schema, data); + Record record = TestHelpers.createCustomRecord(schema, expected); IcebergWritable writable = new IcebergWritable(); - writable.setRecord(record); - writable.setSchema(schema); + writable.wrapRecord(record); + writable.wrapSchema(schema); IcebergSerDe serDe = new IcebergSerDe(); List deserialized = (List) serDe.deserialize(writable); Map result = (Map) deserialized.get(0); assertEquals(expected, result); - assertTrue(result.containsKey("foo")); - assertTrue(result.containsValue("bar")); } @Test public void testDeserializeList() { Schema schema = new Schema(required(1, "list_type", Types.ListType.ofRequired(17, Types.LongType.get()))); List expected = Arrays.asList(1000L, 2000L, 3000L); - List data = new ArrayList<>(); - data.add(expected); - Record record = TestHelpers.createCustomRecord(schema, data); + Record record = TestHelpers.createCustomRecord(schema, expected); IcebergWritable writable = new IcebergWritable(); - writable.setRecord(record); - writable.setSchema(schema); + writable.wrapRecord(record); + writable.wrapSchema(schema); IcebergSerDe serDe = new IcebergSerDe(); List deserialized = (List) serDe.deserialize(writable); @@ -128,10 +111,10 @@ public void testDeserializePrimitives() { List expected = Arrays.asList("foo", 12, 3000L, true, 3.01F, 3.0D, "1998-11-13"); - Record record = TestHelpers.createCustomRecord(schema, expected); + Record record = TestHelpers.createCustomRecord(schema, "foo", 12, 3000L, true, 3.01F, 3.0D, "1998-11-13"); IcebergWritable writable = new IcebergWritable(); - writable.setRecord(record); - writable.setSchema(schema); + writable.wrapRecord(record); + writable.wrapSchema(schema); IcebergSerDe serDe = new IcebergSerDe(); List result = (List) serDe.deserialize(writable); @@ -144,13 +127,11 @@ public void testDeserializeNestedList() { Schema schema = new Schema(required(1, "map_type", Types.MapType .ofRequired(18, 19, Types.StringType.get(), Types.ListType.ofRequired(17, Types.LongType.get())))); Map expected = ImmutableMap.of("foo", Arrays.asList(1000L, 2000L, 3000L)); - List data = new ArrayList<>(); - data.add(expected); - Record record = TestHelpers.createCustomRecord(schema, data); + Record record = TestHelpers.createCustomRecord(schema, expected); IcebergWritable writable = new IcebergWritable(); - writable.setRecord(record); - writable.setSchema(schema); + writable.wrapRecord(record); + writable.wrapSchema(schema); IcebergSerDe serDe = new IcebergSerDe(); List deserialized = (List) serDe.deserialize(writable); From eecb8336d66c3922e2b37552be35418abce5dd0b Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Fri, 12 Jun 2020 14:47:55 +0100 Subject: [PATCH 06/14] Remove HadoopCatalog, clean up SerDe and tests --- .../iceberg/mr/mapred/IcebergSerDe.java | 2 +- .../iceberg/mr/mapred/TableResolver.java | 77 +++--------- .../mapred/TestIcebergSchemaToTypeInfo.java | 119 ++++++------------ .../iceberg/mr/mapred/TestIcebergSerDe.java | 105 ++++------------ .../iceberg/mr/mapred/TestTableResolver.java | 12 -- 5 files changed, 77 insertions(+), 238 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java index 56b842e53866..0bddac485f9e 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -85,7 +85,7 @@ public Class getSerializedClass() { @Override public Writable serialize(Object o, ObjectInspector objectInspector) { - return null; + throw new UnsupportedOperationException("Serialization is not supported."); } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java index b6e2d879cbe7..d85fd4dc62e0 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -20,17 +20,14 @@ package org.apache.iceberg.mr.mapred; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Properties; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.iceberg.Table; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; final class TableResolver { @@ -39,68 +36,37 @@ private TableResolver() { static Table resolveTableFromJob(JobConf conf) throws IOException { Properties properties = new Properties(); - properties.setProperty(InputFormatConfig.CATALOG_NAME, extractProperty(conf, InputFormatConfig.CATALOG_NAME)); - if (conf.get(InputFormatConfig.CATALOG_NAME).equals(InputFormatConfig.HADOOP_CATALOG)) { - properties.setProperty(InputFormatConfig.SNAPSHOT_TABLE, conf.get(InputFormatConfig.SNAPSHOT_TABLE, "true")); - } + properties.setProperty(InputFormatConfig.CATALOG_NAME, + conf.get(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES)); //Default to HadoopTables + properties.setProperty(InputFormatConfig.SNAPSHOT_TABLE, + conf.get(InputFormatConfig.SNAPSHOT_TABLE, "true")); properties.setProperty(InputFormatConfig.TABLE_LOCATION, extractProperty(conf, InputFormatConfig.TABLE_LOCATION)); properties.setProperty(InputFormatConfig.TABLE_NAME, extractProperty(conf, InputFormatConfig.TABLE_NAME)); return resolveTableFromConfiguration(conf, properties); } static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { - String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME); - URI tableLocation = pathAsURI(properties.getProperty(InputFormatConfig.TABLE_LOCATION)); - if (catalogName == null) { - throw new IllegalArgumentException("Catalog property: 'iceberg.catalog' not set in JobConf"); - } + String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + String tableLocation = properties.getProperty(InputFormatConfig.TABLE_LOCATION); + String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); + Preconditions.checkNotNull(tableLocation, "Table location is not set."); + Preconditions.checkNotNull(tableName, "Table name is not set."); switch (catalogName) { case InputFormatConfig.HADOOP_TABLES: HadoopTables tables = new HadoopTables(conf); - return tables.load(tableLocation.getPath()); - case InputFormatConfig.HADOOP_CATALOG: - String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); - TableIdentifier id = TableIdentifier.parse(tableName); if (tableName.endsWith(InputFormatConfig.SNAPSHOT_TABLE_SUFFIX)) { - if (!Boolean.parseBoolean(properties.getProperty(InputFormatConfig.SNAPSHOT_TABLE, - Boolean.TRUE.toString()))) { - String tablePath = id.toString().replaceAll("\\.", "/"); - URI warehouseLocation = pathAsURI(tableLocation.getPath().replaceAll(tablePath, "")); - HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); - return catalog.loadTable(id); - } else { - return resolveMetadataTable(conf, tableLocation.getPath(), tableName); + if (!Boolean.parseBoolean(properties.getProperty( + InputFormatConfig.SNAPSHOT_TABLE, Boolean.TRUE.toString()))) { + return tables.load(tableLocation); } - } else { - URI warehouseLocation = pathAsURI(extractWarehousePath(tableLocation.getPath(), tableName)); - HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); - return catalog.loadTable(id); + return tables.load(tableLocation + "#snapshots"); } + return tables.load(tableLocation); case InputFormatConfig.HIVE_CATALOG: //TODO Implement HiveCatalog return null; - } - return null; - } - - static Table resolveMetadataTable(Configuration conf, String location, String tableName) throws IOException { - URI warehouseLocation = pathAsURI(extractWarehousePath(location, tableName)); - HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); - String baseTableName = StringUtils.removeEnd(tableName, InputFormatConfig.SNAPSHOT_TABLE_SUFFIX); - - TableIdentifier snapshotsId = TableIdentifier.parse(baseTableName + - InputFormatConfig.ICEBERG_SNAPSHOTS_TABLE_SUFFIX); - return catalog.loadTable(snapshotsId); - } - - static URI pathAsURI(String path) throws IOException { - if (path == null) { - throw new IllegalArgumentException("Path is null."); - } - try { - return new URI(path); - } catch (URISyntaxException e) { - throw new IOException("Unable to create URI for table location: '" + path + "'", e); + default: + throw new NoSuchTableException("Table does not exist at location: " + tableLocation); } } @@ -111,11 +77,4 @@ protected static String extractProperty(JobConf conf, String key) { } return value; } - - protected static String extractWarehousePath(String location, String tableName) { - String tablePath = tableName.replaceAll("\\.", "/").replaceAll( - InputFormatConfig.SNAPSHOT_TABLE_SUFFIX, ""); - return location.replaceAll(tablePath, ""); - } - } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java index 22865b84b91d..794d3f540b20 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java @@ -19,8 +19,6 @@ package org.apache.iceberg.mr.mapred; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -48,11 +46,27 @@ public void testGeneratePrimitiveTypeInfo() throws Exception { required(18, "binary", Types.BinaryType.get())); List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - assertEquals(8, types.size()); + assertEquals("Converted TypeInfo should have the same number of columns.", 8, types.size()); + assertEquals("IntegerType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME), types.get(0)); + assertEquals("StringType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), types.get(1)); + assertEquals("BooleanType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME), types.get(2)); + assertEquals("FloatType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME), types.get(3)); + assertEquals("LongType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME), types.get(4)); + assertEquals("DateType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME), types.get(5)); + assertEquals("DoubleType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME), types.get(6)); + assertEquals("BinaryType converted incorrectly.", + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME), types.get(7)); } @Test - public void testGenerateMapTypeInfo() throws Exception { + public void testGenerateMapWithStringKeyTypeInfo() throws Exception { TypeInfo expected = TypeInfoFactory.getMapTypeInfo( TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); @@ -65,98 +79,39 @@ public void testGenerateMapTypeInfo() throws Exception { List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - assertEquals(1, types.size()); - assertEquals(expected, types.get(0)); + assertEquals("Converted TypeInfo should have the same number of columns.", 1, types.size()); + assertEquals("MapType converted incorrectly.", expected, types.get(0)); } @Test - public void testGenerateListTypeInfo() throws Exception { - TypeInfo expected = TypeInfoFactory - .getListTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)); - Schema schema = new Schema( - required(6, "doubles", Types.ListType.ofRequired(17, - Types.DoubleType.get() - ))); - List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - assertEquals(1, types.size()); - assertEquals(expected, types.get(0)); - } - - @Test - public void testGenerateMapAndStructTypeInfo() throws Exception { - List names1 = new ArrayList<>(Arrays.asList("address", "city", "state", "zip")); - List typeInfo1 = new ArrayList<>(Arrays.asList( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME) - )); - TypeInfo mapKeyStructExpected = TypeInfoFactory.getStructTypeInfo(names1, typeInfo1); - - List names2 = new ArrayList<>(Arrays.asList("lat", "long")); - List typeInfo2 = new ArrayList<>(Arrays.asList( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME) - )); - TypeInfo mapValueStructExpected = TypeInfoFactory.getStructTypeInfo(names2, typeInfo2); - - TypeInfo expected = TypeInfoFactory.getMapTypeInfo(mapKeyStructExpected, mapValueStructExpected); + public void testGenerateMapWithIntKeyTypeInfo() throws Exception { + TypeInfo expected = TypeInfoFactory.getMapTypeInfo( + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME), + TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); Schema schema = new Schema( - required(4, "locations", Types.MapType.ofRequired(10, 11, - Types.StructType.of( - required(20, "address", Types.StringType.get()), - required(21, "city", Types.StringType.get()), - required(22, "state", Types.StringType.get()), - required(23, "zip", Types.IntegerType.get()) - ), - Types.StructType.of( - required(12, "lat", Types.FloatType.get()), - required(13, "long", Types.FloatType.get()) - )), "map of address to coordinate")); + optional(7, "properties", Types.MapType.ofOptional(18, 19, + Types.IntegerType.get(), + Types.StringType.get() + ), "string map of properties")); + List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - assertEquals(1, types.size()); - assertEquals(expected, types.get(0)); + assertEquals("Converted TypeInfo should have the same number of columns.", 1, types.size()); + assertEquals("MapType converted incorrectly.", expected, types.get(0)); } @Test - public void testComplexSchema() throws Exception { + public void testGenerateListTypeInfo() throws Exception { + TypeInfo expected = TypeInfoFactory + .getListTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)); Schema schema = new Schema( - required(1, "id", Types.IntegerType.get()), - optional(2, "data", Types.StringType.get()), - optional(3, "preferences", Types.StructType.of( - required(8, "feature1", Types.BooleanType.get()), - optional(9, "feature2", Types.BooleanType.get()) - ), "struct of named boolean options"), required(6, "doubles", Types.ListType.ofRequired(17, Types.DoubleType.get() - )), - optional(7, "properties", Types.MapType.ofOptional(18, 19, - Types.StringType.get(), - Types.StringType.get() - ), "string map of properties") - ); + ))); List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - assertEquals(5, types.size()); - assertEquals(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME), types.get(0)); - assertEquals(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), types.get(1)); - - List preferencesNames = new ArrayList<>(Arrays.asList("feature1", "feature2")); - List preferencesTypeInfo = new ArrayList<>(Arrays.asList( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME) - )); - TypeInfo preferencesTypeExpected = TypeInfoFactory.getStructTypeInfo(preferencesNames, preferencesTypeInfo); - assertEquals(preferencesTypeExpected, types.get(2)); - assertEquals(TypeInfoFactory.getListTypeInfo( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)), types.get(3)); - - TypeInfo propertiesExpected = TypeInfoFactory.getMapTypeInfo( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); - assertEquals(propertiesExpected, types.get(4)); + assertEquals("Converted TypeInfo should have the same number of columns.", 1, types.size()); + assertEquals("ListType converted incorrectly.", expected, types.get(0)); } } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java index b397376e6b70..c81d1a263090 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java @@ -19,18 +19,23 @@ package org.apache.iceberg.mr.mapred; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.iceberg.Schema; import org.apache.iceberg.data.Record; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.junit.Test; import static org.apache.iceberg.types.Types.NestedField.required; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertArrayEquals; public class TestIcebergSerDe { @@ -50,95 +55,27 @@ public void testDeserializeWritable() { .ofRequired(12, 13, Types.IntegerType.get(), Types.StringType.get())), required(14, "list_type", Types.ListType.ofRequired(15, Types.LongType.get())) ); - Map expected = ImmutableMap.of("foo", "bar"); + LocalDate localDate = LocalDate.of(2018, 11, 10); + LocalDateTime localDateTime = LocalDateTime.of(2018, 11, 10, 11, 55); + OffsetDateTime offsetDateTime = OffsetDateTime.of(localDateTime, ZoneOffset.UTC); - Record record = TestHelpers.createCustomRecord(schema, expected); - IcebergWritable writable = new IcebergWritable(); - writable.wrapRecord(record); - writable.wrapSchema(schema); + Object[] input = Lists.newArrayList("foo", 5, 6L, true, 1.02F, 1.4D, new byte[] { (byte) 0xe0}, + localDate, offsetDateTime, localDateTime, ImmutableMap.of(22, "bar"), + Arrays.asList(1000L, 2000L, 3000L)).toArray(); - IcebergSerDe serDe = new IcebergSerDe(); - List deserialized = (List) serDe.deserialize(writable); - Map result = (Map) deserialized.get(0); - - assertEquals(expected, result); - } + //Inputs and outputs differ slightly because of Date/Timestamp conversions for Hive + Object[] expected = Lists.newArrayList("foo", 5, 6L, true, 1.02F, 1.4D, new byte[] { (byte) 0xe0}, + Date.valueOf(localDate), Timestamp.valueOf(offsetDateTime.toLocalDateTime()), Timestamp.valueOf(localDateTime), + ImmutableMap.of(22, "bar"), Arrays.asList(1000L, 2000L, 3000L)).toArray(); - @Test - public void testDeserializeMapWithIntKeyType() { - Schema schema = new Schema(required(1, "map_type", Types.MapType - .ofRequired(18, 19, Types.IntegerType.get(), Types.StringType.get()))); - Map expected = ImmutableMap.of(22, "bar"); - - Record record = TestHelpers.createCustomRecord(schema, expected); + Record record = TestHelpers.createCustomRecord(schema, input); IcebergWritable writable = new IcebergWritable(); writable.wrapRecord(record); writable.wrapSchema(schema); IcebergSerDe serDe = new IcebergSerDe(); List deserialized = (List) serDe.deserialize(writable); - Map result = (Map) deserialized.get(0); - - assertEquals(expected, result); - } - - @Test - public void testDeserializeList() { - Schema schema = new Schema(required(1, "list_type", Types.ListType.ofRequired(17, Types.LongType.get()))); - List expected = Arrays.asList(1000L, 2000L, 3000L); - - Record record = TestHelpers.createCustomRecord(schema, expected); - IcebergWritable writable = new IcebergWritable(); - writable.wrapRecord(record); - writable.wrapSchema(schema); - - IcebergSerDe serDe = new IcebergSerDe(); - List deserialized = (List) serDe.deserialize(writable); - List result = (List) deserialized.get(0); - - assertEquals(expected, result); - } - - @Test - public void testDeserializePrimitives() { - Schema schema = new Schema(required(1, "string_type", Types.StringType.get()), - required(2, "int_type", Types.IntegerType.get()), - required(3, "long_type", Types.LongType.get()), - required(4, "boolean_type", Types.BooleanType.get()), - required(5, "float_type", Types.FloatType.get()), - required(6, "double_type", Types.DoubleType.get()), - required(7, "date_type", Types.DateType.get())); - - List expected = Arrays.asList("foo", 12, 3000L, true, 3.01F, 3.0D, "1998-11-13"); - - Record record = TestHelpers.createCustomRecord(schema, "foo", 12, 3000L, true, 3.01F, 3.0D, "1998-11-13"); - IcebergWritable writable = new IcebergWritable(); - writable.wrapRecord(record); - writable.wrapSchema(schema); - - IcebergSerDe serDe = new IcebergSerDe(); - List result = (List) serDe.deserialize(writable); - - assertEquals(expected, result); - } - - @Test - public void testDeserializeNestedList() { - Schema schema = new Schema(required(1, "map_type", Types.MapType - .ofRequired(18, 19, Types.StringType.get(), Types.ListType.ofRequired(17, Types.LongType.get())))); - Map expected = ImmutableMap.of("foo", Arrays.asList(1000L, 2000L, 3000L)); - - Record record = TestHelpers.createCustomRecord(schema, expected); - IcebergWritable writable = new IcebergWritable(); - writable.wrapRecord(record); - writable.wrapSchema(schema); - - IcebergSerDe serDe = new IcebergSerDe(); - List deserialized = (List) serDe.deserialize(writable); - Map result = (Map) deserialized.get(0); - - assertEquals(expected, result); - assertTrue(result.containsKey("foo")); - assertTrue(result.containsValue(Arrays.asList(1000L, 2000L, 3000L))); + assertArrayEquals("Test values from an Iceberg Record deserialize into expected Java objects.", + expected, deserialized.toArray()); } } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java index e4b9c87a51c9..c77a811d596b 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java @@ -26,18 +26,6 @@ public class TestTableResolver { - @Test - public void extractWarehouseLocationRegularTable() { - // This is the style of input expected from HiveConf - String testLocation = "some/folder/database/table_a"; - String testTableName = "database.table_a"; - - String expected = "some/folder/"; - String result = TableResolver.extractWarehousePath(testLocation, testTableName); - - assertEquals(expected, result); - } - @Test public void extractPropertyFromJobConf() { JobConf conf = new JobConf(); From 2350025f11bf3ce6df226d21a87d5c403ab50b53 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 15 Jun 2020 13:36:50 +0100 Subject: [PATCH 07/14] refactored code from IcebergInputFormat into InputFormatConfig --- .../apache/iceberg/mr/InputFormatConfig.java | 26 +++ .../mr/mapreduce/IcebergInputFormat.java | 206 +++++------------- .../mr/mapreduce/TestIcebergInputFormat.java | 102 +++++---- 3 files changed, 140 insertions(+), 194 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java index e462704efd95..f4b9aae0f55a 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -55,6 +55,12 @@ private InputFormatConfig() {} public static final String TABLE_LOCATION = "location"; public static final String TABLE_NAME = "name"; + public enum InMemoryDataModel { + PIG, + HIVE, + GENERIC // Default data model is of Iceberg Generics + } + public static class ConfigBuilder { private final Configuration conf; @@ -77,6 +83,16 @@ public ConfigBuilder project(Schema schema) { return this; } + public ConfigBuilder schema(Schema schema) { + conf.set(TABLE_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder readFrom(String path) { + conf.set(TABLE_PATH, path); + return this; + } + public ConfigBuilder reuseContainers(boolean reuse) { conf.setBoolean(InputFormatConfig.REUSE_CONTAINERS, reuse); return this; @@ -115,6 +131,16 @@ public ConfigBuilder catalogFunc(Class T is the in memory data model which can either be Pig tuples, Hive rows. Default is Iceberg records */ public class IcebergInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); - static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; - static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; - static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; - static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; - static final String READ_SCHEMA = "iceberg.mr.read.schema"; - static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; - static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; - static final String SPLIT_SIZE = "iceberg.mr.split.size"; - static final String TABLE_PATH = "iceberg.mr.table.path"; - static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - static final String LOCALITY = "iceberg.mr.locality"; - static final String CATALOG = "iceberg.mr.catalog"; - static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; - private transient List splits; - private enum InMemoryDataModel { - PIG, - HIVE, - GENERIC // Default data model is of Iceberg Generics - } - /** * Configures the {@code Job} to use the {@code IcebergInputFormat} and * returns a helper to add further configuration. * * @param job the {@code Job} to configure */ - public static ConfigBuilder configure(Job job) { + public static InputFormatConfig.ConfigBuilder configure(Job job) { job.setInputFormatClass(IcebergInputFormat.class); - return new ConfigBuilder(job.getConfiguration()); - } - - public static class ConfigBuilder { - private final Configuration conf; - - public ConfigBuilder(Configuration conf) { - this.conf = conf; - // defaults - conf.setEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); - conf.setBoolean(SKIP_RESIDUAL_FILTERING, false); - conf.setBoolean(CASE_SENSITIVE, true); - conf.setBoolean(REUSE_CONTAINERS, false); - conf.setBoolean(LOCALITY, false); - } - - public ConfigBuilder readFrom(String path) { - conf.set(TABLE_PATH, path); - Table table = findTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - return this; - } - - public ConfigBuilder filter(Expression expression) { - conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); - return this; - } - - public ConfigBuilder project(Schema schema) { - conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); - return this; - } - - public ConfigBuilder reuseContainers(boolean reuse) { - conf.setBoolean(REUSE_CONTAINERS, reuse); - return this; - } - - public ConfigBuilder caseSensitive(boolean caseSensitive) { - conf.setBoolean(CASE_SENSITIVE, caseSensitive); - return this; - } - - public ConfigBuilder snapshotId(long snapshotId) { - conf.setLong(SNAPSHOT_ID, snapshotId); - return this; - } - - public ConfigBuilder asOfTime(long asOfTime) { - conf.setLong(AS_OF_TIMESTAMP, asOfTime); - return this; - } - - public ConfigBuilder splitSize(long splitSize) { - conf.setLong(SPLIT_SIZE, splitSize); - return this; - } - - /** - * If this API is called. The input splits - * constructed will have host location information - */ - public ConfigBuilder preferLocality() { - conf.setBoolean(LOCALITY, true); - return this; - } - - public ConfigBuilder catalogFunc(Class> catalogFuncClass) { - conf.setClass(CATALOG, catalogFuncClass, Function.class); - return this; - } - - public ConfigBuilder useHiveRows() { - conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.HIVE.name()); - return this; - } - - public ConfigBuilder usePigTuples() { - conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.PIG.name()); - return this; - } - - /** - * Compute platforms pass down filters to data sources. If the data source cannot apply some filters, or only - * partially applies the filter, it will return the residual filter back. If the platform can correctly apply - * the residual filters, then it should call this api. Otherwise the current api will throw an exception if the - * passed in filter is not completely satisfied. - */ - public ConfigBuilder skipResidualFiltering() { - conf.setBoolean(SKIP_RESIDUAL_FILTERING, true); - return this; - } + return new InputFormatConfig.ConfigBuilder(job.getConfiguration()); } @Override @@ -218,36 +109,38 @@ public List getSplits(JobContext context) { Configuration conf = context.getConfiguration(); Table table = findTable(conf); TableScan scan = table.newScan() - .caseSensitive(conf.getBoolean(CASE_SENSITIVE, true)); - long snapshotId = conf.getLong(SNAPSHOT_ID, -1); + .caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); + long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); if (snapshotId != -1) { scan = scan.useSnapshot(snapshotId); } - long asOfTime = conf.getLong(AS_OF_TIMESTAMP, -1); + long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); if (asOfTime != -1) { scan = scan.asOfTime(asOfTime); } - long splitSize = conf.getLong(SPLIT_SIZE, 0); + long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); if (splitSize > 0) { scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); } - String schemaStr = conf.get(READ_SCHEMA); + String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); if (schemaStr != null) { scan.project(SchemaParser.fromJson(schemaStr)); } // TODO add a filter parser to get rid of Serialization - Expression filter = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); + Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); if (filter != null) { scan = scan.filter(filter); } splits = Lists.newArrayList(); - boolean applyResidual = !conf.getBoolean(SKIP_RESIDUAL_FILTERING, false); - InMemoryDataModel model = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); + boolean applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); + InputFormatConfig.InMemoryDataModel model = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, + InputFormatConfig.InMemoryDataModel.GENERIC); try (CloseableIterable tasksIterable = scan.planTasks()) { tasksIterable.forEach(task -> { - if (applyResidual && (model == InMemoryDataModel.HIVE || model == InMemoryDataModel.PIG)) { + if (applyResidual && (model == InputFormatConfig.InMemoryDataModel.HIVE || + model == InputFormatConfig.InMemoryDataModel.PIG)) { //TODO: We do not support residual evaluation for HIVE and PIG in memory data model yet checkResiduals(task); } @@ -265,9 +158,9 @@ private static void checkResiduals(CombinedScanTask task) { Expression residual = fileScanTask.residual(); if (residual != null && !residual.equals(Expressions.alwaysTrue())) { throw new UnsupportedOperationException( - String.format( - "Filter expression %s is not completely satisfied. Additional rows " + - "can be returned not satisfied by the filter expression", residual)); + String.format( + "Filter expression %s is not completely satisfied. Additional rows " + + "can be returned not satisfied by the filter expression", residual)); } }); } @@ -283,7 +176,7 @@ private static final class IcebergRecordReader extends RecordReader private Schema expectedSchema; private boolean reuseContainers; private boolean caseSensitive; - private InMemoryDataModel inMemoryDataModel; + private InputFormatConfig.InMemoryDataModel inMemoryDataModel; private Map namesToPos; private Iterator tasks; private T currentRow; @@ -296,13 +189,14 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { CombinedScanTask task = ((IcebergSplit) split).task; this.context = newContext; this.tasks = task.files().iterator(); - this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); - String readSchemaStr = conf.get(READ_SCHEMA); + this.tableSchema = SchemaParser.fromJson(conf.get(InputFormatConfig.TABLE_SCHEMA)); + String readSchemaStr = conf.get(InputFormatConfig.READ_SCHEMA); this.expectedSchema = readSchemaStr != null ? SchemaParser.fromJson(readSchemaStr) : tableSchema; this.namesToPos = buildNameToPos(expectedSchema); - this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); - this.caseSensitive = conf.getBoolean(CASE_SENSITIVE, true); - this.inMemoryDataModel = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); + this.reuseContainers = conf.getBoolean(InputFormatConfig.REUSE_CONTAINERS, false); + this.caseSensitive = conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true); + this.inMemoryDataModel = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, + InputFormatConfig.InMemoryDataModel.GENERIC); this.currentIterator = open(tasks.next()); } @@ -361,7 +255,7 @@ private CloseableIterator open(FileScanTask currentTask) { DataFile file = currentTask.file(); // schema of rows returned by readers PartitionSpec spec = currentTask.spec(); - Set idColumns = Sets.intersection(spec.identitySourceIds(), TypeUtil.getProjectedIds(expectedSchema)); + Set idColumns = Sets.intersection(spec.identitySourceIds(), TypeUtil.getProjectedIds(expectedSchema)); boolean hasJoinedPartitionColumns = !idColumns.isEmpty(); CloseableIterable iterable; @@ -394,7 +288,7 @@ private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { break; default: throw new UnsupportedOperationException( - String.format("Cannot read %s file: %s", file.format().name(), file.path())); + String.format("Cannot read %s file: %s", file.format().name(), file.path())); } return iterable; @@ -402,12 +296,12 @@ private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { @SuppressWarnings("unchecked") private T withIdentityPartitionColumns( - T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { + T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { switch (inMemoryDataModel) { case PIG: case HIVE: throw new UnsupportedOperationException( - "Adding partition columns to Pig and Hive data model are not supported yet"); + "Adding partition columns to Pig and Hive data model are not supported yet"); case GENERIC: return (T) withIdentityPartitionColumns((Record) row, identityPartitionSchema, spec, partition); } @@ -415,7 +309,7 @@ private T withIdentityPartitionColumns( } private Record withIdentityPartitionColumns( - Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partitionTuple) { + Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partitionTuple) { List partitionFields = spec.fields(); List identityColumns = identityPartitionSchema.columns(); GenericRecord row = GenericRecord.create(expectedSchema.asStruct()); @@ -432,8 +326,8 @@ private Record withIdentityPartitionColumns( for (int j = 0; j < partitionFields.size(); j++) { PartitionField partitionField = partitionFields.get(j); if (name.equals(identityColumn.name()) && - identityColumn.fieldId() == partitionField.sourceId() && - "identity".equals(partitionField.transform().toString())) { + identityColumn.fieldId() == partitionField.sourceId() && + "identity".equals(partitionField.transform().toString())) { row.set(pos, partitionTuple.get(j, spec.javaClasses()[j])); } } @@ -445,7 +339,7 @@ private Record withIdentityPartitionColumns( private CloseableIterable applyResidualFiltering(CloseableIterable iter, Expression residual, Schema readSchema) { - boolean applyResidual = !context.getConfiguration().getBoolean(SKIP_RESIDUAL_FILTERING, false); + boolean applyResidual = !context.getConfiguration().getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); if (applyResidual && residual != null && residual != Expressions.alwaysTrue()) { Evaluator filter = new Evaluator(readSchema.asStruct(), residual, caseSensitive); @@ -457,8 +351,8 @@ private CloseableIterable applyResidualFiltering(CloseableIterable iter, E private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) - .project(readSchema) - .split(task.start(), task.length()); + .project(readSchema) + .split(task.start(), task.length()); if (reuseContainers) { avroReadBuilder.reuseContainers(); } @@ -476,10 +370,10 @@ private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask t private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) - .project(readSchema) - .filter(task.residual()) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); + .project(readSchema) + .filter(task.residual()) + .caseSensitive(caseSensitive) + .split(task.start(), task.length()); if (reuseContainers) { parquetReadBuilder.reuseContainers(); } @@ -498,10 +392,10 @@ private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTas private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) - .project(readSchema) - .filter(task.residual()) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); + .project(readSchema) + .filter(task.residual()) + .caseSensitive(caseSensitive) + .split(task.start(), task.length()); // ORC does not support reuse containers yet switch (inMemoryDataModel) { case PIG: @@ -517,20 +411,20 @@ private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask ta } private static Table findTable(Configuration conf) { - String path = conf.get(TABLE_PATH); + String path = conf.get(InputFormatConfig.TABLE_PATH); Preconditions.checkArgument(path != null, "Table path should not be null"); if (path.contains("/")) { HadoopTables tables = new HadoopTables(conf); return tables.load(path); } - String catalogFuncClass = conf.get(CATALOG); + String catalogFuncClass = conf.get(InputFormatConfig.CATALOG); if (catalogFuncClass != null) { Function catalogFunc = (Function) - DynConstructors.builder(Function.class) - .impl(catalogFuncClass) - .build() - .newInstance(); + DynConstructors.builder(Function.class) + .impl(catalogFuncClass) + .build() + .newInstance(); Catalog catalog = catalogFunc.apply(conf); TableIdentifier tableIdentifier = TableIdentifier.parse(path); return catalog.loadTable(tableIdentifier); @@ -557,7 +451,7 @@ public long getLength() { @Override public String[] getLocations() { - boolean localityPreferred = conf.getBoolean(LOCALITY, false); + boolean localityPreferred = conf.getBoolean(InputFormatConfig.LOCALITY, false); if (!localityPreferred) { return ANYWHERE; } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index a5dbbef0ac4e..05842f80e6bd 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -58,6 +58,7 @@ import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; @@ -128,8 +129,8 @@ public void testUnpartitionedTable() throws Exception { .appendFile(dataFile) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); validate(job, expectedRecords); } @@ -148,8 +149,8 @@ public void testPartitionedTable() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); validate(job, expectedRecords); } @@ -171,9 +172,10 @@ public void testFilterExp() throws Exception { .appendFile(dataFile2) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()) - .filter(Expressions.equal("date", "2020-03-20")); + .schema(table.schema()) + .filter(Expressions.equal("date", "2020-03-20")); validate(job, expectedRecords); } @@ -201,20 +203,22 @@ public void testResiduals() throws Exception { .appendFile(dataFile2) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); + .schema(table.schema()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); validate(job, expectedRecords); // skip residual filtering job = Job.getInstance(conf); configBuilder = IcebergInputFormat.configure(job); configBuilder.skipResidualFiltering().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); + .schema(table.schema()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); validate(job, writeRecords); } @@ -235,8 +239,9 @@ public void testFailedResidualFiltering() throws Exception { .commit(); Job jobShouldFail1 = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); configBuilder.useHiveRows().readFrom(location.toString()) + .schema(table.schema()) .filter(Expressions.and( Expressions.equal("date", "2020-03-20"), Expressions.equal("id", 0))); @@ -248,6 +253,7 @@ public void testFailedResidualFiltering() throws Exception { Job jobShouldFail2 = Job.getInstance(conf); configBuilder = IcebergInputFormat.configure(jobShouldFail2); configBuilder.usePigTuples().readFrom(location.toString()) + .schema(table.schema()) .filter(Expressions.and( Expressions.equal("date", "2020-03-20"), Expressions.equal("id", 0))); @@ -272,8 +278,9 @@ public void testProjection() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder + .schema(table.schema()) .readFrom(location.toString()) .project(projectedSchema); List outputRecords = readRecords(job.getConfiguration()); @@ -311,27 +318,43 @@ public void testIdentityPartitionProjections() throws Exception { append.commit(); // individual fields - validateIdentityPartitionProjections(location.toString(), withColumns("date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("id"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("id"), inputRecords); // field pairs - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "level"), inputRecords); // out-of-order pairs - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "date"), inputRecords); // full projection - validateIdentityPartitionProjections(location.toString(), LOG_SCHEMA, inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), LOG_SCHEMA, inputRecords); // out-of-order triplets - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "level", "date"), inputRecords); } private static Schema withColumns(String... names) { @@ -344,10 +367,11 @@ private static Schema withColumns(String... names) { } private void validateIdentityPartitionProjections( - String tablePath, Schema projectedSchema, List inputRecords) throws Exception { + String tablePath, Schema tableSchema, Schema projectedSchema, List inputRecords) throws Exception { Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder + .schema(tableSchema) .readFrom(tablePath) .project(projectedSchema); List actualRecords = readRecords(job.getConfiguration()); @@ -381,8 +405,9 @@ public void testSnapshotReads() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder + .schema(table.schema()) .readFrom(location.toString()) .snapshotId(snapshotId); @@ -401,8 +426,8 @@ public void testLocality() throws Exception { .appendFile(writeFile(table, null, format, expectedRecords)) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); for (InputSplit split : splits(job.getConfiguration())) { Assert.assertArrayEquals(IcebergInputFormat.IcebergSplit.ANYWHERE, split.getLocations()); @@ -438,9 +463,10 @@ public void testCustomCatalog() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder .catalogFunc(HadoopCatalogFunc.class) + .schema(table.schema()) .readFrom(tableIdentifier.toString()); validate(job, expectedRecords); } From abaafb501982052d6849923ccde51b4d7351c377 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Wed, 17 Jun 2020 14:09:32 +0100 Subject: [PATCH 08/14] Remove system tables --- .../iceberg/mr/mapred/IcebergSerDe.java | 21 +----- .../iceberg/mr/mapred/SystemTableUtil.java | 74 ------------------- .../iceberg/mr/mapred/TableResolver.java | 9 --- 3 files changed, 4 insertions(+), 100 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java index 0bddac485f9e..3544a8fbd61f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Writable; import org.apache.iceberg.Schema; -import org.apache.iceberg.SnapshotsTable; import org.apache.iceberg.Table; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -59,22 +58,10 @@ public void initialize(@Nullable Configuration configuration, Properties serDePr throw new UncheckedIOException("Unable to resolve table from configuration: ", e); } this.schema = table.schema(); - if (table instanceof SnapshotsTable) { - try { - this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(schema); - } catch (Exception e) { - throw new SerDeException(e); - } - } else { - List columns = new ArrayList<>(schema.columns()); - columns.add(Types.NestedField.optional(Integer.MAX_VALUE, - SystemTableUtil.snapshotIdVirtualColumnName(serDeProperties), Types.LongType.get())); - Schema withVirtualColumn = new Schema(columns); - try { - this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(withVirtualColumn); - } catch (Exception e) { - throw new SerDeException(e); - } + try { + this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(schema); + } catch (Exception e) { + throw new SerDeException(e); } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java deleted file mode 100644 index 105a4ea13a98..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import java.util.List; -import java.util.Properties; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.Schema; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.types.Types; - -public class SystemTableUtil { - - static final String VIRTUAL_COLUMN_NAME = "iceberg.hive.snapshot.virtual.column.name"; - - private static final String DEFAULT_SNAPSHOT_ID_COLUMN_NAME = "snapshot__id"; - - private SystemTableUtil() {} - - protected static Schema schemaWithSnapshotIdVirtualColumn(Schema schema, String columnName) { - List columns = Lists.newArrayList(schema.columns()); - columns.add(Types.NestedField.optional(Integer.MAX_VALUE, columnName, Types.LongType.get())); - return new Schema(columns); - } - - protected static Record recordWithSnapshotIdVirtualColumn(Record record, long snapshotId, Schema oldSchema, - String virtualColumnName) { - Schema newSchema = schemaWithSnapshotIdVirtualColumn(oldSchema, virtualColumnName); - Record newRecord = GenericRecord.create(newSchema); - for (int i = 0; i < oldSchema.columns().size(); i++) { - newRecord.set(i, record.get(i)); - } - newRecord.setField(virtualColumnName, snapshotId); - return newRecord; - } - - protected static String snapshotIdVirtualColumnName(Configuration conf) { - String virtualColumnName = conf.get(VIRTUAL_COLUMN_NAME); - if (virtualColumnName == null) { - return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; - } else { - return virtualColumnName; - } - } - - protected static String snapshotIdVirtualColumnName(Properties properties) { - String virtualColumnName = properties.getProperty(VIRTUAL_COLUMN_NAME); - if (virtualColumnName == null) { - return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; - } else { - return virtualColumnName; - } - } - -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java index d85fd4dc62e0..600f7f48e770 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -38,8 +38,6 @@ static Table resolveTableFromJob(JobConf conf) throws IOException { Properties properties = new Properties(); properties.setProperty(InputFormatConfig.CATALOG_NAME, conf.get(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES)); //Default to HadoopTables - properties.setProperty(InputFormatConfig.SNAPSHOT_TABLE, - conf.get(InputFormatConfig.SNAPSHOT_TABLE, "true")); properties.setProperty(InputFormatConfig.TABLE_LOCATION, extractProperty(conf, InputFormatConfig.TABLE_LOCATION)); properties.setProperty(InputFormatConfig.TABLE_NAME, extractProperty(conf, InputFormatConfig.TABLE_NAME)); return resolveTableFromConfiguration(conf, properties); @@ -54,13 +52,6 @@ static Table resolveTableFromConfiguration(Configuration conf, Properties proper switch (catalogName) { case InputFormatConfig.HADOOP_TABLES: HadoopTables tables = new HadoopTables(conf); - if (tableName.endsWith(InputFormatConfig.SNAPSHOT_TABLE_SUFFIX)) { - if (!Boolean.parseBoolean(properties.getProperty( - InputFormatConfig.SNAPSHOT_TABLE, Boolean.TRUE.toString()))) { - return tables.load(tableLocation); - } - return tables.load(tableLocation + "#snapshots"); - } return tables.load(tableLocation); case InputFormatConfig.HIVE_CATALOG: //TODO Implement HiveCatalog From c5b6cd8171aa1d73df7da3cf14ce256c21c57a61 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 17 Jun 2020 17:10:50 +0100 Subject: [PATCH 09/14] revert whitespace changes --- .../mr/mapreduce/IcebergInputFormat.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 3901a3543cbb..3ce4c4d056ec 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -136,11 +136,11 @@ public List getSplits(JobContext context) { splits = Lists.newArrayList(); boolean applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); InputFormatConfig.InMemoryDataModel model = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, - InputFormatConfig.InMemoryDataModel.GENERIC); + InputFormatConfig.InMemoryDataModel.GENERIC); try (CloseableIterable tasksIterable = scan.planTasks()) { tasksIterable.forEach(task -> { if (applyResidual && (model == InputFormatConfig.InMemoryDataModel.HIVE || - model == InputFormatConfig.InMemoryDataModel.PIG)) { + model == InputFormatConfig.InMemoryDataModel.PIG)) { //TODO: We do not support residual evaluation for HIVE and PIG in memory data model yet checkResiduals(task); } @@ -158,9 +158,9 @@ private static void checkResiduals(CombinedScanTask task) { Expression residual = fileScanTask.residual(); if (residual != null && !residual.equals(Expressions.alwaysTrue())) { throw new UnsupportedOperationException( - String.format( - "Filter expression %s is not completely satisfied. Additional rows " + - "can be returned not satisfied by the filter expression", residual)); + String.format( + "Filter expression %s is not completely satisfied. Additional rows " + + "can be returned not satisfied by the filter expression", residual)); } }); } @@ -288,7 +288,7 @@ private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { break; default: throw new UnsupportedOperationException( - String.format("Cannot read %s file: %s", file.format().name(), file.path())); + String.format("Cannot read %s file: %s", file.format().name(), file.path())); } return iterable; @@ -296,12 +296,12 @@ private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { @SuppressWarnings("unchecked") private T withIdentityPartitionColumns( - T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { + T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { switch (inMemoryDataModel) { case PIG: case HIVE: throw new UnsupportedOperationException( - "Adding partition columns to Pig and Hive data model are not supported yet"); + "Adding partition columns to Pig and Hive data model are not supported yet"); case GENERIC: return (T) withIdentityPartitionColumns((Record) row, identityPartitionSchema, spec, partition); } @@ -309,7 +309,7 @@ private T withIdentityPartitionColumns( } private Record withIdentityPartitionColumns( - Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partitionTuple) { + Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partitionTuple) { List partitionFields = spec.fields(); List identityColumns = identityPartitionSchema.columns(); GenericRecord row = GenericRecord.create(expectedSchema.asStruct()); @@ -326,8 +326,8 @@ private Record withIdentityPartitionColumns( for (int j = 0; j < partitionFields.size(); j++) { PartitionField partitionField = partitionFields.get(j); if (name.equals(identityColumn.name()) && - identityColumn.fieldId() == partitionField.sourceId() && - "identity".equals(partitionField.transform().toString())) { + identityColumn.fieldId() == partitionField.sourceId() && + "identity".equals(partitionField.transform().toString())) { row.set(pos, partitionTuple.get(j, spec.javaClasses()[j])); } } @@ -351,8 +351,8 @@ private CloseableIterable applyResidualFiltering(CloseableIterable iter, E private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) - .project(readSchema) - .split(task.start(), task.length()); + .project(readSchema) + .split(task.start(), task.length()); if (reuseContainers) { avroReadBuilder.reuseContainers(); } @@ -370,10 +370,10 @@ private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask t private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) - .project(readSchema) - .filter(task.residual()) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); + .project(readSchema) + .filter(task.residual()) + .caseSensitive(caseSensitive) + .split(task.start(), task.length()); if (reuseContainers) { parquetReadBuilder.reuseContainers(); } @@ -392,10 +392,10 @@ private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTas private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) - .project(readSchema) - .filter(task.residual()) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); + .project(readSchema) + .filter(task.residual()) + .caseSensitive(caseSensitive) + .split(task.start(), task.length()); // ORC does not support reuse containers yet switch (inMemoryDataModel) { case PIG: @@ -421,10 +421,10 @@ private static Table findTable(Configuration conf) { String catalogFuncClass = conf.get(InputFormatConfig.CATALOG); if (catalogFuncClass != null) { Function catalogFunc = (Function) - DynConstructors.builder(Function.class) - .impl(catalogFuncClass) - .build() - .newInstance(); + DynConstructors.builder(Function.class) + .impl(catalogFuncClass) + .build() + .newInstance(); Catalog catalog = catalogFunc.apply(conf); TableIdentifier tableIdentifier = TableIdentifier.parse(path); return catalog.loadTable(tableIdentifier); From f7c5c39387b13c27a44d17e4c5534578763c788a Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Fri, 26 Jun 2020 04:16:03 -0700 Subject: [PATCH 10/14] Refactor IcebergObjectInspector and implement custom object inspectors (#12) --- .../IcebergObjectInspectorGenerator.java | 86 -------- .../mr/mapred/IcebergSchemaToTypeInfo.java | 111 ---------- .../iceberg/mr/mapred/IcebergSerDe.java | 50 +---- .../iceberg/mr/mapred/IcebergWritable.java | 5 + .../iceberg/mr/mapred/TableResolver.java | 21 +- .../IcebergBinaryObjectInspector.java | 61 ++++++ .../IcebergDateObjectInspector.java | 56 +++++ .../IcebergDecimalObjectInspector.java | 83 +++++++ .../IcebergObjectInspector.java | 118 ++++++++++ .../IcebergPrimitiveObjectInspector.java | 78 +++++++ .../IcebergRecordObjectInspector.java | 170 +++++++++++++++ .../IcebergTimestampObjectInspector.java | 73 +++++++ .../TestIcebergObjectInspectorGenerator.java | 41 ---- .../mapred/TestIcebergSchemaToTypeInfo.java | 117 ---------- .../iceberg/mr/mapred/TestIcebergSerDe.java | 91 ++++---- .../TestIcebergBinaryObjectInspector.java | 64 ++++++ .../TestIcebergDateObjectInspector.java | 65 ++++++ .../TestIcebergDecimalObjectInspector.java | 77 +++++++ .../TestIcebergObjectInspector.java | 203 ++++++++++++++++++ .../TestIcebergRecordObjectInspector.java | 64 ++++++ .../TestIcebergTimestampObjectInspector.java | 65 ++++++ 21 files changed, 1244 insertions(+), 455 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java delete mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java delete mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java deleted file mode 100644 index f6838dbae9d2..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.Types; - -class IcebergObjectInspectorGenerator { - - protected ObjectInspector createObjectInspector(Schema schema) throws Exception { - List columnNames = setColumnNames(schema); - List columnTypes = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - List columnOIs = new ArrayList<>(columnTypes.size()); - for (int i = 0; i < columnTypes.size(); i++) { - columnOIs.add(createObjectInspectorWorker(columnTypes.get(i))); - } - return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs, null); - } - - protected ObjectInspector createObjectInspectorWorker(TypeInfo typeInfo) throws Exception { - ObjectInspector.Category typeCategory = typeInfo.getCategory(); - - switch (typeCategory) { - case PRIMITIVE: - PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo; - return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti); - case LIST: - ListTypeInfo ati = (ListTypeInfo) typeInfo; - return ObjectInspectorFactory - .getStandardListObjectInspector(createObjectInspectorWorker(ati.getListElementTypeInfo())); - case MAP: - MapTypeInfo mti = (MapTypeInfo) typeInfo; - return ObjectInspectorFactory.getStandardMapObjectInspector( - createObjectInspectorWorker(mti.getMapKeyTypeInfo()), - createObjectInspectorWorker(mti.getMapValueTypeInfo())); - case STRUCT: - StructTypeInfo sti = (StructTypeInfo) typeInfo; - List ois = new ArrayList<>(sti.getAllStructFieldTypeInfos().size()); - for (TypeInfo structTypeInfos : sti.getAllStructFieldTypeInfos()) { - ois.add(createObjectInspectorWorker(structTypeInfos)); - } - return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois); - default: - throw new SerDeException("Couldn't create Object Inspector for category: '" + typeCategory + "'"); - } - } - - protected List setColumnNames(Schema schema) { - List fields = schema.columns(); - List fieldsList = new ArrayList<>(fields.size()); - for (Types.NestedField field : fields) { - fieldsList.add(field.name()); - } - return fieldsList; - } - -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java deleted file mode 100644 index edc622f73a5f..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.iceberg.Schema; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -/** - * Class to convert Iceberg types to Hive TypeInfo - */ -final class IcebergSchemaToTypeInfo { - - private IcebergSchemaToTypeInfo() { - } - - private static final ImmutableMap primitiveTypeToTypeInfo = ImmutableMap.builder() - .put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME)) - .put(Types.IntegerType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME)) - .put(Types.LongType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) - .put(Types.FloatType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME)) - .put(Types.DoubleType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)) - .put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME)) - .put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)) - .put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME)) - .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)) - .put(Types.TimestampType.withZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)) - .build(); - - public static List getColumnTypes(Schema schema) throws Exception { - List fields = schema.columns(); - List types = new ArrayList<>(fields.size()); - for (Types.NestedField field : fields) { - types.add(generateTypeInfo(field.type())); - } - return types; - } - - private static TypeInfo generateTypeInfo(Type type) throws Exception { - if (primitiveTypeToTypeInfo.containsKey(type)) { - return (TypeInfo) primitiveTypeToTypeInfo.get(type); - } - switch (type.typeId()) { - case UUID: - return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - case FIXED: - return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME); - case TIME: - return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - case DECIMAL: - Types.DecimalType dec = (Types.DecimalType) type; - HiveDecimalUtils.validateParameter(dec.precision(), dec.scale()); - return TypeInfoFactory.getDecimalTypeInfo(dec.precision(), dec.scale()); - case STRUCT: - return generateStructTypeInfo((Types.StructType) type); - case LIST: - return generateListTypeInfo((Types.ListType) type); - case MAP: - return generateMapTypeInfo((Types.MapType) type); - default: - throw new SerDeException("Can't map Iceberg type to Hive TypeInfo: '" + type.typeId() + "'"); - } - } - - private static TypeInfo generateMapTypeInfo(Types.MapType type) throws Exception { - Type keyType = type.keyType(); - Type valueType = type.valueType(); - return TypeInfoFactory.getMapTypeInfo(generateTypeInfo(keyType), generateTypeInfo(valueType)); - } - - private static TypeInfo generateStructTypeInfo(Types.StructType type) throws Exception { - List fields = type.fields(); - List fieldNames = new ArrayList<>(fields.size()); - List typeInfos = new ArrayList<>(fields.size()); - - for (Types.NestedField field : fields) { - fieldNames.add(field.name()); - typeInfos.add(generateTypeInfo(field.type())); - } - return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); - } - - private static TypeInfo generateListTypeInfo(Types.ListType type) throws Exception { - return TypeInfoFactory.getListTypeInfo(generateTypeInfo(type.elementType())); - } -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java index 3544a8fbd61f..871e0e677835 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergSerDe.java @@ -21,15 +21,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.sql.Date; -import java.sql.Timestamp; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import java.util.Properties; import javax.annotation.Nullable; import org.apache.hadoop.conf.Configuration; @@ -38,28 +29,25 @@ import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Writable; -import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; +import org.apache.iceberg.mr.mapred.serde.objectinspector.IcebergObjectInspector; public class IcebergSerDe extends AbstractSerDe { - private Schema schema; private ObjectInspector inspector; - private List row; @Override public void initialize(@Nullable Configuration configuration, Properties serDeProperties) throws SerDeException { - Table table = null; + final Table table; + try { table = TableResolver.resolveTableFromConfiguration(configuration, serDeProperties); } catch (IOException e) { throw new UncheckedIOException("Unable to resolve table from configuration: ", e); } - this.schema = table.schema(); + try { - this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(schema); + this.inspector = IcebergObjectInspector.create(table.schema()); } catch (Exception e) { throw new SerDeException(e); } @@ -67,7 +55,7 @@ public void initialize(@Nullable Configuration configuration, Properties serDePr @Override public Class getSerializedClass() { - return null; + return IcebergWritable.class; } @Override @@ -82,31 +70,7 @@ public SerDeStats getSerDeStats() { @Override public Object deserialize(Writable writable) { - IcebergWritable icebergWritable = (IcebergWritable) writable; - List fields = icebergWritable.schema().columns(); - - if (row == null || row.size() != fields.size()) { - row = new ArrayList(fields.size()); - } else { - row.clear(); - } - for (int i = 0; i < fields.size(); i++) { - Object obj = ((IcebergWritable) writable).record().get(i); - Type fieldType = fields.get(i).type(); - if (fieldType.equals(Types.DateType.get())) { - row.add(Date.valueOf((LocalDate) obj)); - } else if (fieldType.equals(Types.TimestampType.withoutZone())) { - row.add(Timestamp.valueOf((LocalDateTime) obj)); - } else if (fieldType.equals(Types.TimestampType.withZone())) { - LocalDateTime timestamp = ((OffsetDateTime) obj).toLocalDateTime(); - row.add(Timestamp.valueOf(timestamp)); - } else if (fieldType.equals(Types.TimeType.get())) { - row.add(((LocalTime) obj).toString()); - } else { - row.add(obj); - } - } - return Collections.unmodifiableList(row); + return ((IcebergWritable) writable).record(); } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java index 3f3772e27e9e..1eb67f9d3158 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java @@ -33,6 +33,11 @@ public class IcebergWritable implements Writable { private Record record; private Schema schema; + public IcebergWritable(Record record, Schema schema) { + this.record = record; + this.schema = schema; + } + @SuppressWarnings("checkstyle:HiddenField") public void wrapRecord(Record record) { this.record = record; diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java index 600f7f48e770..8aa6857b7b0d 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -20,11 +20,11 @@ package org.apache.iceberg.mr.mapred; import java.io.IOException; +import java.util.Optional; import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.iceberg.Table; -import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -45,27 +45,26 @@ static Table resolveTableFromJob(JobConf conf) throws IOException { static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); - String tableLocation = properties.getProperty(InputFormatConfig.TABLE_LOCATION); - String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); - Preconditions.checkNotNull(tableLocation, "Table location is not set."); - Preconditions.checkNotNull(tableName, "Table name is not set."); + switch (catalogName) { case InputFormatConfig.HADOOP_TABLES: + String tableLocation = properties.getProperty(InputFormatConfig.TABLE_LOCATION); + Preconditions.checkNotNull(tableLocation, "Table location is not set."); HadoopTables tables = new HadoopTables(conf); return tables.load(tableLocation); + case InputFormatConfig.HIVE_CATALOG: + String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); + Preconditions.checkNotNull(tableName, "Table name is not set."); //TODO Implement HiveCatalog return null; default: - throw new NoSuchTableException("Table does not exist at location: " + tableLocation); + throw new RuntimeException("Catalog " + catalogName + " not supported."); } } protected static String extractProperty(JobConf conf, String key) { - String value = conf.get(key); - if (value == null) { - throw new IllegalArgumentException("Property not set in JobConf: " + key); - } - return value; + return Optional.ofNullable(conf.get(key)) + .orElseThrow(() -> new IllegalArgumentException("Property not set in JobConf: " + key)); } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java new file mode 100644 index 000000000000..85103c65307c --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.io.BytesWritable; + +public final class IcebergBinaryObjectInspector extends IcebergPrimitiveObjectInspector + implements BinaryObjectInspector { + + private static final IcebergBinaryObjectInspector INSTANCE = new IcebergBinaryObjectInspector(); + + public static IcebergBinaryObjectInspector get() { + return INSTANCE; + } + + private IcebergBinaryObjectInspector() { + super(TypeInfoFactory.binaryTypeInfo); + } + + @Override + public byte[] getPrimitiveJavaObject(Object o) { + return o == null ? null : ((ByteBuffer) o).array(); + } + + @Override + public BytesWritable getPrimitiveWritableObject(Object o) { + return o == null ? null : new BytesWritable(getPrimitiveJavaObject(o)); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + byte[] bytes = (byte[]) o; + return Arrays.copyOf(bytes, bytes.length); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java new file mode 100644 index 000000000000..2991540437c7 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Date; +import java.time.LocalDate; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public final class IcebergDateObjectInspector extends IcebergPrimitiveObjectInspector implements DateObjectInspector { + + private static final IcebergDateObjectInspector INSTANCE = new IcebergDateObjectInspector(); + + public static IcebergDateObjectInspector get() { + return INSTANCE; + } + + private IcebergDateObjectInspector() { + super(TypeInfoFactory.dateTypeInfo); + } + + @Override + public Date getPrimitiveJavaObject(Object o) { + return o == null ? null : Date.valueOf((LocalDate) o); + } + + @Override + public DateWritable getPrimitiveWritableObject(Object o) { + Date date = getPrimitiveJavaObject(o); + return date == null ? null : new DateWritable(date); + } + + @Override + public Object copyObject(Object o) { + return o == null ? null : new Date(((Date) o).getTime()); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java new file mode 100644 index 000000000000..5d31ce814509 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.math.BigDecimal; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +public final class IcebergDecimalObjectInspector extends IcebergPrimitiveObjectInspector + implements HiveDecimalObjectInspector { + + private static final Cache CACHE = Caffeine.newBuilder() + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(); + + public static IcebergDecimalObjectInspector get(int precision, int scale) { + Preconditions.checkArgument(scale < precision); + Preconditions.checkArgument(precision <= HiveDecimal.MAX_PRECISION); + Preconditions.checkArgument(scale <= HiveDecimal.MAX_SCALE); + + Integer key = precision << 8 | scale; + return CACHE.get(key, k -> new IcebergDecimalObjectInspector(precision, scale)); + } + + private IcebergDecimalObjectInspector(int precision, int scale) { + super(new DecimalTypeInfo(precision, scale)); + } + + @Override + public int precision() { + return ((DecimalTypeInfo) getTypeInfo()).precision(); + } + + @Override + public int scale() { + return ((DecimalTypeInfo) getTypeInfo()).scale(); + } + + @Override + public HiveDecimal getPrimitiveJavaObject(Object o) { + return o == null ? null : HiveDecimal.create((BigDecimal) o); + } + + @Override + public HiveDecimalWritable getPrimitiveWritableObject(Object o) { + HiveDecimal decimal = getPrimitiveJavaObject(o); + return decimal == null ? null : new HiveDecimalWritable(decimal); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + HiveDecimal decimal = (HiveDecimal) o; + return HiveDecimal.create(decimal.bigDecimalValue()); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java new file mode 100644 index 000000000000..ca4875649415 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.util.List; +import javax.annotation.Nullable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; + +public final class IcebergObjectInspector extends TypeUtil.SchemaVisitor { + + public static ObjectInspector create(@Nullable Schema schema) { + if (schema == null) { + return IcebergRecordObjectInspector.empty(); + } + + return TypeUtil.visit(schema, new IcebergObjectInspector()); + } + + public static ObjectInspector create(Types.NestedField... fields) { + return create(new Schema(fields)); + } + + @Override + public ObjectInspector field(Types.NestedField field, ObjectInspector fieldObjectInspector) { + return fieldObjectInspector; + } + + @Override + public ObjectInspector list(Types.ListType listTypeInfo, ObjectInspector listObjectInspector) { + return ObjectInspectorFactory.getStandardListObjectInspector(listObjectInspector); + } + + @Override + public ObjectInspector map(Types.MapType mapType, + ObjectInspector keyObjectInspector, ObjectInspector valueObjectInspector) { + return ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); + } + + @Override + public ObjectInspector primitive(Type.PrimitiveType primitiveType) { + final PrimitiveTypeInfo primitiveTypeInfo; + + switch (primitiveType.typeId()) { + case BINARY: + return IcebergBinaryObjectInspector.get(); + case BOOLEAN: + primitiveTypeInfo = TypeInfoFactory.booleanTypeInfo; + break; + case DATE: + return IcebergDateObjectInspector.get(); + case DECIMAL: + Types.DecimalType type = (Types.DecimalType) primitiveType; + return IcebergDecimalObjectInspector.get(type.precision(), type.scale()); + case DOUBLE: + primitiveTypeInfo = TypeInfoFactory.doubleTypeInfo; + break; + case FLOAT: + primitiveTypeInfo = TypeInfoFactory.floatTypeInfo; + break; + case INTEGER: + primitiveTypeInfo = TypeInfoFactory.intTypeInfo; + break; + case LONG: + primitiveTypeInfo = TypeInfoFactory.longTypeInfo; + break; + case STRING: + primitiveTypeInfo = TypeInfoFactory.stringTypeInfo; + break; + case TIMESTAMP: + boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC(); + return IcebergTimestampObjectInspector.get(adjustToUTC); + + case FIXED: + case TIME: + case UUID: + default: + throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported"); + } + + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveTypeInfo); + } + + @Override + public ObjectInspector schema(Schema schema, ObjectInspector structObjectInspector) { + return structObjectInspector; + } + + @Override + public ObjectInspector struct(Types.StructType structType, List fieldObjectInspectors) { + return new IcebergRecordObjectInspector(structType, fieldObjectInspectors); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java new file mode 100644 index 000000000000..53c3560c2dd7 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; + +abstract class IcebergPrimitiveObjectInspector implements PrimitiveObjectInspector { + + private final PrimitiveTypeInfo typeInfo; + + protected IcebergPrimitiveObjectInspector(PrimitiveTypeInfo typeInfo) { + this.typeInfo = typeInfo; + } + + @Override + public Category getCategory() { + return typeInfo.getCategory(); + } + + @Override + public String getTypeName() { + return typeInfo.getTypeName(); + } + + @Override + public PrimitiveTypeInfo getTypeInfo() { + return typeInfo; + } + + @Override + public PrimitiveObjectInspector.PrimitiveCategory getPrimitiveCategory() { + return typeInfo.getPrimitiveCategory(); + } + + @Override + public Class getJavaPrimitiveClass() { + return typeInfo.getPrimitiveJavaClass(); + } + + @Override + public Class getPrimitiveWritableClass() { + return typeInfo.getPrimitiveWritableClass(); + } + + @Override + public boolean preferWritable() { + return false; + } + + @Override + public int precision() { + return 0; + } + + @Override + public int scale() { + return 0; + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java new file mode 100644 index 000000000000..7005e4239708 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types; + +public final class IcebergRecordObjectInspector extends StructObjectInspector { + + private static final IcebergRecordObjectInspector EMPTY = + new IcebergRecordObjectInspector(Types.StructType.of(), Collections.emptyList()); + + private final List structFields; + + public IcebergRecordObjectInspector(Types.StructType structType, List objectInspectors) { + Preconditions.checkArgument(structType.fields().size() == objectInspectors.size()); + + this.structFields = Lists.newArrayListWithExpectedSize(structType.fields().size()); + + int position = 0; + + for (Types.NestedField field : structType.fields()) { + ObjectInspector oi = objectInspectors.get(position); + IcebergRecordStructField structField = new IcebergRecordStructField(field, oi, position); + structFields.add(structField); + position++; + } + } + + public static IcebergRecordObjectInspector empty() { + return EMPTY; + } + + @Override + public List getAllStructFieldRefs() { + return structFields; + } + + @Override + public StructField getStructFieldRef(String name) { + return ObjectInspectorUtils.getStandardStructFieldRef(name, structFields); + } + + @Override + public Object getStructFieldData(Object o, StructField structField) { + return ((Record) o).get(((IcebergRecordStructField) structField).position()); + } + + @Override + public List getStructFieldsDataAsList(Object o) { + Record record = (Record) o; + return structFields + .stream() + .map(f -> record.get(f.position())) + .collect(Collectors.toList()); + } + + @Override + public String getTypeName() { + return ObjectInspectorUtils.getStandardStructTypeName(this); + } + + @Override + public Category getCategory() { + return Category.STRUCT; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + IcebergRecordObjectInspector that = (IcebergRecordObjectInspector) o; + return structFields.equals(that.structFields); + } + + @Override + public int hashCode() { + return structFields.hashCode(); + } + + private static class IcebergRecordStructField implements StructField { + + private final Types.NestedField field; + private final ObjectInspector oi; + private final int position; + + IcebergRecordStructField(Types.NestedField field, ObjectInspector oi, int position) { + this.field = field; + this.oi = oi; + this.position = position; // position in the record + } + + @Override + public String getFieldName() { + return field.name(); + } + + @Override + public ObjectInspector getFieldObjectInspector() { + return oi; + } + + @Override + public int getFieldID() { + return field.fieldId(); + } + + @Override + public String getFieldComment() { + return field.doc(); + } + + int position() { + return position; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + IcebergRecordStructField that = (IcebergRecordStructField) o; + return field.equals(that.field) && oi.equals(that.oi); + } + + @Override + public int hashCode() { + return 31 * field.hashCode() + oi.hashCode(); + } + + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java new file mode 100644 index 000000000000..569267df8496 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.util.function.Function; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public final class IcebergTimestampObjectInspector extends IcebergPrimitiveObjectInspector + implements TimestampObjectInspector { + + private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = + new IcebergTimestampObjectInspector(o -> ((OffsetDateTime) o).toLocalDateTime()); + + private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = + new IcebergTimestampObjectInspector(o -> (LocalDateTime) o); + + public static IcebergTimestampObjectInspector get(boolean adjustToUTC) { + return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE; + } + + private final Function cast; + + private IcebergTimestampObjectInspector(Function cast) { + super(TypeInfoFactory.timestampTypeInfo); + this.cast = cast; + } + + @Override + public Timestamp getPrimitiveJavaObject(Object o) { + return o == null ? null : Timestamp.valueOf(cast.apply(o)); + } + + @Override + public TimestampWritable getPrimitiveWritableObject(Object o) { + Timestamp ts = getPrimitiveJavaObject(o); + return ts == null ? null : new TimestampWritable(ts); + } + + @Override + public Object copyObject(Object o) { + if (o == null) { + return null; + } + + Timestamp ts = (Timestamp) o; + Timestamp copy = new Timestamp(ts.getTime()); + copy.setNanos(ts.getNanos()); + return copy; + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java deleted file mode 100644 index ab05672746cc..000000000000 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergObjectInspectorGenerator.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import java.util.List; -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.Types; -import org.junit.Test; - -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.junit.Assert.assertEquals; - -public class TestIcebergObjectInspectorGenerator { - - @Test - public void testGetColumnNames() throws Exception { - Schema schema = new Schema(optional(1, "name", Types.StringType.get()), - optional(2, "salary", Types.LongType.get())); - IcebergObjectInspectorGenerator oi = new IcebergObjectInspectorGenerator(); - - List fieldsNames = oi.setColumnNames(schema); - assertEquals(fieldsNames.size(), 2); - } -} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java deleted file mode 100644 index 794d3f540b20..000000000000 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSchemaToTypeInfo.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import java.util.List; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.Types; -import org.junit.Test; - -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.junit.Assert.assertEquals; - -public class TestIcebergSchemaToTypeInfo { - - @Test - public void testGeneratePrimitiveTypeInfo() throws Exception { - Schema schema = new Schema( - required(1, "id", Types.IntegerType.get()), - optional(2, "data", Types.StringType.get()), - required(8, "feature1", Types.BooleanType.get()), - required(12, "lat", Types.FloatType.get()), - required(15, "x", Types.LongType.get()), - required(16, "date", Types.DateType.get()), - required(17, "double", Types.DoubleType.get()), - required(18, "binary", Types.BinaryType.get())); - List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - assertEquals("Converted TypeInfo should have the same number of columns.", 8, types.size()); - assertEquals("IntegerType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME), types.get(0)); - assertEquals("StringType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), types.get(1)); - assertEquals("BooleanType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME), types.get(2)); - assertEquals("FloatType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME), types.get(3)); - assertEquals("LongType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME), types.get(4)); - assertEquals("DateType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME), types.get(5)); - assertEquals("DoubleType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME), types.get(6)); - assertEquals("BinaryType converted incorrectly.", - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME), types.get(7)); - } - - @Test - public void testGenerateMapWithStringKeyTypeInfo() throws Exception { - TypeInfo expected = TypeInfoFactory.getMapTypeInfo( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); - - Schema schema = new Schema( - optional(7, "properties", Types.MapType.ofOptional(18, 19, - Types.StringType.get(), - Types.StringType.get() - ), "string map of properties")); - - List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - assertEquals("Converted TypeInfo should have the same number of columns.", 1, types.size()); - assertEquals("MapType converted incorrectly.", expected, types.get(0)); - } - - @Test - public void testGenerateMapWithIntKeyTypeInfo() throws Exception { - TypeInfo expected = TypeInfoFactory.getMapTypeInfo( - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME), - TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)); - - Schema schema = new Schema( - optional(7, "properties", Types.MapType.ofOptional(18, 19, - Types.IntegerType.get(), - Types.StringType.get() - ), "string map of properties")); - - List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - assertEquals("Converted TypeInfo should have the same number of columns.", 1, types.size()); - assertEquals("MapType converted incorrectly.", expected, types.get(0)); - } - - @Test - public void testGenerateListTypeInfo() throws Exception { - TypeInfo expected = TypeInfoFactory - .getListTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)); - Schema schema = new Schema( - required(6, "doubles", Types.ListType.ofRequired(17, - Types.DoubleType.get() - ))); - List types = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - assertEquals("Converted TypeInfo should have the same number of columns.", 1, types.size()); - assertEquals("ListType converted incorrectly.", expected, types.get(0)); - } -} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java index c81d1a263090..937ddba64da2 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergSerDe.java @@ -19,63 +19,62 @@ package org.apache.iceberg.mr.mapred; -import java.sql.Date; -import java.sql.Timestamp; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.List; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.mr.mapred.serde.objectinspector.IcebergObjectInspector; import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import static org.apache.iceberg.types.Types.NestedField.required; -import static org.junit.Assert.assertArrayEquals; public class TestIcebergSerDe { + private static final Schema schema = new Schema(required(1, "string_field", Types.StringType.get())); + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + @Test - public void testDeserializeWritable() { - Schema schema = new Schema(required(1, "string_type", Types.StringType.get()), - required(2, "int_type", Types.IntegerType.get()), - required(3, "long_type", Types.LongType.get()), - required(4, "boolean_type", Types.BooleanType.get()), - required(5, "float_type", Types.FloatType.get()), - required(6, "double_type", Types.DoubleType.get()), - required(7, "binary_type", Types.BinaryType.get()), - required(8, "date_type", Types.DateType.get()), - required(9, "timestamp_with_zone_type", Types.TimestampType.withZone()), - required(10, "timestamp_without_zone_type", Types.TimestampType.withoutZone()), - required(11, "map_type", Types.MapType - .ofRequired(12, 13, Types.IntegerType.get(), Types.StringType.get())), - required(14, "list_type", Types.ListType.ofRequired(15, Types.LongType.get())) - ); - LocalDate localDate = LocalDate.of(2018, 11, 10); - LocalDateTime localDateTime = LocalDateTime.of(2018, 11, 10, 11, 55); - OffsetDateTime offsetDateTime = OffsetDateTime.of(localDateTime, ZoneOffset.UTC); - - Object[] input = Lists.newArrayList("foo", 5, 6L, true, 1.02F, 1.4D, new byte[] { (byte) 0xe0}, - localDate, offsetDateTime, localDateTime, ImmutableMap.of(22, "bar"), - Arrays.asList(1000L, 2000L, 3000L)).toArray(); - - //Inputs and outputs differ slightly because of Date/Timestamp conversions for Hive - Object[] expected = Lists.newArrayList("foo", 5, 6L, true, 1.02F, 1.4D, new byte[] { (byte) 0xe0}, - Date.valueOf(localDate), Timestamp.valueOf(offsetDateTime.toLocalDateTime()), Timestamp.valueOf(localDateTime), - ImmutableMap.of(22, "bar"), Arrays.asList(1000L, 2000L, 3000L)).toArray(); - - Record record = TestHelpers.createCustomRecord(schema, input); - IcebergWritable writable = new IcebergWritable(); - writable.wrapRecord(record); - writable.wrapSchema(schema); + public void testInitialize() throws IOException, SerDeException { + File location = tmp.newFolder(); + Assert.assertTrue(location.delete()); + + Configuration conf = new Configuration(); + + Properties properties = new Properties(); + properties.setProperty(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + properties.setProperty(InputFormatConfig.TABLE_LOCATION, location.toString()); + + HadoopTables tables = new HadoopTables(conf); + tables.create(schema, PartitionSpec.unpartitioned(), Collections.emptyMap(), location.toString()); IcebergSerDe serDe = new IcebergSerDe(); - List deserialized = (List) serDe.deserialize(writable); - assertArrayEquals("Test values from an Iceberg Record deserialize into expected Java objects.", - expected, deserialized.toArray()); + serDe.initialize(conf, properties); + + Assert.assertEquals(IcebergObjectInspector.create(schema), serDe.getObjectInspector()); } + + @Test + public void testDeserialize() { + IcebergSerDe serDe = new IcebergSerDe(); + + Record record = RandomGenericData.generate(schema, 1, 0).get(0); + IcebergWritable writable = new IcebergWritable(record, schema); + + Assert.assertEquals(record, serDe.deserialize(writable)); + } + } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java new file mode 100644 index 000000000000..5d88da53cd6c --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.nio.ByteBuffer; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.io.BytesWritable; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergBinaryObjectInspector { + + @Test + public void testIcebergBinaryObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.get(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + byte[] bytes = new byte[] {0, 1}; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + + Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(buffer)); + Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(buffer)); + + byte[] copy = (byte[]) oi.copyObject(bytes); + + Assert.assertArrayEquals(bytes, copy); + Assert.assertNotSame(bytes, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java new file mode 100644 index 000000000000..28962aa352f4 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDateObjectInspector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Date; +import java.time.LocalDate; +import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergDateObjectInspector { + + @Test + public void testIcebergDateObjectInspector() { + DateObjectInspector oi = IcebergDateObjectInspector.get(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.DATE, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.dateTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.dateTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(Date.class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(DateWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + LocalDate local = LocalDate.of(2020, 1, 1); + Date date = Date.valueOf("2020-01-01"); + + Assert.assertEquals(date, oi.getPrimitiveJavaObject(local)); + Assert.assertEquals(new DateWritable(date), oi.getPrimitiveWritableObject(local)); + + Date copy = (Date) oi.copyObject(date); + + Assert.assertEquals(date, copy); + Assert.assertNotSame(date, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java new file mode 100644 index 000000000000..77489a4478e0 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergDecimalObjectInspector.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.math.BigDecimal; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergDecimalObjectInspector { + + @Test + public void testCache() { + HiveDecimalObjectInspector oi = IcebergDecimalObjectInspector.get(38, 18); + + Assert.assertSame(oi, IcebergDecimalObjectInspector.get(38, 18)); + Assert.assertNotSame(oi, IcebergDecimalObjectInspector.get(28, 18)); + Assert.assertNotSame(oi, IcebergDecimalObjectInspector.get(38, 28)); + } + + @Test + public void testIcebergDecimalObjectInspector() { + HiveDecimalObjectInspector oi = IcebergDecimalObjectInspector.get(38, 18); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.DECIMAL, oi.getPrimitiveCategory()); + + Assert.assertEquals(new DecimalTypeInfo(38, 18), oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.decimalTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(38, oi.precision()); + Assert.assertEquals(18, oi.scale()); + + Assert.assertEquals(HiveDecimal.class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(HiveDecimalWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + HiveDecimal one = HiveDecimal.create(BigDecimal.ONE); + + Assert.assertEquals(one, oi.getPrimitiveJavaObject(BigDecimal.ONE)); + Assert.assertEquals(new HiveDecimalWritable(one), oi.getPrimitiveWritableObject(BigDecimal.ONE)); + + HiveDecimal copy = (HiveDecimal) oi.copyObject(one); + + Assert.assertEquals(one, copy); + Assert.assertNotSame(one, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java new file mode 100644 index 000000000000..b280b05fb86d --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.AssertHelpers; +import org.apache.iceberg.Schema; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.types.Types.NestedField.required; + + +public class TestIcebergObjectInspector { + + private final Schema schema = new Schema( + required(0, "binary_field", Types.BinaryType.get(), "binary comment"), + required(1, "boolean_field", Types.BooleanType.get(), "boolean comment"), + required(2, "date_field", Types.DateType.get(), "date comment"), + required(3, "decimal_field", Types.DecimalType.of(38, 18), "decimal comment"), + required(4, "double_field", Types.DoubleType.get(), "double comment"), + required(5, "float_field", Types.FloatType.get(), "float comment"), + required(6, "integer_field", Types.IntegerType.get(), "integer comment"), + required(7, "long_field", Types.LongType.get(), "long comment"), + required(8, "string_field", Types.StringType.get(), "string comment"), + required(9, "timestamp_field", Types.TimestampType.withoutZone(), "timestamp comment"), + required(10, "timestamptz_field", Types.TimestampType.withZone(), "timestamptz comment"), + required(11, "list_field", + Types.ListType.ofRequired(12, Types.StringType.get()), "list comment"), + required(13, "map_field", + Types.MapType.ofRequired(14, 15, Types.StringType.get(), Types.IntegerType.get()), + "map comment"), + required(16, "struct_field", Types.StructType.of( + Types.NestedField.required(17, "nested_field", Types.StringType.get(), "nested field comment")), + "struct comment" + ) + ); + + @Test + public void testIcebergObjectInspector() { + ObjectInspector oi = IcebergObjectInspector.create(schema); + Assert.assertNotNull(oi); + Assert.assertEquals(ObjectInspector.Category.STRUCT, oi.getCategory()); + + StructObjectInspector soi = (StructObjectInspector) oi; + + // binary + StructField binaryField = soi.getStructFieldRef("binary_field"); + Assert.assertEquals(0, binaryField.getFieldID()); + Assert.assertEquals("binary_field", binaryField.getFieldName()); + Assert.assertEquals("binary comment", binaryField.getFieldComment()); + Assert.assertEquals(IcebergBinaryObjectInspector.get(), binaryField.getFieldObjectInspector()); + + // boolean + StructField booleanField = soi.getStructFieldRef("boolean_field"); + Assert.assertEquals(1, booleanField.getFieldID()); + Assert.assertEquals("boolean_field", booleanField.getFieldName()); + Assert.assertEquals("boolean comment", booleanField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(boolean.class), booleanField.getFieldObjectInspector()); + + // date + StructField dateField = soi.getStructFieldRef("date_field"); + Assert.assertEquals(2, dateField.getFieldID()); + Assert.assertEquals("date_field", dateField.getFieldName()); + Assert.assertEquals("date comment", dateField.getFieldComment()); + Assert.assertEquals(IcebergDateObjectInspector.get(), dateField.getFieldObjectInspector()); + + // decimal + StructField decimalField = soi.getStructFieldRef("decimal_field"); + Assert.assertEquals(3, decimalField.getFieldID()); + Assert.assertEquals("decimal_field", decimalField.getFieldName()); + Assert.assertEquals("decimal comment", decimalField.getFieldComment()); + Assert.assertEquals(IcebergDecimalObjectInspector.get(38, 18), decimalField.getFieldObjectInspector()); + + // double + StructField doubleField = soi.getStructFieldRef("double_field"); + Assert.assertEquals(4, doubleField.getFieldID()); + Assert.assertEquals("double_field", doubleField.getFieldName()); + Assert.assertEquals("double comment", doubleField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(double.class), doubleField.getFieldObjectInspector()); + + // float + StructField floatField = soi.getStructFieldRef("float_field"); + Assert.assertEquals(5, floatField.getFieldID()); + Assert.assertEquals("float_field", floatField.getFieldName()); + Assert.assertEquals("float comment", floatField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(float.class), floatField.getFieldObjectInspector()); + + // integer + StructField integerField = soi.getStructFieldRef("integer_field"); + Assert.assertEquals(6, integerField.getFieldID()); + Assert.assertEquals("integer_field", integerField.getFieldName()); + Assert.assertEquals("integer comment", integerField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(int.class), integerField.getFieldObjectInspector()); + + // long + StructField longField = soi.getStructFieldRef("long_field"); + Assert.assertEquals(7, longField.getFieldID()); + Assert.assertEquals("long_field", longField.getFieldName()); + Assert.assertEquals("long comment", longField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(long.class), longField.getFieldObjectInspector()); + + // string + StructField stringField = soi.getStructFieldRef("string_field"); + Assert.assertEquals(8, stringField.getFieldID()); + Assert.assertEquals("string_field", stringField.getFieldName()); + Assert.assertEquals("string comment", stringField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(String.class), stringField.getFieldObjectInspector()); + + // timestamp without tz + StructField timestampField = soi.getStructFieldRef("timestamp_field"); + Assert.assertEquals(9, timestampField.getFieldID()); + Assert.assertEquals("timestamp_field", timestampField.getFieldName()); + Assert.assertEquals("timestamp comment", timestampField.getFieldComment()); + Assert.assertEquals(IcebergTimestampObjectInspector.get(false), timestampField.getFieldObjectInspector()); + + // timestamp with tz + StructField timestampTzField = soi.getStructFieldRef("timestamptz_field"); + Assert.assertEquals(10, timestampTzField.getFieldID()); + Assert.assertEquals("timestamptz_field", timestampTzField.getFieldName()); + Assert.assertEquals("timestamptz comment", timestampTzField.getFieldComment()); + Assert.assertEquals(IcebergTimestampObjectInspector.get(true), timestampTzField.getFieldObjectInspector()); + + // list + StructField listField = soi.getStructFieldRef("list_field"); + Assert.assertEquals(11, listField.getFieldID()); + Assert.assertEquals("list_field", listField.getFieldName()); + Assert.assertEquals("list comment", listField.getFieldComment()); + Assert.assertEquals(getListObjectInspector(String.class), listField.getFieldObjectInspector()); + + // map + StructField mapField = soi.getStructFieldRef("map_field"); + Assert.assertEquals(13, mapField.getFieldID()); + Assert.assertEquals("map_field", mapField.getFieldName()); + Assert.assertEquals("map comment", mapField.getFieldComment()); + Assert.assertEquals(getMapObjectInspector(String.class, int.class), mapField.getFieldObjectInspector()); + + // struct + StructField structField = soi.getStructFieldRef("struct_field"); + Assert.assertEquals(16, structField.getFieldID()); + Assert.assertEquals("struct_field", structField.getFieldName()); + Assert.assertEquals("struct comment", structField.getFieldComment()); + + ObjectInspector expectedObjectInspector = new IcebergRecordObjectInspector( + (Types.StructType) schema.findType(16), ImmutableList.of(getPrimitiveObjectInspector(String.class))); + Assert.assertEquals(expectedObjectInspector, structField.getFieldObjectInspector()); + } + + @Test + public void testIcebergObjectInspectorUnsupportedTypes() { + AssertHelpers.assertThrows( + "Hive does not support fixed type", IllegalArgumentException.class, "FIXED type is not supported", + () -> IcebergObjectInspector.create(required(1, "fixed_field", Types.FixedType.ofLength(1)))); + + AssertHelpers.assertThrows( + "Hive does not support time type", IllegalArgumentException.class, "TIME type is not supported", + () -> IcebergObjectInspector.create(required(1, "time_field", Types.TimeType.get()))); + + AssertHelpers.assertThrows( + "Hive does not support UUID type", IllegalArgumentException.class, "UUID type is not supported", + () -> IcebergObjectInspector.create(required(1, "uuid_field", Types.UUIDType.get()))); + } + + private static ObjectInspector getPrimitiveObjectInspector(Class clazz) { + PrimitiveTypeInfo typeInfo = (PrimitiveTypeInfo) TypeInfoFactory.getPrimitiveTypeInfoFromJavaPrimitive(clazz); + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(typeInfo); + } + + private static ObjectInspector getListObjectInspector(Class clazz) { + return ObjectInspectorFactory.getStandardListObjectInspector(getPrimitiveObjectInspector(clazz)); + } + + private static ObjectInspector getMapObjectInspector(Class keyClazz, Class valueClazz) { + return ObjectInspectorFactory.getStandardMapObjectInspector( + getPrimitiveObjectInspector(keyClazz), getPrimitiveObjectInspector(valueClazz)); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java new file mode 100644 index 000000000000..edfaa1722185 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergRecordObjectInspector.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.RandomGenericData; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iceberg.types.Types.NestedField.required; + +public class TestIcebergRecordObjectInspector { + + @Test + public void testIcebergRecordObjectInspector() { + Schema schema = new Schema( + required(1, "integer_field", Types.IntegerType.get()), + required(2, "struct_field", Types.StructType.of( + Types.NestedField.required(3, "string_field", Types.StringType.get()))) + ); + + Record record = RandomGenericData.generate(schema, 1, 0L).get(0); + Record innerRecord = record.get(1, Record.class); + + StructObjectInspector soi = (StructObjectInspector) IcebergObjectInspector.create(schema); + Assert.assertEquals(ImmutableList.of(record.get(0), record.get(1)), soi.getStructFieldsDataAsList(record)); + + StructField integerField = soi.getStructFieldRef("integer_field"); + Assert.assertEquals(record.get(0), soi.getStructFieldData(record, integerField)); + + StructField structField = soi.getStructFieldRef("struct_field"); + Object innerData = soi.getStructFieldData(record, structField); + Assert.assertEquals(innerRecord, innerData); + + StructObjectInspector innerSoi = (StructObjectInspector) structField.getFieldObjectInspector(); + StructField stringField = innerSoi.getStructFieldRef("string_field"); + + Assert.assertEquals(ImmutableList.of(innerRecord.get(0)), innerSoi.getStructFieldsDataAsList(innerRecord)); + Assert.assertEquals(innerRecord.get(0), innerSoi.getStructFieldData(innerData, stringField)); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java new file mode 100644 index 000000000000..a1f6c18b8dd6 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergTimestampObjectInspector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.serde.objectinspector; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TestIcebergTimestampObjectInspector { + + @Test + public void testIcebergTimestampObjectInspector() { + TimestampObjectInspector oi = IcebergTimestampObjectInspector.get(false); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.timestampTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(Timestamp.class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(TimestampWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + LocalDateTime local = LocalDateTime.of(2020, 1, 1, 0, 0); + Timestamp ts = Timestamp.valueOf("2020-01-01 00:00:00"); + + Assert.assertEquals(ts, oi.getPrimitiveJavaObject(local)); + Assert.assertEquals(new TimestampWritable(ts), oi.getPrimitiveWritableObject(local)); + + Timestamp copy = (Timestamp) oi.copyObject(ts); + + Assert.assertEquals(ts, copy); + Assert.assertNotSame(ts, copy); + + Assert.assertFalse(oi.preferWritable()); + } + +} From 38e43396763cd7f3d2ba402579555119b5289daa Mon Sep 17 00:00:00 2001 From: awoodhead Date: Fri, 26 Jun 2020 13:52:24 +0100 Subject: [PATCH 11/14] fix compiler errors --- .../mr/mapreduce/IcebergInputFormat.java | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 9c7c5e97b4bf..e8f76ab63cd2 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -176,7 +176,6 @@ private static final class IcebergRecordReader extends RecordReader private boolean reuseContainers; private boolean caseSensitive; private InputFormatConfig.InMemoryDataModel inMemoryDataModel; - private Map namesToPos; private Iterator tasks; private T currentRow; private CloseableIterator currentIterator; @@ -191,12 +190,11 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { this.tableSchema = SchemaParser.fromJson(conf.get(InputFormatConfig.TABLE_SCHEMA)); String readSchemaStr = conf.get(InputFormatConfig.READ_SCHEMA); this.expectedSchema = readSchemaStr != null ? SchemaParser.fromJson(readSchemaStr) : tableSchema; - this.namesToPos = buildNameToPos(expectedSchema); this.reuseContainers = conf.getBoolean(InputFormatConfig.REUSE_CONTAINERS, false); this.caseSensitive = conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true); this.inMemoryDataModel = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, InputFormatConfig.InMemoryDataModel.GENERIC); - this.currentIterator = open(tasks.next()); + this.currentIterator = open(tasks.next(), expectedSchema).iterator(); } @Override @@ -241,35 +239,6 @@ public void close() throws IOException { currentIterator.close(); } - private static Map buildNameToPos(Schema expectedSchema) { - Map nameToPos = Maps.newHashMap(); - for (int pos = 0; pos < expectedSchema.asStruct().fields().size(); pos++) { - Types.NestedField field = expectedSchema.asStruct().fields().get(pos); - nameToPos.put(field.name(), pos); - } - return nameToPos; - } - - private CloseableIterator open(FileScanTask currentTask) { - DataFile file = currentTask.file(); - // schema of rows returned by readers - PartitionSpec spec = currentTask.spec(); - Set idColumns = Sets.intersection(spec.identitySourceIds(), TypeUtil.getProjectedIds(expectedSchema)); - boolean hasJoinedPartitionColumns = !idColumns.isEmpty(); - - CloseableIterable iterable; - if (hasJoinedPartitionColumns) { - Schema readDataSchema = TypeUtil.selectNot(expectedSchema, idColumns); - Schema identityPartitionSchema = TypeUtil.select(expectedSchema, idColumns); - iterable = CloseableIterable.transform(open(currentTask, readDataSchema), - row -> withIdentityPartitionColumns(row, identityPartitionSchema, spec, file.partition())); - } else { - iterable = open(currentTask, expectedSchema); - } - - return iterable.iterator(); - } - private CloseableIterable open(FileScanTask currentTask, Schema readSchema) { DataFile file = currentTask.file(); // TODO we should make use of FileIO to create inputFile From b3a0cfe243b58719ab5b3b26f0dbd108b3f35c6d Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 29 Jun 2020 15:10:05 +0100 Subject: [PATCH 12/14] tidied up and fleshed out tests --- .../iceberg/mr/mapred/TableResolver.java | 36 ++++---- .../iceberg/mr/mapred/TestTableResolver.java | 88 ++++++++++++++++--- 2 files changed, 92 insertions(+), 32 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java index 8aa6857b7b0d..58b0e367009c 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -20,11 +20,11 @@ package org.apache.iceberg.mr.mapred; import java.io.IOException; -import java.util.Optional; +import java.util.Map; import java.util.Properties; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.JobConf; import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -34,37 +34,33 @@ final class TableResolver { private TableResolver() { } - static Table resolveTableFromJob(JobConf conf) throws IOException { - Properties properties = new Properties(); - properties.setProperty(InputFormatConfig.CATALOG_NAME, - conf.get(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES)); //Default to HadoopTables - properties.setProperty(InputFormatConfig.TABLE_LOCATION, extractProperty(conf, InputFormatConfig.TABLE_LOCATION)); - properties.setProperty(InputFormatConfig.TABLE_NAME, extractProperty(conf, InputFormatConfig.TABLE_NAME)); - return resolveTableFromConfiguration(conf, properties); + static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { + Configuration configuration = new Configuration(conf); + for (Map.Entry entry : properties.entrySet()) { + configuration.set(entry.getKey().toString(), entry.getValue().toString()); + } + return resolveTableFromConfiguration(configuration); } - static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { - String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + static Table resolveTableFromConfiguration(Configuration conf) throws IOException { + //Default to HadoopTables + String catalogName = conf.get(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); switch (catalogName) { case InputFormatConfig.HADOOP_TABLES: - String tableLocation = properties.getProperty(InputFormatConfig.TABLE_LOCATION); - Preconditions.checkNotNull(tableLocation, "Table location is not set."); + String tableLocation = conf.get(InputFormatConfig.TABLE_LOCATION); + Preconditions.checkNotNull(tableLocation, InputFormatConfig.TABLE_LOCATION + " is not set."); HadoopTables tables = new HadoopTables(conf); return tables.load(tableLocation); case InputFormatConfig.HIVE_CATALOG: - String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); - Preconditions.checkNotNull(tableName, "Table name is not set."); + String tableName = conf.get(InputFormatConfig.TABLE_NAME); + Preconditions.checkNotNull(tableName, InputFormatConfig.TABLE_NAME + " is not set."); //TODO Implement HiveCatalog return null; default: - throw new RuntimeException("Catalog " + catalogName + " not supported."); + throw new NoSuchNamespaceException("Catalog " + catalogName + " not supported."); } } - protected static String extractProperty(JobConf conf, String key) { - return Optional.ofNullable(conf.get(key)) - .orElseThrow(() -> new IllegalArgumentException("Property not set in JobConf: " + key)); - } } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java index c77a811d596b..a0835dab15e1 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java @@ -19,32 +19,96 @@ package org.apache.iceberg.mr.mapred; +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.mapred.JobConf; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; -import static org.junit.Assert.assertEquals; +import static org.apache.iceberg.types.Types.NestedField.required; public class TestTableResolver { + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private Schema schema = new Schema(required(1, "string_field", Types.StringType.get())); + private File tableLocation; + + @Before + public void before() throws IOException, SerDeException { + tableLocation = tmp.newFolder(); + Configuration conf = new Configuration(); + HadoopTables tables = new HadoopTables(conf); + tables.create(schema, PartitionSpec.unpartitioned(), Collections.emptyMap(), tableLocation.toString()); + } + @Test - public void extractPropertyFromJobConf() { - JobConf conf = new JobConf(); - String key = "iceberg.catalog"; - String value = "hadoop.tables"; + public void resolveTableFromConfigurationDefault() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + + Table table = TableResolver.resolveTableFromConfiguration(conf); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } + + @Test + public void resolveTableFromConfigurationHadoopTables() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); - conf.set(key, value); + Table table = TableResolver.resolveTableFromConfiguration(conf); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } - String result = TableResolver.extractProperty(conf, key); + @Test(expected = NullPointerException.class) + public void resolveTableFromConfigurationHadoopTablesNoLocation() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); - assertEquals(value, result); + TableResolver.resolveTableFromConfiguration(conf); } - @Test(expected = IllegalArgumentException.class) - public void extractNonExistentProperty() { + @Test(expected = NoSuchNamespaceException.class) + public void resolveTableFromConfigurationInvalidName() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, "invalid-name"); + + TableResolver.resolveTableFromConfiguration(conf); + } + + @Test + public void resolveTableFromJobConfDefault() throws IOException { JobConf conf = new JobConf(); - String key = "iceberg.catalog"; + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + + Table table = TableResolver.resolveTableFromConfiguration(conf); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); + } + + @Test + public void resolveTableFromPropertiesDefault() throws IOException { + Configuration conf = new Configuration(); + Properties properties = new Properties(); + properties.setProperty(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); - TableResolver.extractProperty(conf, key); + Table table = TableResolver.resolveTableFromConfiguration(conf, properties); + Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); } } From f0d8a1ba24f664fc4c07dd43988b06bb5a886e27 Mon Sep 17 00:00:00 2001 From: Adrien Guillo Date: Wed, 1 Jul 2020 05:42:27 -0700 Subject: [PATCH 13/14] Fix binary object inspector and handle fixed and UUID types (#13) * Refactor TestIcebergObjectInspector * Inherit from AbstractPrimitiveJavaObjectInspector rather than IcebergPrimitiveObjectInspector * Avoid creating an intermediate Date object * Fix IcebergRecordStructField.equals * Use inheritance to implement static Timestamp object inspectors * Handle UUID type as String * Handle fixed type as binary (byte array) --- .../IcebergBinaryObjectInspector.java | 32 ++++-- .../IcebergDateObjectInspector.java | 8 +- .../IcebergDecimalObjectInspector.java | 13 +-- .../IcebergObjectInspector.java | 7 +- .../IcebergPrimitiveObjectInspector.java | 78 -------------- .../IcebergRecordObjectInspector.java | 5 +- .../IcebergTimestampObjectInspector.java | 32 +++--- .../TestIcebergBinaryObjectInspector.java | 44 +++++++- .../TestIcebergObjectInspector.java | 100 +++++++++++------- 9 files changed, 160 insertions(+), 159 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java index 85103c65307c..6ec48ebc6370 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergBinaryObjectInspector.java @@ -21,26 +21,46 @@ import java.nio.ByteBuffer; import java.util.Arrays; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.BytesWritable; +import org.apache.iceberg.util.ByteBuffers; -public final class IcebergBinaryObjectInspector extends IcebergPrimitiveObjectInspector - implements BinaryObjectInspector { +public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements BinaryObjectInspector { - private static final IcebergBinaryObjectInspector INSTANCE = new IcebergBinaryObjectInspector(); + private static final IcebergBinaryObjectInspector BYTE_ARRAY = new IcebergBinaryObjectInspector() { + @Override + byte[] toByteArray(Object o) { + return (byte[]) o; + } + }; + + private static final IcebergBinaryObjectInspector BYTE_BUFFER = new IcebergBinaryObjectInspector() { + @Override + byte[] toByteArray(Object o) { + return ByteBuffers.toByteArray((ByteBuffer) o); + } + }; - public static IcebergBinaryObjectInspector get() { - return INSTANCE; + public static IcebergBinaryObjectInspector byteArray() { + return BYTE_ARRAY; + } + + public static IcebergBinaryObjectInspector byteBuffer() { + return BYTE_BUFFER; } private IcebergBinaryObjectInspector() { super(TypeInfoFactory.binaryTypeInfo); } + abstract byte[] toByteArray(Object object); + @Override public byte[] getPrimitiveJavaObject(Object o) { - return o == null ? null : ((ByteBuffer) o).array(); + return toByteArray(o); } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java index 2991540437c7..2af9a172e963 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDateObjectInspector.java @@ -22,10 +22,13 @@ import java.sql.Date; import java.time.LocalDate; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.util.DateTimeUtil; -public final class IcebergDateObjectInspector extends IcebergPrimitiveObjectInspector implements DateObjectInspector { +public final class IcebergDateObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements DateObjectInspector { private static final IcebergDateObjectInspector INSTANCE = new IcebergDateObjectInspector(); @@ -44,8 +47,7 @@ public Date getPrimitiveJavaObject(Object o) { @Override public DateWritable getPrimitiveWritableObject(Object o) { - Date date = getPrimitiveJavaObject(o); - return date == null ? null : new DateWritable(date); + return o == null ? null : new DateWritable(DateTimeUtil.daysFromDate((LocalDate) o)); } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java index 5d31ce814509..fa8f91c2e99c 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergDecimalObjectInspector.java @@ -25,11 +25,12 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -public final class IcebergDecimalObjectInspector extends IcebergPrimitiveObjectInspector +public final class IcebergDecimalObjectInspector extends AbstractPrimitiveJavaObjectInspector implements HiveDecimalObjectInspector { private static final Cache CACHE = Caffeine.newBuilder() @@ -49,16 +50,6 @@ private IcebergDecimalObjectInspector(int precision, int scale) { super(new DecimalTypeInfo(precision, scale)); } - @Override - public int precision() { - return ((DecimalTypeInfo) getTypeInfo()).precision(); - } - - @Override - public int scale() { - return ((DecimalTypeInfo) getTypeInfo()).scale(); - } - @Override public HiveDecimal getPrimitiveJavaObject(Object o) { return o == null ? null : HiveDecimal.create((BigDecimal) o); diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java index ca4875649415..5e251f7092a9 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergObjectInspector.java @@ -67,7 +67,7 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { switch (primitiveType.typeId()) { case BINARY: - return IcebergBinaryObjectInspector.get(); + return IcebergBinaryObjectInspector.byteBuffer(); case BOOLEAN: primitiveTypeInfo = TypeInfoFactory.booleanTypeInfo; break; @@ -79,6 +79,8 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { case DOUBLE: primitiveTypeInfo = TypeInfoFactory.doubleTypeInfo; break; + case FIXED: + return IcebergBinaryObjectInspector.byteArray(); case FLOAT: primitiveTypeInfo = TypeInfoFactory.floatTypeInfo; break; @@ -89,15 +91,14 @@ public ObjectInspector primitive(Type.PrimitiveType primitiveType) { primitiveTypeInfo = TypeInfoFactory.longTypeInfo; break; case STRING: + case UUID: primitiveTypeInfo = TypeInfoFactory.stringTypeInfo; break; case TIMESTAMP: boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC(); return IcebergTimestampObjectInspector.get(adjustToUTC); - case FIXED: case TIME: - case UUID: default: throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported"); } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java deleted file mode 100644 index 53c3560c2dd7..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergPrimitiveObjectInspector.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred.serde.objectinspector; - -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; - -abstract class IcebergPrimitiveObjectInspector implements PrimitiveObjectInspector { - - private final PrimitiveTypeInfo typeInfo; - - protected IcebergPrimitiveObjectInspector(PrimitiveTypeInfo typeInfo) { - this.typeInfo = typeInfo; - } - - @Override - public Category getCategory() { - return typeInfo.getCategory(); - } - - @Override - public String getTypeName() { - return typeInfo.getTypeName(); - } - - @Override - public PrimitiveTypeInfo getTypeInfo() { - return typeInfo; - } - - @Override - public PrimitiveObjectInspector.PrimitiveCategory getPrimitiveCategory() { - return typeInfo.getPrimitiveCategory(); - } - - @Override - public Class getJavaPrimitiveClass() { - return typeInfo.getPrimitiveJavaClass(); - } - - @Override - public Class getPrimitiveWritableClass() { - return typeInfo.getPrimitiveWritableClass(); - } - - @Override - public boolean preferWritable() { - return false; - } - - @Override - public int precision() { - return 0; - } - - @Override - public int scale() { - return 0; - } - -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java index 7005e4239708..c49396b1f168 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergRecordObjectInspector.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -157,12 +158,12 @@ public boolean equals(Object o) { } IcebergRecordStructField that = (IcebergRecordStructField) o; - return field.equals(that.field) && oi.equals(that.oi); + return field.equals(that.field) && oi.equals(that.oi) && position == that.position; } @Override public int hashCode() { - return 31 * field.hashCode() + oi.hashCode(); + return Objects.hash(field, oi, position); } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java index 569267df8496..974e59f69194 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/serde/objectinspector/IcebergTimestampObjectInspector.java @@ -22,34 +22,42 @@ import java.sql.Timestamp; import java.time.LocalDateTime; import java.time.OffsetDateTime; -import java.util.function.Function; import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -public final class IcebergTimestampObjectInspector extends IcebergPrimitiveObjectInspector - implements TimestampObjectInspector { +public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector + implements TimestampObjectInspector { - private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = - new IcebergTimestampObjectInspector(o -> ((OffsetDateTime) o).toLocalDateTime()); + private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = new IcebergTimestampObjectInspector() { + @Override + LocalDateTime toLocalDateTime(Object o) { + return ((OffsetDateTime) o).toLocalDateTime(); + } + }; - private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = - new IcebergTimestampObjectInspector(o -> (LocalDateTime) o); + private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = new IcebergTimestampObjectInspector() { + @Override + LocalDateTime toLocalDateTime(Object o) { + return (LocalDateTime) o; + } + }; public static IcebergTimestampObjectInspector get(boolean adjustToUTC) { return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE; } - private final Function cast; - - private IcebergTimestampObjectInspector(Function cast) { + private IcebergTimestampObjectInspector() { super(TypeInfoFactory.timestampTypeInfo); - this.cast = cast; } + + abstract LocalDateTime toLocalDateTime(Object object); + @Override public Timestamp getPrimitiveJavaObject(Object o) { - return o == null ? null : Timestamp.valueOf(cast.apply(o)); + return o == null ? null : Timestamp.valueOf(toLocalDateTime(o)); } @Override diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java index 5d88da53cd6c..f28e78b4867b 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergBinaryObjectInspector.java @@ -31,8 +31,8 @@ public class TestIcebergBinaryObjectInspector { @Test - public void testIcebergBinaryObjectInspector() { - BinaryObjectInspector oi = IcebergBinaryObjectInspector.get(); + public void testIcebergByteArrayObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteArray(); Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); @@ -48,11 +48,49 @@ public void testIcebergBinaryObjectInspector() { Assert.assertNull(oi.getPrimitiveWritableObject(null)); byte[] bytes = new byte[] {0, 1}; - ByteBuffer buffer = ByteBuffer.wrap(bytes); + Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(bytes)); + Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(bytes)); + + byte[] copy = (byte[]) oi.copyObject(bytes); + + Assert.assertArrayEquals(bytes, copy); + Assert.assertNotSame(bytes, copy); + + Assert.assertFalse(oi.preferWritable()); + } + + @Test + public void testIcebergByteBufferObjectInspector() { + BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteBuffer(); + + Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory()); + Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory()); + + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo()); + Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName()); + + Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass()); + Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass()); + + Assert.assertNull(oi.copyObject(null)); + Assert.assertNull(oi.getPrimitiveJavaObject(null)); + Assert.assertNull(oi.getPrimitiveWritableObject(null)); + + byte[] bytes = new byte[] {0, 1, 2, 3}; + + ByteBuffer buffer = ByteBuffer.wrap(bytes); Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(buffer)); Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(buffer)); + ByteBuffer slice = ByteBuffer.wrap(bytes, 1, 2).slice(); + Assert.assertArrayEquals(new byte[] {1, 2}, oi.getPrimitiveJavaObject(slice)); + Assert.assertEquals(new BytesWritable(new byte[] {1, 2}), oi.getPrimitiveWritableObject(slice)); + + slice.position(1); + Assert.assertArrayEquals(new byte[] {2}, oi.getPrimitiveJavaObject(slice)); + Assert.assertEquals(new BytesWritable(new byte[] {2}), oi.getPrimitiveWritableObject(slice)); + byte[] copy = (byte[]) oi.copyObject(bytes); Assert.assertArrayEquals(bytes, copy); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java index b280b05fb86d..908c9dbfbef8 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/serde/objectinspector/TestIcebergObjectInspector.java @@ -38,31 +38,37 @@ public class TestIcebergObjectInspector { + private int id = 0; + private final Schema schema = new Schema( - required(0, "binary_field", Types.BinaryType.get(), "binary comment"), - required(1, "boolean_field", Types.BooleanType.get(), "boolean comment"), - required(2, "date_field", Types.DateType.get(), "date comment"), - required(3, "decimal_field", Types.DecimalType.of(38, 18), "decimal comment"), - required(4, "double_field", Types.DoubleType.get(), "double comment"), - required(5, "float_field", Types.FloatType.get(), "float comment"), - required(6, "integer_field", Types.IntegerType.get(), "integer comment"), - required(7, "long_field", Types.LongType.get(), "long comment"), - required(8, "string_field", Types.StringType.get(), "string comment"), - required(9, "timestamp_field", Types.TimestampType.withoutZone(), "timestamp comment"), - required(10, "timestamptz_field", Types.TimestampType.withZone(), "timestamptz comment"), - required(11, "list_field", - Types.ListType.ofRequired(12, Types.StringType.get()), "list comment"), - required(13, "map_field", - Types.MapType.ofRequired(14, 15, Types.StringType.get(), Types.IntegerType.get()), + required(id++, "binary_field", Types.BinaryType.get(), "binary comment"), + required(id++, "boolean_field", Types.BooleanType.get(), "boolean comment"), + required(id++, "date_field", Types.DateType.get(), "date comment"), + required(id++, "decimal_field", Types.DecimalType.of(38, 18), "decimal comment"), + required(id++, "double_field", Types.DoubleType.get(), "double comment"), + required(id++, "fixed_field", Types.FixedType.ofLength(3), "fixed comment"), + required(id++, "float_field", Types.FloatType.get(), "float comment"), + required(id++, "integer_field", Types.IntegerType.get(), "integer comment"), + required(id++, "long_field", Types.LongType.get(), "long comment"), + required(id++, "string_field", Types.StringType.get(), "string comment"), + required(id++, "timestamp_field", Types.TimestampType.withoutZone(), "timestamp comment"), + required(id++, "timestamptz_field", Types.TimestampType.withZone(), "timestamptz comment"), + required(id++, "uuid_field", Types.UUIDType.get(), "uuid comment"), + required(id++, "list_field", + Types.ListType.ofRequired(id++, Types.StringType.get()), "list comment"), + required(id++, "map_field", + Types.MapType.ofRequired(id++, id++, Types.StringType.get(), Types.IntegerType.get()), "map comment"), - required(16, "struct_field", Types.StructType.of( - Types.NestedField.required(17, "nested_field", Types.StringType.get(), "nested field comment")), + required(id++, "struct_field", Types.StructType.of( + Types.NestedField.required(id++, "nested_field", Types.StringType.get(), "nested field comment")), "struct comment" ) ); @Test public void testIcebergObjectInspector() { + int fieldId = 0; + ObjectInspector oi = IcebergObjectInspector.create(schema); Assert.assertNotNull(oi); Assert.assertEquals(ObjectInspector.Category.STRUCT, oi.getCategory()); @@ -71,119 +77,131 @@ public void testIcebergObjectInspector() { // binary StructField binaryField = soi.getStructFieldRef("binary_field"); - Assert.assertEquals(0, binaryField.getFieldID()); + Assert.assertEquals(fieldId++, binaryField.getFieldID()); Assert.assertEquals("binary_field", binaryField.getFieldName()); Assert.assertEquals("binary comment", binaryField.getFieldComment()); - Assert.assertEquals(IcebergBinaryObjectInspector.get(), binaryField.getFieldObjectInspector()); + Assert.assertEquals(IcebergBinaryObjectInspector.byteBuffer(), binaryField.getFieldObjectInspector()); // boolean StructField booleanField = soi.getStructFieldRef("boolean_field"); - Assert.assertEquals(1, booleanField.getFieldID()); + Assert.assertEquals(fieldId++, booleanField.getFieldID()); Assert.assertEquals("boolean_field", booleanField.getFieldName()); Assert.assertEquals("boolean comment", booleanField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(boolean.class), booleanField.getFieldObjectInspector()); // date StructField dateField = soi.getStructFieldRef("date_field"); - Assert.assertEquals(2, dateField.getFieldID()); + Assert.assertEquals(fieldId++, dateField.getFieldID()); Assert.assertEquals("date_field", dateField.getFieldName()); Assert.assertEquals("date comment", dateField.getFieldComment()); Assert.assertEquals(IcebergDateObjectInspector.get(), dateField.getFieldObjectInspector()); // decimal StructField decimalField = soi.getStructFieldRef("decimal_field"); - Assert.assertEquals(3, decimalField.getFieldID()); + Assert.assertEquals(fieldId++, decimalField.getFieldID()); Assert.assertEquals("decimal_field", decimalField.getFieldName()); Assert.assertEquals("decimal comment", decimalField.getFieldComment()); Assert.assertEquals(IcebergDecimalObjectInspector.get(38, 18), decimalField.getFieldObjectInspector()); // double StructField doubleField = soi.getStructFieldRef("double_field"); - Assert.assertEquals(4, doubleField.getFieldID()); + Assert.assertEquals(fieldId++, doubleField.getFieldID()); Assert.assertEquals("double_field", doubleField.getFieldName()); Assert.assertEquals("double comment", doubleField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(double.class), doubleField.getFieldObjectInspector()); + // fixed + StructField fixedField = soi.getStructFieldRef("fixed_field"); + Assert.assertEquals(fieldId++, fixedField.getFieldID()); + Assert.assertEquals("fixed_field", fixedField.getFieldName()); + Assert.assertEquals("fixed comment", fixedField.getFieldComment()); + Assert.assertEquals(IcebergBinaryObjectInspector.byteArray(), fixedField.getFieldObjectInspector()); + // float StructField floatField = soi.getStructFieldRef("float_field"); - Assert.assertEquals(5, floatField.getFieldID()); + Assert.assertEquals(fieldId++, floatField.getFieldID()); Assert.assertEquals("float_field", floatField.getFieldName()); Assert.assertEquals("float comment", floatField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(float.class), floatField.getFieldObjectInspector()); // integer StructField integerField = soi.getStructFieldRef("integer_field"); - Assert.assertEquals(6, integerField.getFieldID()); + Assert.assertEquals(fieldId++, integerField.getFieldID()); Assert.assertEquals("integer_field", integerField.getFieldName()); Assert.assertEquals("integer comment", integerField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(int.class), integerField.getFieldObjectInspector()); // long StructField longField = soi.getStructFieldRef("long_field"); - Assert.assertEquals(7, longField.getFieldID()); + Assert.assertEquals(fieldId++, longField.getFieldID()); Assert.assertEquals("long_field", longField.getFieldName()); Assert.assertEquals("long comment", longField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(long.class), longField.getFieldObjectInspector()); // string StructField stringField = soi.getStructFieldRef("string_field"); - Assert.assertEquals(8, stringField.getFieldID()); + Assert.assertEquals(fieldId++, stringField.getFieldID()); Assert.assertEquals("string_field", stringField.getFieldName()); Assert.assertEquals("string comment", stringField.getFieldComment()); Assert.assertEquals(getPrimitiveObjectInspector(String.class), stringField.getFieldObjectInspector()); // timestamp without tz StructField timestampField = soi.getStructFieldRef("timestamp_field"); - Assert.assertEquals(9, timestampField.getFieldID()); + Assert.assertEquals(fieldId++, timestampField.getFieldID()); Assert.assertEquals("timestamp_field", timestampField.getFieldName()); Assert.assertEquals("timestamp comment", timestampField.getFieldComment()); Assert.assertEquals(IcebergTimestampObjectInspector.get(false), timestampField.getFieldObjectInspector()); // timestamp with tz StructField timestampTzField = soi.getStructFieldRef("timestamptz_field"); - Assert.assertEquals(10, timestampTzField.getFieldID()); + Assert.assertEquals(fieldId++, timestampTzField.getFieldID()); Assert.assertEquals("timestamptz_field", timestampTzField.getFieldName()); Assert.assertEquals("timestamptz comment", timestampTzField.getFieldComment()); Assert.assertEquals(IcebergTimestampObjectInspector.get(true), timestampTzField.getFieldObjectInspector()); + // UUID + StructField uuidField = soi.getStructFieldRef("uuid_field"); + Assert.assertEquals(fieldId++, uuidField.getFieldID()); + Assert.assertEquals("uuid_field", uuidField.getFieldName()); + Assert.assertEquals("uuid comment", uuidField.getFieldComment()); + Assert.assertEquals(getPrimitiveObjectInspector(String.class), uuidField.getFieldObjectInspector()); + // list StructField listField = soi.getStructFieldRef("list_field"); - Assert.assertEquals(11, listField.getFieldID()); + Assert.assertEquals(fieldId++, listField.getFieldID()); Assert.assertEquals("list_field", listField.getFieldName()); Assert.assertEquals("list comment", listField.getFieldComment()); Assert.assertEquals(getListObjectInspector(String.class), listField.getFieldObjectInspector()); + // skip element id + fieldId++; + // map StructField mapField = soi.getStructFieldRef("map_field"); - Assert.assertEquals(13, mapField.getFieldID()); + Assert.assertEquals(fieldId++, mapField.getFieldID()); Assert.assertEquals("map_field", mapField.getFieldName()); Assert.assertEquals("map comment", mapField.getFieldComment()); Assert.assertEquals(getMapObjectInspector(String.class, int.class), mapField.getFieldObjectInspector()); + // skip key and value ids + fieldId += 2; + // struct StructField structField = soi.getStructFieldRef("struct_field"); - Assert.assertEquals(16, structField.getFieldID()); + Assert.assertEquals(fieldId, structField.getFieldID()); Assert.assertEquals("struct_field", structField.getFieldName()); Assert.assertEquals("struct comment", structField.getFieldComment()); ObjectInspector expectedObjectInspector = new IcebergRecordObjectInspector( - (Types.StructType) schema.findType(16), ImmutableList.of(getPrimitiveObjectInspector(String.class))); + (Types.StructType) schema.findType(fieldId), ImmutableList.of(getPrimitiveObjectInspector(String.class))); Assert.assertEquals(expectedObjectInspector, structField.getFieldObjectInspector()); } @Test public void testIcebergObjectInspectorUnsupportedTypes() { - AssertHelpers.assertThrows( - "Hive does not support fixed type", IllegalArgumentException.class, "FIXED type is not supported", - () -> IcebergObjectInspector.create(required(1, "fixed_field", Types.FixedType.ofLength(1)))); - AssertHelpers.assertThrows( "Hive does not support time type", IllegalArgumentException.class, "TIME type is not supported", () -> IcebergObjectInspector.create(required(1, "time_field", Types.TimeType.get()))); - - AssertHelpers.assertThrows( - "Hive does not support UUID type", IllegalArgumentException.class, "UUID type is not supported", - () -> IcebergObjectInspector.create(required(1, "uuid_field", Types.UUIDType.get()))); } private static ObjectInspector getPrimitiveObjectInspector(Class clazz) { From de445c4aa7416a21b978b7f448b9d7fde27c7ced Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 1 Jul 2020 14:19:09 +0100 Subject: [PATCH 14/14] throw UnsupportedOperationException instead of returning null --- .../apache/iceberg/mr/mapred/TableResolver.java | 4 ++-- .../iceberg/mr/mapred/TestTableResolver.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java index 58b0e367009c..7ee670d5e782 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -56,8 +56,8 @@ static Table resolveTableFromConfiguration(Configuration conf) throws IOExceptio case InputFormatConfig.HIVE_CATALOG: String tableName = conf.get(InputFormatConfig.TABLE_NAME); Preconditions.checkNotNull(tableName, InputFormatConfig.TABLE_NAME + " is not set."); - //TODO Implement HiveCatalog - return null; + throw new UnsupportedOperationException(InputFormatConfig.HIVE_CATALOG + " is not supported yet"); + default: throw new NoSuchNamespaceException("Catalog " + catalogName + " not supported."); } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java index a0835dab15e1..ff5b6455072a 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java @@ -111,4 +111,21 @@ public void resolveTableFromPropertiesDefault() throws IOException { Assert.assertEquals(tableLocation.getAbsolutePath(), table.location()); } + @Test(expected = UnsupportedOperationException.class) + public void resolveTableFromConfigurationHiveCatalog() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HIVE_CATALOG); + conf.set(InputFormatConfig.TABLE_NAME, "table_a"); + + TableResolver.resolveTableFromConfiguration(conf); + } + + @Test(expected = NullPointerException.class) + public void resolveTableFromConfigurationHiveCatalogMissingTableName() throws IOException { + Configuration conf = new Configuration(); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HIVE_CATALOG); + + TableResolver.resolveTableFromConfiguration(conf); + } + }