From f8dc2b09feac246310e6c8139bfa734330cf49a0 Mon Sep 17 00:00:00 2001 From: Ratandeep Ratt Date: Tue, 3 Mar 2020 06:28:34 -0800 Subject: [PATCH 01/51] InputFormat support for Iceberg --- build.gradle | 18 + .../apache/iceberg/mr/IcebergInputFormat.java | 345 ++++++++++++++++++ .../org/apache/iceberg/mr/ReadSupport.java | 50 +++ .../apache/iceberg/mr/SerializationUtil.java | 59 +++ .../iceberg/mr/TestIcebergInputFormat.java | 120 ++++++ settings.gradle | 2 + 6 files changed, 594 insertions(+) create mode 100644 mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java diff --git a/build.gradle b/build.gradle index c942b54e9baf..9f675fb9fab9 100644 --- a/build.gradle +++ b/build.gradle @@ -193,6 +193,24 @@ project(':iceberg-hive') { } } +project(':iceberg-mr') { + dependencies { + compile project(':iceberg-api') + compile project(':iceberg-core') + compile project(':iceberg-hive') + compile project(':iceberg-orc') + compile project(':iceberg-parquet') + + compileOnly("org.apache.hadoop:hadoop-client") { + exclude group: 'org.apache.avro', module: 'avro' + } + + testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') + testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') + testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') + } +} + project(':iceberg-orc') { dependencies { compile project(':iceberg-api') diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java new file mode 100644 index 000000000000..e6621616017a --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java @@ -0,0 +1,345 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import java.io.Closeable; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.iceberg.CombinedScanTask; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.common.DynClasses; +import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.hive.HiveCatalogs; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.types.TypeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class IcebergInputFormat extends InputFormat { + private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); + + static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; + static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; + static final String TABLE_PATH = "iceberg.mr.table.path"; + static final String READ_SCHEMA = "iceberg.mr.read.schema"; + static final String READ_SUPPORT = "iceberg.mr.read.support"; + + private transient Table table; + private transient List splits; + + public IcebergInputFormat() { + } + + @Override + public List getSplits(JobContext context) { + if (splits != null) { + LOG.info("Returning cached splits: {}", splits.size()); + return splits; + } + + Configuration conf = context.getConfiguration(); + table = getTable(conf); + TableScan scan = table.newScan(); + //TODO add caseSensitive, snapshot id etc.. + + Expression filterExpression = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); + if (filterExpression != null) { + scan = scan.filter(filterExpression); + } + + final String schemaStr = conf.get(READ_SCHEMA); + if (schemaStr != null) { + // Not sure if this is having any effect? + scan.project(SchemaParser.fromJson(schemaStr)); + } + + splits = Lists.newArrayList(); + try (CloseableIterable tasksIterable = scan.planTasks()) { + tasksIterable.forEach(task -> splits.add(new IcebergSplit(task))); + } catch (IOException e) { + throw new RuntimeIOException(e, "Failed to close table scan: %s", scan); + } + + return splits; + } + + @Override + public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { + return new IcebergRecordReader(); + } + + public static ConfBuilder updateConf( + Configuration conf, String path, Class> readSupportClass) { + return new ConfBuilder(conf, path, readSupportClass); + } + + public static class ConfBuilder { + private final Configuration conf; + + public ConfBuilder(Configuration conf, String path, Class> readSupportClass) { + this.conf = conf; + conf.set(TABLE_PATH, path); + conf.set(READ_SUPPORT, readSupportClass.getName()); + Table table = getTable(conf); + conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + } + + public ConfBuilder filterExpression(Expression expression) throws IOException { + conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); + return this; + } + + public ConfBuilder project(Schema schema) { + conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + //TODO: other options split-size, snapshotid etc.. + public Configuration updatedConf() { + return conf; + } + } + + private static final class IcebergRecordReader extends RecordReader { + private TaskAttemptContext context; + private Iterator tasks; + private Iterator currentIterator; + private T currentRow; + private Schema expectedSchema; + private Schema tableSchema; + private ReadSupport readSupport; + private Closeable currentCloseable; + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) { + Configuration conf = context.getConfiguration(); + CombinedScanTask task = ((IcebergSplit) split).task; + this.context = context; + this.tasks = task.files().iterator(); + this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); + String readSchemaStr = conf.get(READ_SCHEMA); + if (readSchemaStr != null) { + this.expectedSchema = SchemaParser.fromJson(readSchemaStr); + } + this.readSupport = readSupport(conf); + this.currentIterator = open(tasks.next()); + } + + @Override + public boolean nextKeyValue() throws IOException { + while (true) { + if (currentIterator.hasNext()) { + currentRow = currentIterator.next(); + return true; + } else if (tasks.hasNext()) { + currentCloseable.close(); + currentIterator = open(tasks.next()); + } else { + return false; + } + } + } + + @Override + public Void getCurrentKey() { + return null; + } + + @Override + public T getCurrentValue() { + return currentRow; + } + + @Override + public float getProgress() { + return context.getProgress(); + } + + @Override + public void close() throws IOException { + currentCloseable.close(); + } + + private ReadSupport readSupport(Configuration conf) { + String readSupportClassName = conf.get(READ_SUPPORT); + try { + return DynClasses + .builder() + .impl(readSupportClassName) + .>buildChecked() + .newInstance(); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + throw new RuntimeException(String.format("Unable to instantiate read support %s", readSupportClassName), e); + } + } + + private Iterator open(FileScanTask currentTask) { + DataFile file = currentTask.file(); + // schema of rows returned by readers + PartitionSpec spec = currentTask.spec(); + Set idColumns = spec.identitySourceIds(); + + boolean hasJoinedPartitionColumns = !idColumns.isEmpty(); + Schema readSchema = expectedSchema != null ? expectedSchema : tableSchema; + if (hasJoinedPartitionColumns) { + readSchema = TypeUtil.selectNot(tableSchema, idColumns); + Schema partitionSchema = TypeUtil.select(tableSchema, idColumns); + return Iterators.transform( + open(currentTask, readSchema), + row -> readSupport.withPartitionColumns(row, partitionSchema, spec, file.partition())); + } else { + return open(currentTask, readSchema); + } + } + + private Iterator open(FileScanTask currentTask, Schema readSchema) { + DataFile file = currentTask.file(); + // TODO should we somehow make use of FileIO to create inputFile? + InputFile inputFile = HadoopInputFile.fromLocation(file.path(), context.getConfiguration()); + CloseableIterable iterable; + switch (file.format()) { + case AVRO: + iterable = newAvroIterable(inputFile, currentTask, readSchema); + break; + case ORC: + iterable = newOrcIterable(inputFile, currentTask, readSchema); + break; + case PARQUET: + iterable = newParquetIterable(inputFile, currentTask, readSchema); + break; + default: + throw new UnsupportedOperationException( + String.format("Cannot read %s file: %s", file.format().name(), file.path())); + } + currentCloseable = iterable; + return iterable.iterator(); + } + + private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) + .createReaderFunc(readSupport.avroReadBiFunction()) + .createReaderFunc(readSupport.avroReadFunction()) + .project(readSchema) + .split(task.start(), task.length()); + //.reuseContainers(reuseContainers); + return avroReadBuilder.build(); + } + + + private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) + .createBatchedReaderFunc(readSupport.parquetBatchReadFunction()) + .createReaderFunc(readSupport.parquetReadFunction()) + .createBatchedReaderFunc(readSupport.parquetBatchReadFunction()) + .project(readSchema) + //.caseSensitive(caseSensitive) + .split(task.start(), task.length()); + return parquetReadBuilder.build(); + } + + private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) + .createReaderFunc(readSupport.orcReadFunction()) + .schema(readSchema) + //.caseSensitive(caseSensitive) + .split(task.start(), task.length()); + + return orcReadBuilder.build(); + } + } + + private static Table getTable(Configuration conf) { + String path = conf.get(TABLE_PATH); + if (path.contains("/")) { + HadoopTables tables = new HadoopTables(conf); + return tables.load(path); + } else { + Catalog catalog = HiveCatalogs.loadCatalog(conf); + TableIdentifier tableIdentifier = TableIdentifier.parse(path); + return catalog.loadTable(tableIdentifier); + } + } + + private static class IcebergSplit extends InputSplit implements Writable { + private static final String[] ANYWHERE = new String[]{"*"}; + CombinedScanTask task; + + IcebergSplit(CombinedScanTask task) { + this.task = task; + } + + public IcebergSplit() { + } + + @Override + public long getLength() { + return task.files().stream().mapToLong(FileScanTask::length).sum(); + } + + @Override + public String[] getLocations() { + //TODO: add locations for hdfs + return ANYWHERE; + } + + @Override + public void write(DataOutput out) throws IOException { + byte[] data = SerializationUtil.serializeToBytes(this.task); + out.writeInt(data.length); + out.write(data); + } + + @Override + public void readFields(DataInput in) throws IOException { + byte[] data = new byte[in.readInt()]; + in.readFully(data); + this.task = SerializationUtil.deserializeFromBytes(data); + } + } +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java b/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java new file mode 100644 index 000000000000..078c0f650341 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java @@ -0,0 +1,50 @@ +package org.apache.iceberg.mr; + +import java.util.function.BiFunction; +import org.apache.avro.io.DatumReader; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.orc.OrcValueReader; +import org.apache.iceberg.parquet.ParquetValueReader; +import org.apache.iceberg.parquet.VectorizedReader; +import org.apache.orc.TypeDescription; +import org.apache.parquet.schema.MessageType; + +import java.util.function.Function; + + +/** + * ReadSupport for MR InputFormat, providing value readers + * for different data formats and appending identity partition columns + * to the input row + * @param + */ +public interface ReadSupport { + /** + * Add identity partition columns to input row + */ + default T withPartitionColumns(T row, Schema partitionSchema, PartitionSpec spec, StructLike partitionData) { + return row; + } + + default Function> parquetReadFunction() { + return null; + } + + default Function> parquetBatchReadFunction() { + return null; + } + + default Function> avroReadFunction() { + return null; + } + + default BiFunction> avroReadBiFunction() { + return null; + } + + default Function> orcReadFunction() { + return null; + } +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java new file mode 100644 index 000000000000..3ec2a038cdac --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java @@ -0,0 +1,59 @@ +package org.apache.iceberg.mr; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import org.apache.iceberg.exceptions.RuntimeIOException; + + +public class SerializationUtil { + + private SerializationUtil() { + } + + public static byte[] serializeToBytes(Object obj) throws IOException { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + GZIPOutputStream gos = new GZIPOutputStream(baos); + ObjectOutputStream oos = new ObjectOutputStream(gos)) { + oos.writeObject(obj); + return baos.toByteArray(); + } + } + + @SuppressWarnings("unchecked") + public static T deserializeFromBytes(byte[] bytes) { + if (bytes == null) { + return null; + } + + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + GZIPInputStream gis = new GZIPInputStream(bais); + ObjectInputStream ois = new ObjectInputStream(gis)) { + return (T) ois.readObject(); + } catch (IOException e) { + throw new RuntimeIOException(e); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Could not read object ", e); + } + } + + public static String serializeToBase64(Object obj) throws IOException { + byte[] bytes = serializeToBytes(obj); + return new String(Base64.getMimeEncoder().encode(bytes), StandardCharsets.UTF_8); + } + + @SuppressWarnings("unchecked") + public static T deserializeFromBase64(String base64) { + if (base64 == null) { + return null; + } + byte[] bytes = Base64.getMimeDecoder().decode(base64.getBytes(StandardCharsets.UTF_8)); + return deserializeFromBytes(bytes); + } +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java new file mode 100644 index 000000000000..58190cccf335 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java @@ -0,0 +1,120 @@ +package org.apache.iceberg.mr; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.avro.generic.GenericData; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.TaskAttemptContextImpl; +import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.Files; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.avro.RandomAvroData; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; + + +public class TestIcebergInputFormat { + private static final Configuration CONF = new Configuration(); + private static final HadoopTables TABLES = new HadoopTables(CONF); + + private Table table; + private File tableLocation; + + private static final Schema SCHEMA = new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "data", Types.StringType.get()), + required(3, "date", Types.StringType.get())); + + private static final PartitionSpec PARTITION_BY_DATE = PartitionSpec + .builderFor(SCHEMA) + .identity("date") + .build(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + IcebergInputFormat icebergInputFormat; + + @Test + public void test() throws IOException, InterruptedException { + tableLocation = new File(temp.newFolder(), "table"); + Table table = TABLES.create(SCHEMA, PARTITION_BY_DATE, tableLocation.toString()); + List records = RandomAvroData.generate(SCHEMA, 5, 0L); + File file = temp.newFile(); + Assert.assertTrue(file.delete()); + try (FileAppender appender = Avro.write(Files.localOutput(file)) + .schema(SCHEMA) + .named("avro") + .build()) { + appender.addAll(records); + } + + DataFile dataFile = DataFiles.builder(PARTITION_BY_DATE) + .withPartition(partitionData("2020-03-15")) + .withRecordCount(records.size()) + .withFileSizeInBytes(file.length()) + .withPath(file.toString()) + .withFormat("avro") + .build(); + + table.newAppend().appendFile(dataFile).commit(); + + Configuration conf = new Configuration(); + conf = IcebergInputFormat + .updateConf(conf, tableLocation.getAbsolutePath(), TestReadSupport.class) + .updatedConf(); + TaskAttemptContext context = new TaskAttemptContextImpl(new JobConf(conf), new TaskAttemptID()); + icebergInputFormat = new IcebergInputFormat<>(); + List splits = icebergInputFormat.getSplits(context); + final RecordReader recordReader = icebergInputFormat.createRecordReader(splits.get(0), context); + recordReader.initialize(splits.get(0), context); + while (recordReader.nextKeyValue()) { + System.out.println(recordReader.getCurrentValue()); + } + } + + private StructLike partitionData(String date) { + return new StructLike() { + + @Override + public int size() { + return 1; + } + + @Override + public T get(int pos, Class javaClass) { + return (T) date; + } + + @Override + public void set(int pos, T value) { + } + }; + } + + public static class TestReadSupport implements ReadSupport { + @Override + public GenericData.Record withPartitionColumns( + GenericData.Record row, Schema partitionSchema, PartitionSpec spec, StructLike partitionData) { + return row; + } + } +} diff --git a/settings.gradle b/settings.gradle index 854b3b2d35c8..0c9e59228094 100644 --- a/settings.gradle +++ b/settings.gradle @@ -22,6 +22,7 @@ include 'api' include 'common' include 'core' include 'data' +include 'mr' include 'orc' include 'parquet' include 'spark' @@ -34,6 +35,7 @@ project(':api').name = 'iceberg-api' project(':common').name = 'iceberg-common' project(':core').name = 'iceberg-core' project(':data').name = 'iceberg-data' +project(':mr').name = 'iceberg-mr' project(':orc').name = 'iceberg-orc' project(':arrow').name = 'iceberg-arrow' project(':parquet').name = 'iceberg-parquet' From 5f5ffbd30f0ae7881225d886a27bdcb87c69078c Mon Sep 17 00:00:00 2001 From: Ratandeep Ratt Date: Tue, 17 Mar 2020 23:32:37 -0700 Subject: [PATCH 02/51] Address review comments --- build.gradle | 1 + .../apache/iceberg/mr/IcebergInputFormat.java | 282 +++++++++++++----- .../org/apache/iceberg/mr/ReadSupport.java | 50 ---- .../apache/iceberg/mr/SerializationUtil.java | 28 +- .../iceberg/mr/TestIcebergInputFormat.java | 47 +-- 5 files changed, 255 insertions(+), 153 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java diff --git a/build.gradle b/build.gradle index 9f675fb9fab9..abbcc2d06b44 100644 --- a/build.gradle +++ b/build.gradle @@ -200,6 +200,7 @@ project(':iceberg-mr') { compile project(':iceberg-hive') compile project(':iceberg-orc') compile project(':iceberg-parquet') + compile project(':iceberg-data') compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java index e6621616017a..68c1803951d9 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java @@ -32,21 +32,28 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.avro.Avro; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.common.DynClasses; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataReader; +import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.hadoop.HadoopInputFile; @@ -57,6 +64,7 @@ import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,16 +72,89 @@ public class IcebergInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); + static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; + static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; - static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - static final String TABLE_PATH = "iceberg.mr.table.path"; + static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; static final String READ_SCHEMA = "iceberg.mr.read.schema"; - static final String READ_SUPPORT = "iceberg.mr.read.support"; + static final String REUSE_CONTAINERS = "iceberg.mr.case.sensitive"; + static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; + static final String SPLIT_SIZE = "iceberg.mr.split.size"; + static final String TABLE_PATH = "iceberg.mr.table.path"; + static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - private transient Table table; private transient List splits; - public IcebergInputFormat() { + public enum InMemoryDataModel { + PIG, + HIVE, + DEFAULT + } + + /** + * Configures the {@code Job} to use the {@code IcebergInputFormat} and + * returns a helper to add further configuration. + * + * @param job the {@code Job} to configure + */ + public static ConfigBuilder configure(Job job) { + job.setInputFormatClass(IcebergInputFormat.class); + return new ConfigBuilder(job.getConfiguration()); + } + + public static class ConfigBuilder { + private final Configuration conf; + + public ConfigBuilder(Configuration conf) { + this.conf = conf; + } + + public ConfigBuilder readFrom(String path) { + conf.set(TABLE_PATH, path); + Table table = getTable(conf); + conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + return this; + } + + public ConfigBuilder filter(Expression expression) { + conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); + return this; + } + + public ConfigBuilder project(Schema schema) { + conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder reuseContainers(boolean reuse) { + conf.setBoolean(REUSE_CONTAINERS, reuse); + return this; + } + + public ConfigBuilder caseSensitive(boolean caseSensitive) { + conf.setBoolean(CASE_SENSITIVE, caseSensitive); + return this; + } + + public ConfigBuilder snapshotId(long snapshotId) { + conf.setLong(SNAPSHOT_ID, snapshotId); + return this; + } + + public ConfigBuilder asOfTime(long asOfTime) { + conf.setLong(AS_OF_TIMESTAMP, asOfTime); + return this; + } + + public ConfigBuilder splitSize(long splitSize) { + conf.setLong(SPLIT_SIZE, splitSize); + return this; + } + + public ConfigBuilder inMemoryDataModel(InMemoryDataModel inMemoryDataModel) { + conf.set(IN_MEMORY_DATA_MODEL, inMemoryDataModel.name()); + return this; + } } @Override @@ -84,21 +165,33 @@ public List getSplits(JobContext context) { } Configuration conf = context.getConfiguration(); - table = getTable(conf); - TableScan scan = table.newScan(); - //TODO add caseSensitive, snapshot id etc.. + Table table = getTable(conf); + TableScan scan = table.newScan() + .caseSensitive(conf.getBoolean(CASE_SENSITIVE, true)); - Expression filterExpression = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); - if (filterExpression != null) { - scan = scan.filter(filterExpression); + long snapshotId = conf.getLong(SNAPSHOT_ID, -1); + if (snapshotId != -1) { + scan = scan.useSnapshot(snapshotId); } - - final String schemaStr = conf.get(READ_SCHEMA); + long asOfTime = conf.getLong(AS_OF_TIMESTAMP, -1); + if (asOfTime != -1) { + scan = scan.asOfTime(asOfTime); + } + long splitSize = conf.getLong(SPLIT_SIZE, -1); + if (splitSize != -1) { + scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); + } + String schemaStr = conf.get(READ_SCHEMA); if (schemaStr != null) { - // Not sure if this is having any effect? scan.project(SchemaParser.fromJson(schemaStr)); } + // TODO add a filter parser to get rid of Serialization + Expression filterExpression = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); + if (filterExpression != null) { + scan = scan.filter(filterExpression); + } + splits = Lists.newArrayList(); try (CloseableIterable tasksIterable = scan.planTasks()) { tasksIterable.forEach(task -> splits.add(new IcebergSplit(task))); @@ -111,39 +204,7 @@ public List getSplits(JobContext context) { @Override public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { - return new IcebergRecordReader(); - } - - public static ConfBuilder updateConf( - Configuration conf, String path, Class> readSupportClass) { - return new ConfBuilder(conf, path, readSupportClass); - } - - public static class ConfBuilder { - private final Configuration conf; - - public ConfBuilder(Configuration conf, String path, Class> readSupportClass) { - this.conf = conf; - conf.set(TABLE_PATH, path); - conf.set(READ_SUPPORT, readSupportClass.getName()); - Table table = getTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - } - - public ConfBuilder filterExpression(Expression expression) throws IOException { - conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); - return this; - } - - public ConfBuilder project(Schema schema) { - conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); - return this; - } - - //TODO: other options split-size, snapshotid etc.. - public Configuration updatedConf() { - return conf; - } + return new IcebergRecordReader<>(); } private static final class IcebergRecordReader extends RecordReader { @@ -153,21 +214,25 @@ private static final class IcebergRecordReader extends RecordReader private T currentRow; private Schema expectedSchema; private Schema tableSchema; - private ReadSupport readSupport; + private InMemoryDataModel inMemoryDataModel; private Closeable currentCloseable; + private boolean reuseContainers; + private boolean caseSensitive; @Override - public void initialize(InputSplit split, TaskAttemptContext context) { - Configuration conf = context.getConfiguration(); + public void initialize(InputSplit split, TaskAttemptContext newContext) { + Configuration conf = newContext.getConfiguration(); CombinedScanTask task = ((IcebergSplit) split).task; - this.context = context; + this.context = newContext; this.tasks = task.files().iterator(); this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); String readSchemaStr = conf.get(READ_SCHEMA); if (readSchemaStr != null) { this.expectedSchema = SchemaParser.fromJson(readSchemaStr); } - this.readSupport = readSupport(conf); + this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); + this.caseSensitive = conf.getBoolean(CASE_SENSITIVE, true); + this.inMemoryDataModel = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.DEFAULT); this.currentIterator = open(tasks.next()); } @@ -206,19 +271,6 @@ public void close() throws IOException { currentCloseable.close(); } - private ReadSupport readSupport(Configuration conf) { - String readSupportClassName = conf.get(READ_SUPPORT); - try { - return DynClasses - .builder() - .impl(readSupportClassName) - .>buildChecked() - .newInstance(); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException(String.format("Unable to instantiate read support %s", readSupportClassName), e); - } - } - private Iterator open(FileScanTask currentTask) { DataFile file = currentTask.file(); // schema of rows returned by readers @@ -229,10 +281,10 @@ private Iterator open(FileScanTask currentTask) { Schema readSchema = expectedSchema != null ? expectedSchema : tableSchema; if (hasJoinedPartitionColumns) { readSchema = TypeUtil.selectNot(tableSchema, idColumns); - Schema partitionSchema = TypeUtil.select(tableSchema, idColumns); + Schema identityPartitionSchema = TypeUtil.select(tableSchema, idColumns); return Iterators.transform( open(currentTask, readSchema), - row -> readSupport.withPartitionColumns(row, partitionSchema, spec, file.partition())); + row -> withPartitionColumns(row, identityPartitionSchema, spec, file.partition())); } else { return open(currentTask, readSchema); } @@ -261,34 +313,103 @@ private Iterator open(FileScanTask currentTask, Schema readSchema) { return iterable.iterator(); } + @SuppressWarnings("unchecked") + private T withPartitionColumns(T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { + switch (inMemoryDataModel) { + case PIG: + case HIVE: + // TODO implement adding partition columns to records for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + return (T) icebergRecordWithPartitionsColumns((Record) row, identityPartitionSchema, spec, partition); + } + return row; + } + + private static Record icebergRecordWithPartitionsColumns( + Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { + List fields = Lists.newArrayList(record.struct().fields()); + fields.addAll(identityPartitionSchema.asStruct().fields()); + GenericRecord row = GenericRecord.create(Types.StructType.of(fields)); + int size = record.struct().fields().size(); + for (int i = 0; i < size; i++) { + row.set(i, record.get(i)); + } + List partitionFields = spec.fields(); + List identityColumns = identityPartitionSchema.columns(); + for (int i = 0; i < identityColumns.size(); i++) { + Types.NestedField identityColumn = identityColumns.get(i); + + for (int j = 0; j < partitionFields.size(); j++) { + PartitionField partitionField = partitionFields.get(j); + if (identityColumn.fieldId() == partitionField.sourceId() && + "identity".equals(partitionField.transform().toString())) { + row.set(size + i, partition.get(j, spec.javaClasses()[i])); + } else { + row.set(size + i, null); + } + } + } + return row; + } + private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) - .createReaderFunc(readSupport.avroReadBiFunction()) - .createReaderFunc(readSupport.avroReadFunction()) .project(readSchema) .split(task.start(), task.length()); - //.reuseContainers(reuseContainers); + + if (reuseContainers) { + avroReadBuilder.reuseContainers(); + } + + switch (inMemoryDataModel) { + case PIG: + case HIVE: + //TODO implement value readers for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + avroReadBuilder.createReaderFunc(DataReader::create); + } return avroReadBuilder.build(); } - private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) - .createBatchedReaderFunc(readSupport.parquetBatchReadFunction()) - .createReaderFunc(readSupport.parquetReadFunction()) - .createBatchedReaderFunc(readSupport.parquetBatchReadFunction()) .project(readSchema) - //.caseSensitive(caseSensitive) + .filter(task.residual()) + .caseSensitive(caseSensitive) .split(task.start(), task.length()); + if (reuseContainers) { + parquetReadBuilder.reuseContainers(); + } + + switch (inMemoryDataModel) { + case PIG: + case HIVE: + //TODO implement value readers for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + parquetReadBuilder.createReaderFunc( + fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); + } return parquetReadBuilder.build(); } private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) - .createReaderFunc(readSupport.orcReadFunction()) .schema(readSchema) - //.caseSensitive(caseSensitive) + .caseSensitive(caseSensitive) .split(task.start(), task.length()); + // ORC does not support reuse containers yet + switch (inMemoryDataModel) { + case PIG: + case HIVE: + //TODO implement value readers for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + //TODO: We do not have support for Iceberg generics for ORC + throw new UnsupportedOperationException(); + } return orcReadBuilder.build(); } @@ -308,15 +429,12 @@ private static Table getTable(Configuration conf) { private static class IcebergSplit extends InputSplit implements Writable { private static final String[] ANYWHERE = new String[]{"*"}; - CombinedScanTask task; + private CombinedScanTask task; IcebergSplit(CombinedScanTask task) { this.task = task; } - public IcebergSplit() { - } - @Override public long getLength() { return task.files().stream().mapToLong(FileScanTask::length).sum(); diff --git a/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java b/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java deleted file mode 100644 index 078c0f650341..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.apache.iceberg.mr; - -import java.util.function.BiFunction; -import org.apache.avro.io.DatumReader; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.orc.OrcValueReader; -import org.apache.iceberg.parquet.ParquetValueReader; -import org.apache.iceberg.parquet.VectorizedReader; -import org.apache.orc.TypeDescription; -import org.apache.parquet.schema.MessageType; - -import java.util.function.Function; - - -/** - * ReadSupport for MR InputFormat, providing value readers - * for different data formats and appending identity partition columns - * to the input row - * @param - */ -public interface ReadSupport { - /** - * Add identity partition columns to input row - */ - default T withPartitionColumns(T row, Schema partitionSchema, PartitionSpec spec, StructLike partitionData) { - return row; - } - - default Function> parquetReadFunction() { - return null; - } - - default Function> parquetBatchReadFunction() { - return null; - } - - default Function> avroReadFunction() { - return null; - } - - default BiFunction> avroReadBiFunction() { - return null; - } - - default Function> orcReadFunction() { - return null; - } -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java index 3ec2a038cdac..af31eb26789f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java +++ b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iceberg.mr; import java.io.ByteArrayInputStream; @@ -17,12 +36,14 @@ public class SerializationUtil { private SerializationUtil() { } - public static byte[] serializeToBytes(Object obj) throws IOException { + public static byte[] serializeToBytes(Object obj) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); GZIPOutputStream gos = new GZIPOutputStream(baos); ObjectOutputStream oos = new ObjectOutputStream(gos)) { oos.writeObject(obj); return baos.toByteArray(); + } catch (IOException e) { + throw new RuntimeIOException("Failed to serialize object", e); } } @@ -37,18 +58,17 @@ public static T deserializeFromBytes(byte[] bytes) { ObjectInputStream ois = new ObjectInputStream(gis)) { return (T) ois.readObject(); } catch (IOException e) { - throw new RuntimeIOException(e); + throw new RuntimeIOException("Failed to deserialize object", e); } catch (ClassNotFoundException e) { throw new RuntimeException("Could not read object ", e); } } - public static String serializeToBase64(Object obj) throws IOException { + public static String serializeToBase64(Object obj) { byte[] bytes = serializeToBytes(obj); return new String(Base64.getMimeEncoder().encode(bytes), StandardCharsets.UTF_8); } - @SuppressWarnings("unchecked") public static T deserializeFromBase64(String base64) { if (base64 == null) { return null; diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java index 58190cccf335..428342d175ce 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iceberg.mr; import java.io.File; @@ -6,11 +25,12 @@ import org.apache.avro.generic.GenericData; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TaskAttemptContextImpl; -import org.apache.hadoop.mapred.TaskAttemptID; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.Files; @@ -36,7 +56,6 @@ public class TestIcebergInputFormat { private static final Configuration CONF = new Configuration(); private static final HadoopTables TABLES = new HadoopTables(CONF); - private Table table; private File tableLocation; private static final Schema SCHEMA = new Schema( @@ -77,14 +96,16 @@ public void test() throws IOException, InterruptedException { table.newAppend().appendFile(dataFile).commit(); - Configuration conf = new Configuration(); - conf = IcebergInputFormat - .updateConf(conf, tableLocation.getAbsolutePath(), TestReadSupport.class) - .updatedConf(); - TaskAttemptContext context = new TaskAttemptContextImpl(new JobConf(conf), new TaskAttemptID()); + Job job = Job.getInstance(new Configuration()); + IcebergInputFormat + .configure(job) + .readFrom(tableLocation.getAbsolutePath()); + + TaskAttemptContext context = new TaskAttemptContextImpl(new JobConf(job.getConfiguration()), new TaskAttemptID()); icebergInputFormat = new IcebergInputFormat<>(); List splits = icebergInputFormat.getSplits(context); - final RecordReader recordReader = icebergInputFormat.createRecordReader(splits.get(0), context); + final RecordReader recordReader = + icebergInputFormat.createRecordReader(splits.get(0), context); recordReader.initialize(splits.get(0), context); while (recordReader.nextKeyValue()) { System.out.println(recordReader.getCurrentValue()); @@ -109,12 +130,4 @@ public void set(int pos, T value) { } }; } - - public static class TestReadSupport implements ReadSupport { - @Override - public GenericData.Record withPartitionColumns( - GenericData.Record row, Schema partitionSchema, PartitionSpec spec, StructLike partitionData) { - return row; - } - } } From 1d07d8c8a88ef9c9ee56c750c519fc13e455d697 Mon Sep 17 00:00:00 2001 From: Ratandeep Ratt Date: Tue, 17 Mar 2020 23:32:37 -0700 Subject: [PATCH 03/51] Address review comments --- build.gradle | 1 + .../apache/iceberg/mr/IcebergInputFormat.java | 325 +++++++++++++----- .../org/apache/iceberg/mr/ReadSupport.java | 50 --- .../apache/iceberg/mr/SerializationUtil.java | 28 +- .../iceberg/mr/TestIcebergInputFormat.java | 49 ++- 5 files changed, 293 insertions(+), 160 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java diff --git a/build.gradle b/build.gradle index 9f675fb9fab9..abbcc2d06b44 100644 --- a/build.gradle +++ b/build.gradle @@ -200,6 +200,7 @@ project(':iceberg-mr') { compile project(':iceberg-hive') compile project(':iceberg-orc') compile project(':iceberg-parquet') + compile project(':iceberg-data') compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java index e6621616017a..038fd73339c8 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java @@ -21,32 +21,44 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.io.Closeable; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.avro.Avro; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.common.DynClasses; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataReader; +import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.hadoop.HadoopInputFile; @@ -57,6 +69,7 @@ import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,16 +77,95 @@ public class IcebergInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); + static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; + static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; - static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - static final String TABLE_PATH = "iceberg.mr.table.path"; + static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; static final String READ_SCHEMA = "iceberg.mr.read.schema"; - static final String READ_SUPPORT = "iceberg.mr.read.support"; + static final String REUSE_CONTAINERS = "iceberg.mr.case.sensitive"; + static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; + static final String SPLIT_SIZE = "iceberg.mr.split.size"; + static final String TABLE_PATH = "iceberg.mr.table.path"; + static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; + private static final String LOCALITY = "iceberg.mr.locality"; - private transient Table table; private transient List splits; - public IcebergInputFormat() { + public enum InMemoryDataModel { + PIG, + HIVE, + DEFAULT // Default data model is of Iceberg Generics + } + + /** + * Configures the {@code Job} to use the {@code IcebergInputFormat} and + * returns a helper to add further configuration. + * + * @param job the {@code Job} to configure + */ + public static ConfigBuilder configure(Job job) { + job.setInputFormatClass(IcebergInputFormat.class); + return new ConfigBuilder(job.getConfiguration()); + } + + public static class ConfigBuilder { + private final Configuration conf; + + public ConfigBuilder(Configuration conf) { + this.conf = conf; + } + + public ConfigBuilder readFrom(String path) { + conf.set(TABLE_PATH, path); + Table table = getTable(conf); + conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + return this; + } + + public ConfigBuilder filter(Expression expression) { + conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); + return this; + } + + public ConfigBuilder project(Schema schema) { + conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder reuseContainers(boolean reuse) { + conf.setBoolean(REUSE_CONTAINERS, reuse); + return this; + } + + public ConfigBuilder caseSensitive(boolean caseSensitive) { + conf.setBoolean(CASE_SENSITIVE, caseSensitive); + return this; + } + + public ConfigBuilder snapshotId(long snapshotId) { + conf.setLong(SNAPSHOT_ID, snapshotId); + return this; + } + + public ConfigBuilder asOfTime(long asOfTime) { + conf.setLong(AS_OF_TIMESTAMP, asOfTime); + return this; + } + + public ConfigBuilder splitSize(long splitSize) { + conf.setLong(SPLIT_SIZE, splitSize); + return this; + } + + public ConfigBuilder locality(boolean localityPreferred) { + conf.setBoolean(LOCALITY, localityPreferred); + return this; + } + + public ConfigBuilder inMemoryDataModel(InMemoryDataModel inMemoryDataModel) { + conf.set(IN_MEMORY_DATA_MODEL, inMemoryDataModel.name()); + return this; + } } @Override @@ -84,24 +176,35 @@ public List getSplits(JobContext context) { } Configuration conf = context.getConfiguration(); - table = getTable(conf); - TableScan scan = table.newScan(); - //TODO add caseSensitive, snapshot id etc.. + Table table = getTable(conf); + TableScan scan = table.newScan() + .caseSensitive(conf.getBoolean(CASE_SENSITIVE, true)); + long snapshotId = conf.getLong(SNAPSHOT_ID, -1); + if (snapshotId != -1) { + scan = scan.useSnapshot(snapshotId); + } + long asOfTime = conf.getLong(AS_OF_TIMESTAMP, -1); + if (asOfTime != -1) { + scan = scan.asOfTime(asOfTime); + } + long splitSize = conf.getLong(SPLIT_SIZE, -1); + if (splitSize != -1) { + scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); + } + String schemaStr = conf.get(READ_SCHEMA); + if (schemaStr != null) { + scan.project(SchemaParser.fromJson(schemaStr)); + } + // TODO add a filter parser to get rid of Serialization Expression filterExpression = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); if (filterExpression != null) { scan = scan.filter(filterExpression); } - final String schemaStr = conf.get(READ_SCHEMA); - if (schemaStr != null) { - // Not sure if this is having any effect? - scan.project(SchemaParser.fromJson(schemaStr)); - } - splits = Lists.newArrayList(); try (CloseableIterable tasksIterable = scan.planTasks()) { - tasksIterable.forEach(task -> splits.add(new IcebergSplit(task))); + tasksIterable.forEach(task -> splits.add(new IcebergSplit(conf, task))); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to close table scan: %s", scan); } @@ -111,39 +214,7 @@ public List getSplits(JobContext context) { @Override public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { - return new IcebergRecordReader(); - } - - public static ConfBuilder updateConf( - Configuration conf, String path, Class> readSupportClass) { - return new ConfBuilder(conf, path, readSupportClass); - } - - public static class ConfBuilder { - private final Configuration conf; - - public ConfBuilder(Configuration conf, String path, Class> readSupportClass) { - this.conf = conf; - conf.set(TABLE_PATH, path); - conf.set(READ_SUPPORT, readSupportClass.getName()); - Table table = getTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - } - - public ConfBuilder filterExpression(Expression expression) throws IOException { - conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); - return this; - } - - public ConfBuilder project(Schema schema) { - conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); - return this; - } - - //TODO: other options split-size, snapshotid etc.. - public Configuration updatedConf() { - return conf; - } + return new IcebergRecordReader<>(); } private static final class IcebergRecordReader extends RecordReader { @@ -153,21 +224,25 @@ private static final class IcebergRecordReader extends RecordReader private T currentRow; private Schema expectedSchema; private Schema tableSchema; - private ReadSupport readSupport; + private InMemoryDataModel inMemoryDataModel; private Closeable currentCloseable; + private boolean reuseContainers; + private boolean caseSensitive; @Override - public void initialize(InputSplit split, TaskAttemptContext context) { - Configuration conf = context.getConfiguration(); + public void initialize(InputSplit split, TaskAttemptContext newContext) { + Configuration conf = newContext.getConfiguration(); CombinedScanTask task = ((IcebergSplit) split).task; - this.context = context; + this.context = newContext; this.tasks = task.files().iterator(); this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); String readSchemaStr = conf.get(READ_SCHEMA); if (readSchemaStr != null) { this.expectedSchema = SchemaParser.fromJson(readSchemaStr); } - this.readSupport = readSupport(conf); + this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); + this.caseSensitive = conf.getBoolean(CASE_SENSITIVE, true); + this.inMemoryDataModel = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.DEFAULT); this.currentIterator = open(tasks.next()); } @@ -206,33 +281,19 @@ public void close() throws IOException { currentCloseable.close(); } - private ReadSupport readSupport(Configuration conf) { - String readSupportClassName = conf.get(READ_SUPPORT); - try { - return DynClasses - .builder() - .impl(readSupportClassName) - .>buildChecked() - .newInstance(); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - throw new RuntimeException(String.format("Unable to instantiate read support %s", readSupportClassName), e); - } - } - private Iterator open(FileScanTask currentTask) { DataFile file = currentTask.file(); // schema of rows returned by readers PartitionSpec spec = currentTask.spec(); Set idColumns = spec.identitySourceIds(); - - boolean hasJoinedPartitionColumns = !idColumns.isEmpty(); Schema readSchema = expectedSchema != null ? expectedSchema : tableSchema; + boolean hasJoinedPartitionColumns = !idColumns.isEmpty(); if (hasJoinedPartitionColumns) { readSchema = TypeUtil.selectNot(tableSchema, idColumns); - Schema partitionSchema = TypeUtil.select(tableSchema, idColumns); + Schema identityPartitionSchema = TypeUtil.select(tableSchema, idColumns); return Iterators.transform( open(currentTask, readSchema), - row -> readSupport.withPartitionColumns(row, partitionSchema, spec, file.partition())); + row -> withPartitionColumns(row, identityPartitionSchema, spec, file.partition())); } else { return open(currentTask, readSchema); } @@ -261,34 +322,101 @@ private Iterator open(FileScanTask currentTask, Schema readSchema) { return iterable.iterator(); } + @SuppressWarnings("unchecked") + private T withPartitionColumns(T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { + switch (inMemoryDataModel) { + case PIG: + case HIVE: + // TODO implement adding partition columns to records for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + return (T) icebergRecordWithPartitionsColumns((Record) row, identityPartitionSchema, spec, partition); + } + return row; + } + + private static Record icebergRecordWithPartitionsColumns( + Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { + List fields = Lists.newArrayList(record.struct().fields()); + fields.addAll(identityPartitionSchema.asStruct().fields()); + GenericRecord row = GenericRecord.create(Types.StructType.of(fields)); + int size = record.struct().fields().size(); + for (int i = 0; i < size; i++) { + row.set(i, record.get(i)); + } + List partitionFields = spec.fields(); + List identityColumns = identityPartitionSchema.columns(); + for (int i = 0; i < identityColumns.size(); i++) { + Types.NestedField identityColumn = identityColumns.get(i); + + for (int j = 0; j < partitionFields.size(); j++) { + PartitionField partitionField = partitionFields.get(j); + if (identityColumn.fieldId() == partitionField.sourceId() && + "identity".equals(partitionField.transform().toString())) { + row.set(size + i, partition.get(j, spec.javaClasses()[i])); + } + } + } + return row; + } + private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) - .createReaderFunc(readSupport.avroReadBiFunction()) - .createReaderFunc(readSupport.avroReadFunction()) .project(readSchema) .split(task.start(), task.length()); - //.reuseContainers(reuseContainers); + + if (reuseContainers) { + avroReadBuilder.reuseContainers(); + } + + switch (inMemoryDataModel) { + case PIG: + case HIVE: + //TODO implement value readers for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + avroReadBuilder.createReaderFunc(DataReader::create); + } return avroReadBuilder.build(); } - private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) - .createBatchedReaderFunc(readSupport.parquetBatchReadFunction()) - .createReaderFunc(readSupport.parquetReadFunction()) - .createBatchedReaderFunc(readSupport.parquetBatchReadFunction()) .project(readSchema) - //.caseSensitive(caseSensitive) + .filter(task.residual()) + .caseSensitive(caseSensitive) .split(task.start(), task.length()); + if (reuseContainers) { + parquetReadBuilder.reuseContainers(); + } + + switch (inMemoryDataModel) { + case PIG: + case HIVE: + //TODO implement value readers for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + parquetReadBuilder.createReaderFunc( + fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); + } return parquetReadBuilder.build(); } private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) - .createReaderFunc(readSupport.orcReadFunction()) .schema(readSchema) - //.caseSensitive(caseSensitive) + .caseSensitive(caseSensitive) .split(task.start(), task.length()); + // ORC does not support reuse containers yet + switch (inMemoryDataModel) { + case PIG: + case HIVE: + //TODO implement value readers for Pig and Hive + throw new UnsupportedOperationException(); + case DEFAULT: + //TODO: We do not have support for Iceberg generics for ORC + throw new UnsupportedOperationException(); + } return orcReadBuilder.build(); } @@ -308,13 +436,13 @@ private static Table getTable(Configuration conf) { private static class IcebergSplit extends InputSplit implements Writable { private static final String[] ANYWHERE = new String[]{"*"}; - CombinedScanTask task; + private CombinedScanTask task; + private transient String[] locations; + private transient Configuration conf; - IcebergSplit(CombinedScanTask task) { + IcebergSplit(Configuration conf, CombinedScanTask task) { this.task = task; - } - - public IcebergSplit() { + this.conf = conf; } @Override @@ -324,8 +452,29 @@ public long getLength() { @Override public String[] getLocations() { - //TODO: add locations for hdfs - return ANYWHERE; + boolean localityPreferred = conf.getBoolean(LOCALITY, false); + if (!localityPreferred) { + return ANYWHERE; + } + if (locations != null) { + return locations; + } + + Set locationSets = Sets.newHashSet(); + for (FileScanTask f : task.files()) { + Path path = new Path(f.file().path().toString()); + try { + FileSystem fs = path.getFileSystem(conf); + for (BlockLocation b : fs.getFileBlockLocations(path, f.start(), f.length())) { + locationSets.addAll(Arrays.asList(b.getHosts())); + } + } catch (IOException ioe) { + LOG.warn("Failed to get block locations for path {}", path, ioe); + } + } + + locations = locationSets.toArray(new String[0]); + return locations; } @Override diff --git a/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java b/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java deleted file mode 100644 index 078c0f650341..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/ReadSupport.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.apache.iceberg.mr; - -import java.util.function.BiFunction; -import org.apache.avro.io.DatumReader; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.orc.OrcValueReader; -import org.apache.iceberg.parquet.ParquetValueReader; -import org.apache.iceberg.parquet.VectorizedReader; -import org.apache.orc.TypeDescription; -import org.apache.parquet.schema.MessageType; - -import java.util.function.Function; - - -/** - * ReadSupport for MR InputFormat, providing value readers - * for different data formats and appending identity partition columns - * to the input row - * @param - */ -public interface ReadSupport { - /** - * Add identity partition columns to input row - */ - default T withPartitionColumns(T row, Schema partitionSchema, PartitionSpec spec, StructLike partitionData) { - return row; - } - - default Function> parquetReadFunction() { - return null; - } - - default Function> parquetBatchReadFunction() { - return null; - } - - default Function> avroReadFunction() { - return null; - } - - default BiFunction> avroReadBiFunction() { - return null; - } - - default Function> orcReadFunction() { - return null; - } -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java index 3ec2a038cdac..af31eb26789f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java +++ b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iceberg.mr; import java.io.ByteArrayInputStream; @@ -17,12 +36,14 @@ public class SerializationUtil { private SerializationUtil() { } - public static byte[] serializeToBytes(Object obj) throws IOException { + public static byte[] serializeToBytes(Object obj) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); GZIPOutputStream gos = new GZIPOutputStream(baos); ObjectOutputStream oos = new ObjectOutputStream(gos)) { oos.writeObject(obj); return baos.toByteArray(); + } catch (IOException e) { + throw new RuntimeIOException("Failed to serialize object", e); } } @@ -37,18 +58,17 @@ public static T deserializeFromBytes(byte[] bytes) { ObjectInputStream ois = new ObjectInputStream(gis)) { return (T) ois.readObject(); } catch (IOException e) { - throw new RuntimeIOException(e); + throw new RuntimeIOException("Failed to deserialize object", e); } catch (ClassNotFoundException e) { throw new RuntimeException("Could not read object ", e); } } - public static String serializeToBase64(Object obj) throws IOException { + public static String serializeToBase64(Object obj) { byte[] bytes = serializeToBytes(obj); return new String(Base64.getMimeEncoder().encode(bytes), StandardCharsets.UTF_8); } - @SuppressWarnings("unchecked") public static T deserializeFromBase64(String base64) { if (base64 == null) { return null; diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java index 58190cccf335..d1c389176e2e 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iceberg.mr; import java.io.File; @@ -5,12 +24,13 @@ import java.util.List; import org.apache.avro.generic.GenericData; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.TaskAttemptContextImpl; -import org.apache.hadoop.mapred.TaskAttemptID; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.Files; @@ -36,7 +56,6 @@ public class TestIcebergInputFormat { private static final Configuration CONF = new Configuration(); private static final HadoopTables TABLES = new HadoopTables(CONF); - private Table table; private File tableLocation; private static final Schema SCHEMA = new Schema( @@ -77,14 +96,16 @@ public void test() throws IOException, InterruptedException { table.newAppend().appendFile(dataFile).commit(); - Configuration conf = new Configuration(); - conf = IcebergInputFormat - .updateConf(conf, tableLocation.getAbsolutePath(), TestReadSupport.class) - .updatedConf(); - TaskAttemptContext context = new TaskAttemptContextImpl(new JobConf(conf), new TaskAttemptID()); + Job job = Job.getInstance(new Configuration()); + IcebergInputFormat + .configure(job) + .readFrom(tableLocation.getAbsolutePath()); + + TaskAttemptContext context = new TaskAttemptContextImpl(new JobConf(job.getConfiguration()), new TaskAttemptID()); icebergInputFormat = new IcebergInputFormat<>(); List splits = icebergInputFormat.getSplits(context); - final RecordReader recordReader = icebergInputFormat.createRecordReader(splits.get(0), context); + final RecordReader recordReader = + icebergInputFormat.createRecordReader(splits.get(0), context); recordReader.initialize(splits.get(0), context); while (recordReader.nextKeyValue()) { System.out.println(recordReader.getCurrentValue()); @@ -109,12 +130,4 @@ public void set(int pos, T value) { } }; } - - public static class TestReadSupport implements ReadSupport { - @Override - public GenericData.Record withPartitionColumns( - GenericData.Record row, Schema partitionSchema, PartitionSpec spec, StructLike partitionData) { - return row; - } - } } From b48fb17b8420773f0958aeec34b365d90ca40461 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 18 Mar 2020 14:30:12 +0000 Subject: [PATCH 04/51] added mapred inputformat --- build.gradle | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index abbcc2d06b44..7763bdfb2672 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ buildscript { } dependencies { classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0' - classpath 'com.palantir.baseline:gradle-baseline-java:0.55.0' + classpath 'com.palantir.baseline:gradle-baseline-java:0.58.0' classpath 'com.diffplug.spotless:spotless-plugin-gradle:3.14.0' classpath 'gradle.plugin.org.inferred:gradle-processors:2.1.0' classpath 'me.champeau.gradle:jmh-gradle-plugin:0.4.8' @@ -209,6 +209,12 @@ project(':iceberg-mr') { testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') + + testCompile("com.klarna:hiverunner:3.2.1") { + exclude group: 'javax.jms', module: 'jms' + exclude group: 'org.codehaus.jettison', module: 'jettison' + exclude group: 'org.datanucleus', module: '*' + } } } From e38f88b85c3c61cf142324fd5b6f0355bfa3782a Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 18 Mar 2020 14:30:43 +0000 Subject: [PATCH 05/51] added mapred inputformat --- .../iceberg/mr/mapred/IcebergInputFormat.java | 196 ++++++++++++++++++ .../mr/mapred/IcebergReaderFactory.java | 89 ++++++++ .../iceberg/mr/mapred/IcebergWritable.java | 61 ++++++ .../mr/mapred/TestIcebergInputFormat.java | 134 ++++++++++++ 4 files changed, 480 insertions(+) create mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java create mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java create mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java new file mode 100644 index 000000000000..e68d337bd159 --- /dev/null +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.iceberg.mr.mapred; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; +import org.apache.iceberg.CombinedScanTask; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.InputFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class IcebergInputFormat implements InputFormat { + private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); + + private Table table; + + @Override + public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + //TODO: Change this to use whichever Catalog the table was made with i.e. HiveCatalog instead etc. + HadoopTables tables = new HadoopTables(job); + String tableDir = job.get("location"); + + URI location = null; + try { + location = new URI(tableDir); + } catch (URISyntaxException e) { + throw new IOException("Unable to create URI for table location: '" + tableDir + "'"); + } + table = tables.load(location.getPath()); + + List tasks = Lists.newArrayList(table.newScan().planTasks()); + return createSplits(tasks); + } + + private InputSplit[] createSplits(List tasks) { + InputSplit[] splits = new InputSplit[tasks.size()]; + for (int i = 0; i < tasks.size(); i++) { + splits[i] = new IcebergSplit(tasks.get(i)); + } + return splits; + } + + @Override + public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { + return new IcebergRecordReader(split, job); + } + + public class IcebergRecordReader implements RecordReader { + private JobConf context; + private IcebergSplit split; + + private Iterator tasks; + private CloseableIterable reader; + private Iterator recordIterator; + private Record currentRecord; + + public IcebergRecordReader(InputSplit split, JobConf conf) throws IOException { + this.split = (IcebergSplit) split; + this.context = conf; + initialise(); + } + + private void initialise() { + tasks = split.getTask().files().iterator(); + nextTask(); + } + + private void nextTask(){ + FileScanTask currentTask = tasks.next(); + DataFile file = currentTask.file(); + InputFile inputFile = HadoopInputFile.fromLocation(file.path(), context); + Schema tableSchema = table.schema(); + boolean reuseContainers = true; // FIXME: read from config + + IcebergReaderFactory readerFactory = new IcebergReaderFactory(); + reader = readerFactory.createReader(file, currentTask, inputFile, tableSchema, reuseContainers); + recordIterator = reader.iterator(); + } + + @Override + public boolean next(Void key, IcebergWritable value) { + if (recordIterator.hasNext()) { + currentRecord = recordIterator.next(); + value.setRecord(currentRecord); + return true; + } + + if(tasks.hasNext()){ + nextTask(); + currentRecord = recordIterator.next(); + value.setRecord(currentRecord); + return true; + } + return false; + } + + @Override + public Void createKey() { + return null; + } + + @Override + public IcebergWritable createValue() { + IcebergWritable record = new IcebergWritable(); + record.setRecord(currentRecord); + record.setSchema(table.schema()); + return record; + } + + @Override + public long getPos() throws IOException { + return 0; + } + + @Override + public void close() throws IOException { + + } + + @Override + public float getProgress() throws IOException { + return 0; + } + } + + private static class IcebergSplit implements InputSplit { + + private CombinedScanTask task; + + IcebergSplit(CombinedScanTask task) { + this.task = task; + } + + @Override + public long getLength() throws IOException { + return 0; + } + + @Override + public String[] getLocations() throws IOException { + return new String[0]; + } + + @Override + public void write(DataOutput out) throws IOException { + + } + + @Override + public void readFields(DataInput in) throws IOException { + + } + + public CombinedScanTask getTask() { + return task; + } + } + +} diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java new file mode 100644 index 000000000000..d39ffb67d632 --- /dev/null +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.iceberg.mr.mapred; + +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataReader; +import org.apache.iceberg.data.parquet.GenericParquetReaders; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; + +class IcebergReaderFactory { + + IcebergReaderFactory() { + } + + public CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, Schema tableSchema, boolean reuseContainers) { + switch (file.format()) { + case AVRO: + return buildAvroReader(currentTask, inputFile, tableSchema, reuseContainers); + case ORC: + return buildOrcReader(currentTask, inputFile, tableSchema, reuseContainers); + case PARQUET: + return buildParquetReader(currentTask, inputFile, tableSchema, reuseContainers); + + default: + throw new UnsupportedOperationException(String.format("Cannot read %s file: %s", file.format().name(), file.path())); + } + } + + // FIXME: use generic reader function + private CloseableIterable buildAvroReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { + Avro.ReadBuilder builder = Avro.read(file) + .createReaderFunc(DataReader::create) + .project(schema) + .split(task.start(), task.length()); + + if (reuseContainers) { + builder.reuseContainers(); + } + + return builder.build(); + } + + // FIXME: use generic reader function + private CloseableIterable buildOrcReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { + ORC.ReadBuilder builder = ORC.read(file) +// .createReaderFunc() // FIXME: implement + .schema(schema) + .split(task.start(), task.length()); + + return builder.build(); + } + + // FIXME: use generic reader function + private CloseableIterable buildParquetReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { + Parquet.ReadBuilder builder = Parquet.read(file) + .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) + .project(schema) + .split(task.start(), task.length()); + + if (reuseContainers) { + builder.reuseContainers(); + } + + return builder.build(); + } +} diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java new file mode 100644 index 000000000000..2f01529d0ce6 --- /dev/null +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.iceberg.mr.mapred; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.Record; + +public class IcebergWritable implements Writable { + + private Record record; + private Schema schema; + + public IcebergWritable() {} + + public void setRecord(Record record) { + this.record = record; + } + + public Record getRecord() { + return record; + } + + public Schema getSchema() { + return schema; + } + + public void setSchema(Schema schema) { + this.schema = schema; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + + } +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java new file mode 100644 index 000000000000..5decb75ff5e7 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -0,0 +1,134 @@ +/** + * Copyright (C) 2020 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iceberg.mr.mapred; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.types.Types; +import org.iceberg.mr.mapred.IcebergInputFormat; +import org.iceberg.mr.mapred.IcebergWritable; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.google.common.collect.Lists; +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.StandaloneHiveRunner; +import com.klarna.hiverunner.annotations.HiveSQL; + +@RunWith(StandaloneHiveRunner.class) +public class TestIcebergInputFormat { + + @HiveSQL(files = {}, autoStart = true) + private HiveShell shell; + + private File tableLocation; + private Table table; + + @Before + public void before() throws IOException { + tableLocation = java.nio.file.Files.createTempDirectory("temp").toFile(); + Schema schema = new Schema(optional(1, "name", Types.StringType.get()), + optional(2, "salary", Types.LongType.get())); + PartitionSpec spec = PartitionSpec.unpartitioned(); + HadoopTables tables = new HadoopTables(); + table = tables.create(schema, spec, tableLocation.getAbsolutePath()); + + DataFile fileA = DataFiles + .builder(spec) + .withPath("src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet") + .withFileSizeInBytes(1024) + .withRecordCount(3) // needs at least one record or else metrics will filter it out + .build(); + + table.newAppend().appendFile(fileA).commit(); + } + + @Test + public void testInputFormat() { + shell.execute("CREATE DATABASE source_db"); + shell.execute(new StringBuilder() + .append("CREATE TABLE source_db.table_a ") + .append("ROW FORMAT SERDE 'com.expediagroup.hiveberg.IcebergSerDe' ") + .append("STORED AS ") + .append("INPUTFORMAT 'com.expediagroup.hiveberg.IcebergInputFormat' ") + .append("OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' ") + .append("LOCATION '") + .append(tableLocation.getAbsolutePath()) + .append("'") + .toString()); + + List result = shell.executeStatement("SELECT * FROM source_db.table_a"); + + assertEquals(3, result.size()); + assertArrayEquals(new Object[]{"Michael", 3000L}, result.get(0)); + assertArrayEquals(new Object[]{"Andy", 3000L}, result.get(1)); + assertArrayEquals(new Object[]{"Berta", 4000L}, result.get(2)); + } + + @Test + public void testGetSplits() throws IOException { + IcebergInputFormat format = new IcebergInputFormat(); + JobConf conf = new JobConf(); + conf.set("location", "file:" + tableLocation); + InputSplit[] splits = format.getSplits(conf, 1); + assertEquals(splits.length, 1); + } + + @Test + public void testGetRecordReader() throws IOException { + IcebergInputFormat format = new IcebergInputFormat(); + JobConf conf = new JobConf(); + conf.set("location", "file:" + tableLocation); + InputSplit[] splits = format.getSplits(conf, 1); + RecordReader reader = format.getRecordReader(splits[0], conf, null); + IcebergWritable value = (IcebergWritable) reader.createValue(); + + List records = Lists.newArrayList(); + boolean unfinished = true; + while(unfinished) { + if (reader.next(null, value)) { + records.add(value.getRecord().copy()); + } else { + unfinished = false; + } + } + assertEquals(3, records.size() ); + } + + @After + public void after() throws IOException { + FileUtils.deleteDirectory(tableLocation); + } +} From 274adbbb8cdc5c78488baeeb8892c053306ec5d3 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 19 Mar 2020 18:09:37 +0000 Subject: [PATCH 06/51] move hive runner to version that matches Hive 2.3.6 used here --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index a32b663b085b..d8f134d057b8 100644 --- a/build.gradle +++ b/build.gradle @@ -236,10 +236,10 @@ project(':iceberg-mr') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - testCompile("com.klarna:hiverunner:3.2.1") { - exclude group: 'javax.jms', module: 'jms' + testCompile("com.klarna:hiverunner:5.1.1") { + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.calcite', module: '*' exclude group: 'org.codehaus.jettison', module: 'jettison' - exclude group: 'org.datanucleus', module: '*' } } } From ad612adefa015a6e18958a1a5c33a74e0127f865 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 19 Mar 2020 18:09:44 +0000 Subject: [PATCH 07/51] checkstyle fixes --- .../iceberg/mr/mapred/IcebergInputFormat.java | 9 ++-- .../mr/mapred/IcebergReaderFactory.java | 10 ++-- .../iceberg/mr/mapred/IcebergWritable.java | 2 +- .../mr/mapred/TestIcebergInputFormat.java | 49 ++++++++++--------- 4 files changed, 38 insertions(+), 32 deletions(-) diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java index e68d337bd159..567f0c52f3e7 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java @@ -16,8 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.iceberg.mr.mapred; +import com.google.common.collect.Lists; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -25,7 +27,6 @@ import java.net.URISyntaxException; import java.util.Iterator; import java.util.List; - import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -44,8 +45,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.Lists; - public class IcebergInputFormat implements InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); @@ -102,7 +101,7 @@ private void initialise() { nextTask(); } - private void nextTask(){ + private void nextTask() { FileScanTask currentTask = tasks.next(); DataFile file = currentTask.file(); InputFile inputFile = HadoopInputFile.fromLocation(file.path(), context); @@ -122,7 +121,7 @@ public boolean next(Void key, IcebergWritable value) { return true; } - if(tasks.hasNext()){ + if (tasks.hasNext()) { nextTask(); currentRecord = recordIterator.next(); value.setRecord(currentRecord); diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java index d39ffb67d632..1b77d27e5dd7 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.iceberg.mr.mapred; import org.apache.iceberg.DataFile; @@ -35,7 +36,8 @@ class IcebergReaderFactory { IcebergReaderFactory() { } - public CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, Schema tableSchema, boolean reuseContainers) { + public CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, + Schema tableSchema, boolean reuseContainers) { switch (file.format()) { case AVRO: return buildAvroReader(currentTask, inputFile, tableSchema, reuseContainers); @@ -45,7 +47,8 @@ public CloseableIterable createReader(DataFile file, FileScanTask curren return buildParquetReader(currentTask, inputFile, tableSchema, reuseContainers); default: - throw new UnsupportedOperationException(String.format("Cannot read %s file: %s", file.format().name(), file.path())); + throw new UnsupportedOperationException(String.format("Cannot read %s file: %s", file.format().name(), + file.path())); } } @@ -74,7 +77,8 @@ private CloseableIterable buildOrcReader(FileScanTask task, InputFile file, Sche } // FIXME: use generic reader function - private CloseableIterable buildParquetReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { + private CloseableIterable buildParquetReader(FileScanTask task, InputFile file, Schema schema, + boolean reuseContainers) { Parquet.ReadBuilder builder = Parquet.read(file) .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) .project(schema) diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java index 2f01529d0ce6..6a73b88aefa0 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ + package org.iceberg.mr.mapred; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; - import org.apache.hadoop.io.Writable; import org.apache.iceberg.Schema; import org.apache.iceberg.data.Record; diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 5decb75ff5e7..4dd18e097aa1 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -1,28 +1,31 @@ -/** - * Copyright (C) 2020 Expedia, Inc. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.iceberg.mr.mapred; -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +package org.apache.iceberg.mr.mapred; +import com.google.common.collect.Lists; +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.StandaloneHiveRunner; +import com.klarna.hiverunner.annotations.HiveSQL; import java.io.File; import java.io.IOException; import java.util.List; - import org.apache.commons.io.FileUtils; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -42,10 +45,10 @@ import org.junit.Test; import org.junit.runner.RunWith; -import com.google.common.collect.Lists; -import com.klarna.hiverunner.HiveShell; -import com.klarna.hiverunner.StandaloneHiveRunner; -import com.klarna.hiverunner.annotations.HiveSQL; +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + @RunWith(StandaloneHiveRunner.class) public class TestIcebergInputFormat { @@ -117,14 +120,14 @@ public void testGetRecordReader() throws IOException { List records = Lists.newArrayList(); boolean unfinished = true; - while(unfinished) { + while (unfinished) { if (reader.next(null, value)) { records.add(value.getRecord().copy()); } else { unfinished = false; } } - assertEquals(3, records.size() ); + assertEquals(3, records.size()); } @After From 1bb76bfa06671a875c83dda6d35a7fdaf49134d5 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 19 Mar 2020 18:22:09 +0000 Subject: [PATCH 08/51] added test table data --- ...d-46fb-804e-e9806abf81c7-00000.parquet.crc | Bin 0 -> 16 bytes ...-ae0d-46fb-804e-e9806abf81c7-00000.parquet | Bin 0 -> 686 bytes ...32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc | Bin 0 -> 44 bytes ...ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc | Bin 0 -> 28 bytes .../test-table/metadata/.v1.metadata.json.crc | Bin 0 -> 16 bytes .../test-table/metadata/.v2.metadata.json.crc | Bin 0 -> 20 bytes .../metadata/.version-hint.text.crc | Bin 0 -> 12 bytes ...3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro | Bin 0 -> 4544 bytes ...-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro | Bin 0 -> 2100 bytes .../test-table/metadata/v1.metadata.json | 31 ++++++++++++ .../test-table/metadata/v2.metadata.json | 47 ++++++++++++++++++ .../test-table/metadata/version-hint.text | 1 + 12 files changed, 79 insertions(+) create mode 100644 mr/src/test/resources/test-table/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc create mode 100644 mr/src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet create mode 100644 mr/src/test/resources/test-table/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc create mode 100644 mr/src/test/resources/test-table/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc create mode 100644 mr/src/test/resources/test-table/metadata/.v1.metadata.json.crc create mode 100644 mr/src/test/resources/test-table/metadata/.v2.metadata.json.crc create mode 100644 mr/src/test/resources/test-table/metadata/.version-hint.text.crc create mode 100644 mr/src/test/resources/test-table/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro create mode 100644 mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro create mode 100644 mr/src/test/resources/test-table/metadata/v1.metadata.json create mode 100644 mr/src/test/resources/test-table/metadata/v2.metadata.json create mode 100644 mr/src/test/resources/test-table/metadata/version-hint.text diff --git a/mr/src/test/resources/test-table/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc b/mr/src/test/resources/test-table/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..caa4e3e7edee6e4a126e7a28b57bfdc329056058 GIT binary patch literal 16 XcmYc;N@ieSU}8A*GUY<|OV3&WD>(*K literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet b/mr/src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet new file mode 100644 index 0000000000000000000000000000000000000000..772144ca6e7be33d4f390f9f7f5de20c5fb967be GIT binary patch literal 686 zcmb7C&ui2`6n>habTC#PM4(XrkiC!3m$Z_SOia^Y<^6(f$X+TvW2De zR1m}-#DmbACqd}Jg9i^HcoBN=;?;k|qc6K6h3dg!nD=Jh`@ZkJ$84-UbSTg}G!j%r zN2QwSLVa#M2{(P28f0x0O#tBP3k}b5g+agHj|X$FJ|CU1=kGjm`)&BNS6zC!+grYI zkHCX-gCmZ4lu(Pd@1r57;Xl5719jo&wVUvI{nPu)Wg^fkv{TBHsl1`Rxl{@P7~02$ znWsbjFRvIoQ&$u#aC~s#);tDjg>~{tSOi_HRtz%ohq4N-3f6~LHTP3Ln>-?* ztITAgrkQk+wKP!KER_;n)ejZ@SgC2lygH~Brgqs{=K5Bz)a&}63RekgGL^1%As!@Dto+`Y)jtZd6A97f(sreGL3TH zdA!Gyh<6;p%eFCNY6Q>Z&N#^=hGIX>r8q^9j0SvP%y27zaI-NX^S_YkR*8GE4@Pg> z&vwSLSRQ89uFUdeYh*bM$8I=QJs2lxWch6`irca0HCF;x#;)6JMz(9ao^08n7p!`# Vb|8buxz2}JPt CUlWl4 literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc b/mr/src/test/resources/test-table/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..68cbb3719c4c209a5f89af23ed3fc55f220881d3 GIT binary patch literal 28 kcmYc;N@ieSU}6wRd@k2gs=ec9!HHX5i=4mi4L>mj0EyTPEC2ui literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/.v1.metadata.json.crc b/mr/src/test/resources/test-table/metadata/.v1.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..87238dff5fd450a287db59ad77696ebd8c7ddc99 GIT binary patch literal 16 XcmYc;N@ieSU}8A)IQin^wij0cEh`5^ literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/.v2.metadata.json.crc b/mr/src/test/resources/test-table/metadata/.v2.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..500d5ca3c03cf017664a4e9772ccda954abac742 GIT binary patch literal 20 bcmYc;N@ieSU}CTjo~$-ulh)Lt1hazxHc19h literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/.version-hint.text.crc b/mr/src/test/resources/test-table/metadata/.version-hint.text.crc new file mode 100644 index 0000000000000000000000000000000000000000..20031206a3b58c7bd0e0b0cf48215fa64e60ea8c GIT binary patch literal 12 TcmYc;N@ieSU}BKEx{ntC5%2=_ literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro b/mr/src/test/resources/test-table/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro new file mode 100644 index 0000000000000000000000000000000000000000..cfd5b85f8fe17c93a8a19ff50be99fed1447ce52 GIT binary patch literal 4544 zcmb_fTWB0r7*5ilwu!Y$EtG~Tr$I{Hv~$UI35bG-Mw%9{kSvq4yJwSQW@k1tvk7Tg zMO5(8KE#Jm6jG=J1zS-cqV-m-Vk$yPC|afQ#air(XrbUk6+LGz=bV|{*=&|PExR+{ zf4=|!zyCkq4yK=JY;CnOV<-=gH5~@dbOC`RupO(IaX_$) z)5;RO&m#{FoI|#wA;WRw3XJ_%YzI0;n?U^Ju)#9xph-!TW$>;63-*}lXygF{xRm@_ zxCmK5WT6AK9M%zgKbEVdkax=YQLt8k&KQe8>rcxUI#UvPx|p14hvp>Mk)caCy3x&`rPgQyH7+{EW+Tk2WdGy@X-8 zG{+HT9y2!tDbSjPdXdinC~u+MOQt$O9*p7))F1a2O&%7=uve{kZ}bpPXzP=EAIV*j z>dCz?N{bh&Ja3+&6mZUr<+sA z($ZwhkQ*sbU*Hyb}%W&xig8mB4&BG z%OZ}B+cSF4(ft+nbau%6#6aEU7GSxr6KzmsqRQa{j}%amC2mf`?x}mioU6~DBUbd#6fX*_#Z$> zVp0!-M@&-;oYhyyHg&k6sgp^k{2$8v6pu#*FSMeUF^Cu_jtC{d7*LCn0AvOEUX7$e zZ^ppLxf{@bM9%OQVlvnk-cx==gj63R<-W)-c8;ix6bTeVN+CnFCQ>q=!b+B7jm#7> zm2u2S8`s24;Zlxxfwht%YK|jC+8dBclRpXWe@gd<4t_|uiwc)k(mOG{sFYyZZ6A6#oloa}FGxkKG>@4kkIe=twqc=nUacm4h6eP_(u z&L98WoV(OCIC5pC|JbQbJ6_v4@Xoo^TN`%{zcfG9wf)7F<)b$b4~!q!bqckN3_Lb| zXWJj!7nRX{W|uGCf1&g3BNJa~hkl*;XrTSXq_jNo+`-&b`g(HY b(|<1fwerT%vkx4Z+x+v~^znZiqv`(-(X8@) literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro b/mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro new file mode 100644 index 0000000000000000000000000000000000000000..81e4ba7929460647c0ffe287d5325c71d69306e2 GIT binary patch literal 2100 zcmb`HPfXKL9LMLP#1IAvQ9yzTG{V75T?bpYkr;R2fC2y9h=}#IecPVt+S$H=3oH@B zg%~47Q4`^2yg<|o=utT+k;H=sjRrj$&P4Q}F}{xWwd>dk4lKJWK_zwf7Kg)4ii z8^Exp9@oVbOoQdxQ3mHSkclxG6je=P{EPz9#_u#xPy*_hPaqkR#|Z>wNoQh%qa|-- z08WuNOpuN>q$I71V@P6Rp+F$-FRhcIl5|$eYDpdIsLKEiBaBpq*E3M$37DRV6zn=M z=qLn|1SMWqKt@lg*jYHpC|Ox1rG=x1{EP$>AS+`gW;{19ZC;_XgeUGGivFstkc*m6%J4I=Yk95vcJ3*^HeOt35`@VW?N$ z4eJbTHk0{37@Hr;`x&x`G*qzJZ6seg92}`RP|#IWB`DUCbjkq31}n-0)4@f8Q7#${ zM#3ByWP>ah2)7^+3Q$X$lCSva(21K_k*cyaW>e5o@lCBYcr}YONHiwJkSb?tZp?d& zz?KrI;AI#E>uf@h5@@g-s+VCHirI)xir7wi8<7CAZr{b#N0$q-jW*>-#odG`sFK<{ zV}&WJg3UWaaTG=8YzSPYn1)9tUnlaKE5h`F!|n1hi_)XrTHB`=#mv&2u9yu_M6uvD zpm@0gQh=7TGs4@@^&9U5> Mo9vz2b?$cm0y@>v9smFU literal 0 HcmV?d00001 diff --git a/mr/src/test/resources/test-table/metadata/v1.metadata.json b/mr/src/test/resources/test-table/metadata/v1.metadata.json new file mode 100644 index 000000000000..d14ac4529e3f --- /dev/null +++ b/mr/src/test/resources/test-table/metadata/v1.metadata.json @@ -0,0 +1,31 @@ +{ + "format-version" : 1, + "table-uuid" : "5ec03633-03bc-4c4b-8ef9-f799c143e3e7", + "location" : "/Users/cmathiesen/projects/opensource/forks/eg-iceberg-fork/incubator-iceberg/mr/src/test/resources/test-table", + "last-updated-ms" : 1582645440292, + "last-column-id" : 2, + "schema" : { + "type" : "struct", + "fields" : [ { + "id" : 1, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 2, + "name" : "salary", + "required" : false, + "type" : "long" + } ] + }, + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "properties" : { }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ] +} \ No newline at end of file diff --git a/mr/src/test/resources/test-table/metadata/v2.metadata.json b/mr/src/test/resources/test-table/metadata/v2.metadata.json new file mode 100644 index 000000000000..ea938f9a0dbe --- /dev/null +++ b/mr/src/test/resources/test-table/metadata/v2.metadata.json @@ -0,0 +1,47 @@ +{ + "format-version" : 1, + "table-uuid" : "5ec03633-03bc-4c4b-8ef9-f799c143e3e7", + "location" : "/Users/cmathiesen/projects/opensource/forks/eg-iceberg-fork/incubator-iceberg/mr/src/test/resources/test-table", + "last-updated-ms" : 1582645443979, + "last-column-id" : 2, + "schema" : { + "type" : "struct", + "fields" : [ { + "id" : 1, + "name" : "name", + "required" : false, + "type" : "string" + }, { + "id" : 2, + "name" : "salary", + "required" : false, + "type" : "long" + } ] + }, + "partition-spec" : [ ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "properties" : { }, + "current-snapshot-id" : 7829799286772121706, + "snapshots" : [ { + "snapshot-id" : 7829799286772121706, + "timestamp-ms" : 1582645443979, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1582645439954", + "added-data-files" : "1", + "added-records" : "3", + "changed-partition-count" : "1", + "total-records" : "3", + "total-data-files" : "1" + }, + "manifest-list" : "/Users/cmathiesen/projects/opensource/forks/eg-iceberg-fork/incubator-iceberg/mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro" + } ], + "snapshot-log" : [ { + "timestamp-ms" : 1582645443979, + "snapshot-id" : 7829799286772121706 + } ] +} \ No newline at end of file diff --git a/mr/src/test/resources/test-table/metadata/version-hint.text b/mr/src/test/resources/test-table/metadata/version-hint.text new file mode 100644 index 000000000000..d8263ee98605 --- /dev/null +++ b/mr/src/test/resources/test-table/metadata/version-hint.text @@ -0,0 +1 @@ +2 \ No newline at end of file From 03f375afcaf2b43a003b6202daa6033ae99da530 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Wed, 25 Mar 2020 12:14:33 +0000 Subject: [PATCH 09/51] Shading modules needed for mapred api --- build.gradle | 358 +++++++++++++++++- .../apache/iceberg/mr/IcebergInputFormat.java | 14 +- .../IcebergObjectInspectorGenerator.java | 83 ++++ .../mr/mapred/IcebergReaderFactory.java | 3 +- .../mr/mapred/IcebergSchemaToTypeInfo.java | 117 ++++++ .../org/iceberg/mr/mapred/IcebergSerDe.java | 91 +++++ .../mr/mapred/TestIcebergInputFormat.java | 4 +- versions.lock | 249 ++++++++---- 8 files changed, 810 insertions(+), 109 deletions(-) create mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java create mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java create mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java diff --git a/build.gradle b/build.gradle index d8f134d057b8..2d8b66c80996 100644 --- a/build.gradle +++ b/build.gradle @@ -80,40 +80,69 @@ subprojects { sourceCompatibility = '1.8' targetCompatibility = '1.8' +} +apply from: 'baseline.gradle' +apply from: 'deploy.gradle' +apply from: 'tasks.gradle' +apply from: 'jmh.gradle' + +project(':iceberg-api') { dependencies { - compile 'org.slf4j:slf4j-api' compile('com.google.guava:guava') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } - compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' testCompile 'junit:junit' testCompile 'org.slf4j:slf4j-simple' testCompile 'org.mockito:mockito-core' + + testCompile "org.apache.avro:avro" + testCompile 'joda-time:joda-time' } } -apply from: 'baseline.gradle' -apply from: 'deploy.gradle' -apply from: 'tasks.gradle' -apply from: 'jmh.gradle' +project(':iceberg-common') { -project(':iceberg-api') { dependencies { - testCompile "org.apache.avro:avro" - testCompile 'joda-time:joda-time' + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' } } -project(':iceberg-common') {} - project(':iceberg-core') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { compile project(':iceberg-api') compile project(':iceberg-common') + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compile("org.apache.avro:avro") { exclude group: 'org.tukaani' // xz compression is not supported } @@ -128,14 +157,61 @@ project(':iceberg-core') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + + dependencies { + exclude (dependency('org.apache.avro:avro')) + exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) + exclude (dependency('com.fasterxml.jackson.core:jackson-core')) + exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) + exclude (dependency('org.checkerframework:checker-qual')) + exclude (dependency('com.github.ben-manes.caffeine:caffeine')) + exclude (dependency('org.slf4j:slf4j-api')) + exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) + exclude (dependency('org.apache.commons:commons-compress')) + } + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' + } } project(':iceberg-data') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { compile project(':iceberg-api') compile project(':iceberg-core') compileOnly project(':iceberg-parquet') compileOnly project(':iceberg-orc') + + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compileOnly("org.apache.hadoop:hadoop-common") { exclude group: 'commons-beanutils' exclude group: 'org.apache.avro', module: 'avro' @@ -154,12 +230,52 @@ project(':iceberg-data') { // Only for TestSplitScan as of Gradle 5.0+ maxHeapSize '1500m' } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + + dependencies { + exclude (dependency('org.apache.avro:avro')) + exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) + exclude (dependency('com.fasterxml.jackson.core:jackson-core')) + exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) + exclude (dependency('org.checkerframework:checker-qual')) + exclude (dependency('com.github.ben-manes.caffeine:caffeine')) + exclude (dependency('org.slf4j:slf4j-api')) + exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) + exclude (dependency('org.apache.commons:commons-compress')) + exclude (dependency('org.apache.iceberg:iceberg-common')) + exclude (dependency('org.apache.iceberg:iceberg-api')) + exclude (dependency('org.apache.iceberg:iceberg-core')) + + } + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' + } } project(':iceberg-hive') { dependencies { compile project(':iceberg-core') + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compileOnly "org.apache.avro:avro" compileOnly("org.apache.hive:hive-metastore") { @@ -221,17 +337,33 @@ project(':iceberg-hive') { project(':iceberg-mr') { dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') - compile project(':iceberg-hive') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-data') + compile project(path: ':iceberg-core', configuration: 'shadow') + compile project(path: ':iceberg-orc', configuration: 'shadow') + compile project(path: ':iceberg-parquet', configuration: 'shadow') + compile project(path: ':iceberg-data', configuration: 'shadow') compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' } + compileOnly('com.github.ben-manes.caffeine:caffeine') + compileOnly ('org.apache.calcite:calcite-core') + + compile("org.apache.hive:hive-serde:2.3.6") { + exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' + exclude group: 'org.apache.ant', module: '*' + exclude group: 'javax.servlet', module: 'jsp-api' + exclude group: 'commons-beanutils', module: 'commons-beanutils-core' + exclude group: 'javax.annotation', module: '*' + exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet' + exclude group: 'org.apache.logging.log4j', module: 'log4j-1.2-api' + exclude group: 'commons-beanutils', module: 'commons-beanutils' + exclude group: 'org.apache.geronimo.specs', module: 'geronimo-annotation_1.0_spec' + exclude group: 'org.checkerframework', module: 'checker-qual' + exclude group: 'org.codehaus.mojo', module: 'animal-sniffer-annotations' + exclude group: 'commons-collections', module: 'commons-collections' + } + testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') @@ -245,10 +377,27 @@ project(':iceberg-mr') { } project(':iceberg-orc') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { compile project(':iceberg-api') compile project(':iceberg-core') + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compile("org.apache.orc:orc-core::nohive") { exclude group: 'org.apache.hadoop' exclude group: 'commons-lang' @@ -268,13 +417,73 @@ project(':iceberg-orc') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } + + configurations { + compile { + exclude group: 'javax.xml.bind' + } + } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + + zip64 true + + dependencies { + exclude (dependency('com.fasterxml.jackson.core:jackson-core')) + exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) + exclude (dependency('org.apache.iceberg:iceberg-common')) + exclude (dependency('org.apache.avro:avro')) + exclude (dependency('org.jetbrains:annotations')) + exclude (dependency('org.checkerframework:checker-qual')) + exclude (dependency('io.airlift:aircompressor')) + exclude (dependency('com.github.ben-manes.caffeine:caffeine')) + exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) + exclude (dependency('org.apache.commons:commons-compress')) + exclude (dependency('org.apache.orc:orc-shims')) + exclude (dependency('org.apache.iceberg:iceberg-core')) + exclude (dependency('org.slf4j:slf4j-api')) + exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) + exclude (dependency('org.apache.orc:orc-core')) + exclude (dependency('org.apache.iceberg:iceberg-api')) + + } + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' + } } project(':iceberg-parquet') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { compile project(':iceberg-api') compile project(':iceberg-core') + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compileOnly 'org.slf4j:slf4j-api' + compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compile("org.apache.parquet:parquet-avro") { exclude group: 'org.apache.avro', module: 'avro' // already shaded by Parquet @@ -289,6 +498,99 @@ project(':iceberg-parquet') { testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') } + + configurations { + compile { + exclude group: 'javax.xml.bind' + } + } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compileOnly] + + dependencies { + exclude (dependency('commons-collections:commons-collections:3.2.2')) + exclude (dependency('org.apache.hadoop:hadoop-yarn-api')) + exclude (dependency('org.apache.htrace:htrace-core')) + exclude (dependency('com.sun.xml.bind:jaxb-impl')) + exclude (dependency('org.apache.directory.server:apacheds-i18n')) + exclude (dependency('com.fasterxml.jackson.core:jackson-core')) + exclude (dependency('xmlenc:xmlenc')) + exclude (dependency('org.codehaus.jackson:jackson-mapper-asl')) + exclude (dependency('org.sonatype.sisu.inject:cglib')) + exclude (dependency('commons-codec:commons-codec')) + exclude (dependency('org.slf4j:slf4j-api')) + exclude (dependency('com.sun.jersey:jersey-json')) + exclude (dependency('org.apache.directory.api:api-util')) + exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) + exclude (dependency('org.apache.commons:commons-math3')) + exclude (dependency('org.codehaus.jackson:jackson-xc')) + exclude (dependency('org.apache.directory.api:api-asn1-api')) + exclude (dependency('org.apache.hadoop:hadoop-yarn-common')) + exclude (dependency('commons-configuration:commons-configuration')) + exclude (dependency('org.apache.commons:commons-compress')) + exclude (dependency('javax.xml.bind:jaxb-api')) + exclude (dependency('org.apache.hadoop:hadoop-auth')) + exclude (dependency('commons-lang:commons-lang')) + exclude (dependency('org.apache.curator:curator-client')) + exclude (dependency('org.apache.hadoop:hadoop-common')) + exclude (dependency('com.sun.jersey:jersey-client')) + exclude (dependency('com.sun.jersey:jersey-core')) + exclude (dependency('javax.servlet:servlet-api')) + exclude (dependency('org.checkerframework:checker-qual')) + exclude (dependency('io.netty:netty')) + exclude (dependency(' org.apache.httpcomponents:httpcore')) + exclude (dependency('org.apache.avro:avro')) + exclude (dependency('javax.inject:javax.inject')) + exclude (dependency('log4j:log4j')) + exclude (dependency('org.codehaus.jackson:jackson-jaxrs')) + exclude (dependency('jline:jline')) + exclude (dependency('org.apache.directory.server:apacheds-kerberos-codec')) + exclude (dependency('aopalliance:aopalliance')) + exclude (dependency('asm:asm')) + exclude (dependency('commons-httpclient:commons-httpclient')) + exclude (dependency('commons-collections:commons-collections')) + exclude (dependency('commons-io:commons-io')) + exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) + exclude (dependency('org.apache.hadoop:hadoop-annotations')) + exclude (dependency('org.apache.zookeeper:zookeeper')) + exclude (dependency('org.codehaus.jackson:jackson-core-asl')) + exclude (dependency('org.apache.httpcomponents:httpcore')) + exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) + exclude (dependency('org.fusesource.leveldbjni:leveldbjni-all')) + exclude (dependency('com.sun.jersey:jersey-server')) + exclude (dependency('commons-logging:commons-logging')) + exclude (dependency('javax.activation:activation')) + exclude (dependency('org.apache.httpcomponents:httpclient')) + exclude (dependency('org.mortbay.jetty:jetty-util')) + exclude (dependency('org.apache.curator:curator-recipes')) + exclude (dependency('commons-cli:commons-cli')) + exclude (dependency('com.sun.jersey.contribs:jersey-guice')) + exclude (dependency('commons-collections:commons-collections')) + exclude (dependency('com.google.code.findbugs:jsr305')) + exclude (dependency('commons-digester:commons-digester')) + exclude (dependency('jline:jline')) + exclude (dependency('org.codehaus.jettison:jettison')) + exclude (dependency('org.apache.hadoop:hadoop-yarn-server-common')) + exclude (dependency('commons-net:commons-net')) + exclude (dependency('javax.servlet.jsp:jsp-api')) + exclude (dependency('org.apache.curator:curator-framework')) + } + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' + relocate 'commons-collections', 'org.apache.iceberg.shaded.commons-collections' + } } project(':iceberg-arrow') { @@ -296,6 +598,17 @@ project(':iceberg-arrow') { compile project(':iceberg-api') compile project(':iceberg-parquet') + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compile 'org.slf4j:slf4j-api' + compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compile("org.apache.arrow:arrow-vector") { exclude group: 'io.netty', module: 'netty-buffer' exclude group: 'io.netty', module: 'netty-common' @@ -318,6 +631,17 @@ project(':iceberg-spark') { compile project(':iceberg-arrow') compile project(':iceberg-hive') + compile('com.google.guava:guava') { + // may be LGPL - use ALv2 findbugs-annotations instead + exclude group: 'com.google.code.findbugs' + } + compile 'org.slf4j:slf4j-api' + compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + + testCompile 'junit:junit' + testCompile 'org.slf4j:slf4j-simple' + testCompile 'org.mockito:mockito-core' + compileOnly "org.apache.avro:avro" compileOnly("org.apache.spark:spark-hive_2.11") { exclude group: 'org.apache.avro', module: 'avro' diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java index 038fd73339c8..d1424cde4fa3 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java @@ -58,12 +58,11 @@ import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataReader; -import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.hadoop.HadoopInputFile; import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.hive.HiveCatalogs; +//import org.apache.iceberg.hive.HiveCatalogs; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.orc.ORC; @@ -396,8 +395,8 @@ private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTas //TODO implement value readers for Pig and Hive throw new UnsupportedOperationException(); case DEFAULT: - parquetReadBuilder.createReaderFunc( - fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); + //parquetReadBuilder.createReaderFunc( + //fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); } return parquetReadBuilder.build(); } @@ -428,9 +427,10 @@ private static Table getTable(Configuration conf) { HadoopTables tables = new HadoopTables(conf); return tables.load(path); } else { - Catalog catalog = HiveCatalogs.loadCatalog(conf); - TableIdentifier tableIdentifier = TableIdentifier.parse(path); - return catalog.loadTable(tableIdentifier); + //Catalog catalog = HiveCatalogs.loadCatalog(conf); + //TableIdentifier tableIdentifier = TableIdentifier.parse(path); + //return catalog.loadTable(tableIdentifier); + return null; } } diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java new file mode 100644 index 000000000000..750e3498effb --- /dev/null +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2020 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.iceberg.mr.mapred; + +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Types; + +import java.util.ArrayList; +import java.util.List; + +class IcebergObjectInspectorGenerator { + + protected ObjectInspector createObjectInspector(Schema schema) throws Exception { + List columnNames = setColumnNames(schema); + List columnTypes = IcebergSchemaToTypeInfo.getColumnTypes(schema); + + List columnOIs = new ArrayList<>(columnTypes.size()); + for(int i = 0; i < columnTypes.size(); i++) { + columnOIs.add(createObjectInspectorWorker(columnTypes.get(i))); + } + return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs, null); + } + + protected ObjectInspector createObjectInspectorWorker(TypeInfo typeInfo) throws Exception { + ObjectInspector.Category typeCategory = typeInfo.getCategory(); + + switch(typeCategory) { + case PRIMITIVE: + PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo; + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti); + case LIST: + ListTypeInfo ati = (ListTypeInfo) typeInfo; + return ObjectInspectorFactory + .getStandardListObjectInspector(createObjectInspectorWorker(ati.getListElementTypeInfo())); + case MAP: + MapTypeInfo mti = (MapTypeInfo) typeInfo; + return ObjectInspectorFactory.getStandardMapObjectInspector( + createObjectInspectorWorker(mti.getMapKeyTypeInfo()), + createObjectInspectorWorker(mti.getMapValueTypeInfo())); + case STRUCT: + StructTypeInfo sti = (StructTypeInfo) typeInfo; + List ois = new ArrayList<>(sti.getAllStructFieldTypeInfos().size()); + for (TypeInfo structTypeInfos : sti.getAllStructFieldTypeInfos()) { + ois.add(createObjectInspectorWorker(structTypeInfos)); + } + return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois); + default: + throw new SerDeException("Couldn't create Object Inspector for category: '" + typeCategory + "'"); + } + } + + protected List setColumnNames(Schema schema) { + List fields = schema.columns(); + List fieldsList = new ArrayList<>(fields.size()); + for (Types.NestedField field : fields) { + fieldsList.add(field.name()); + } + return fieldsList; + } + +} diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java index 1b77d27e5dd7..f49b2ae411be 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java @@ -25,7 +25,6 @@ import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataReader; -import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.orc.ORC; @@ -80,7 +79,7 @@ private CloseableIterable buildOrcReader(FileScanTask task, InputFile file, Sche private CloseableIterable buildParquetReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { Parquet.ReadBuilder builder = Parquet.read(file) - .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) + //.createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) .project(schema) .split(task.start(), task.length()); diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java new file mode 100644 index 000000000000..d8ca94b4060d --- /dev/null +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.iceberg.mr.mapred; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.iceberg.Schema; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class to convert Iceberg types to Hive TypeInfo + */ +final class IcebergSchemaToTypeInfo { + + private IcebergSchemaToTypeInfo() { + + } + + private static final ImmutableMap primitiveTypeToTypeInfo = ImmutableMap.builder() + .put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME)) + .put(Types.IntegerType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME)) + .put(Types.LongType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) + .put(Types.FloatType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME)) + .put(Types.DoubleType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)) + .put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME)) + .put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)) + .put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME)) + .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)).build(); + + public static List getColumnTypes(Schema schema) throws Exception { + List fields = schema.columns(); + List types = new ArrayList<>(fields.size()); + for (Types.NestedField field: fields) { + types.add(generateTypeInfo(field.type())); + } + return types; + } + + private static TypeInfo generateTypeInfo(Type type) throws Exception { + if (primitiveTypeToTypeInfo.containsKey(type)) { + return (TypeInfo) primitiveTypeToTypeInfo.get(type); + } + switch(type.typeId()) { + case UUID: + return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); + case FIXED: + return TypeInfoFactory.getPrimitiveTypeInfo("binary"); + case TIME: + return TypeInfoFactory.getPrimitiveTypeInfo("long"); + case DECIMAL: + Types.DecimalType dec = (Types.DecimalType)type; + int scale = dec.scale(); + int precision = dec.precision(); + try { + HiveDecimalUtils.validateParameter(precision, scale); + } catch (Exception e) { + //TODO Log that precision / scale isn't valid + throw e; + } + return TypeInfoFactory.getDecimalTypeInfo(precision, scale); + case STRUCT: + return generateStructTypeInfo((Types.StructType)type); + case LIST: + return generateListTypeInfo((Types.ListType)type); + case MAP: + return generateMapTypeInfo((Types.MapType)type); + default: + throw new SerDeException("Can't map Iceberg type to Hive TypeInfo: '" + type.typeId() + "'"); + } + } + + private static TypeInfo generateMapTypeInfo(Types.MapType type) throws Exception { + Type keyType = type.keyType(); + Type valueType = type.valueType(); + return TypeInfoFactory.getMapTypeInfo(generateTypeInfo(keyType), generateTypeInfo(valueType)); + } + + private static TypeInfo generateStructTypeInfo(Types.StructType type) throws Exception { + List fields = type.fields(); + List fieldNames = new ArrayList<>(fields.size()); + List typeInfos = new ArrayList<>(fields.size()); + + for (Types.NestedField field : fields) { + fieldNames.add(field.name()); + typeInfos.add(generateTypeInfo(field.type())); + } + return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); + } + + private static TypeInfo generateListTypeInfo(Types.ListType type) throws Exception { + return TypeInfoFactory.getListTypeInfo(generateTypeInfo(type.elementType())); + } +} diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java new file mode 100644 index 000000000000..888e16348c60 --- /dev/null +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java @@ -0,0 +1,91 @@ +/** + * Copyright (C) 2020 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.iceberg.mr.mapred; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import javax.annotation.Nullable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.serde2.AbstractSerDe; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.SerDeStats; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.io.Writable; +import org.apache.iceberg.Schema; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.hadoop.HadoopFileIO; +import org.apache.iceberg.types.Types; + +public class IcebergSerDe extends AbstractSerDe { + + private Schema schema; + private TableMetadata metadata; + private ObjectInspector inspector; + private List columnNames; + private List columnTypes; + + @Override + public void initialize(@Nullable Configuration configuration, Properties properties) throws SerDeException { + //TODO Add methods to dynamically find most recent metadata + String tableDir = properties.getProperty("location") + "/metadata/v2.metadata.json"; + this.metadata = TableMetadataParser.read(new HadoopFileIO(configuration), tableDir); + this.schema = metadata.schema(); + + try { + this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(schema); + } catch (Exception e) { + throw new SerDeException(e); + } + } + + @Override + public Class getSerializedClass() { + return null; + } + + @Override + public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException { + return null; + } + + @Override + public SerDeStats getSerDeStats() { + return null; + } + + @Override + public Object deserialize(Writable writable) throws SerDeException { + IcebergWritable icebergWritable = (IcebergWritable) writable; + Schema schema = icebergWritable.getSchema(); + List fields = schema.columns(); + List row = new ArrayList<>(); + + for(Types.NestedField field: fields){ + Object obj = ((IcebergWritable) writable).getRecord().getField(field.name()); + row.add(obj); + } + return Collections.unmodifiableList(row); + } + + @Override + public ObjectInspector getObjectInspector() throws SerDeException { + return inspector; + } +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 4dd18e097aa1..803217f4e980 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -83,9 +83,9 @@ public void testInputFormat() { shell.execute("CREATE DATABASE source_db"); shell.execute(new StringBuilder() .append("CREATE TABLE source_db.table_a ") - .append("ROW FORMAT SERDE 'com.expediagroup.hiveberg.IcebergSerDe' ") + .append("ROW FORMAT SERDE 'org.iceberg.mr.mapred.IcebergSerDe' ") .append("STORED AS ") - .append("INPUTFORMAT 'com.expediagroup.hiveberg.IcebergInputFormat' ") + .append("INPUTFORMAT 'org.iceberg.mr.mapred.IcebergInputFormat' ") .append("OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' ") .append("LOCATION '") .append(tableLocation.getAbsolutePath()) diff --git a/versions.lock b/versions.lock index eadc7b4e6055..c7cf1f751f8c 100644 --- a/versions.lock +++ b/versions.lock @@ -1,90 +1,104 @@ # Run ./gradlew --write-locks to regenerate this file ant:ant:1.6.5 (2 constraints: 2d1539e1) aopalliance:aopalliance:1.0 (1 constraints: 170a83ac) -asm:asm:3.1 (2 constraints: 4f19c3c6) +asm:asm:3.1 (3 constraints: 251fd7ad) +asm:asm-commons:3.1 (1 constraints: 9c0f1f7a) +asm:asm-tree:3.1 (1 constraints: 2307035c) +ch.qos.logback:logback-classic:1.0.9 (3 constraints: d3258a88) +ch.qos.logback:logback-core:1.0.9 (4 constraints: dd32435a) +co.cask.tephra:tephra-api:0.6.0 (3 constraints: 0828ded1) +co.cask.tephra:tephra-core:0.6.0 (2 constraints: 831cd90d) +co.cask.tephra:tephra-hbase-compat-1.0:0.6.0 (1 constraints: 370d6920) com.carrotsearch:hppc:0.7.2 (1 constraints: f70cda14) com.clearspring.analytics:stream:2.7.0 (1 constraints: 1a0dd136) com.esotericsoftware:kryo-shaded:4.0.2 (2 constraints: b71345a6) com.esotericsoftware:minlog:1.3.0 (1 constraints: 670e7c4f) -com.fasterxml.jackson.core:jackson-annotations:2.10.2 (5 constraints: 4155160f) -com.fasterxml.jackson.core:jackson-core:2.10.2 (6 constraints: bb52b302) -com.fasterxml.jackson.core:jackson-databind:2.10.2 (11 constraints: 7c9eca9a) +com.fasterxml.jackson.core:jackson-annotations:2.10.2 (7 constraints: 27711495) +com.fasterxml.jackson.core:jackson-core:2.10.2 (8 constraints: a16ee0bc) +com.fasterxml.jackson.core:jackson-databind:2.10.2 (13 constraints: 62bac5bf) com.fasterxml.jackson.module:jackson-module-paranamer:2.10.2 (1 constraints: 03162c16) com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.2 (1 constraints: 7f0da251) com.github.ben-manes.caffeine:caffeine:2.7.0 (1 constraints: 0b050a36) com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 (1 constraints: e90b08f3) com.github.luben:zstd-jni:1.3.2-2 (1 constraints: 760d7c51) -com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (1 constraints: 6d05ab40) -com.google.code.findbugs:jsr305:3.0.2 (7 constraints: fc5db58f) -com.google.code.gson:gson:2.2.4 (2 constraints: 9518bfd2) +com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (10 constraints: 078609f3) +com.google.code.findbugs:jsr305:3.0.2 (15 constraints: f4c0e31b) +com.google.code.gson:gson:2.2.4 (6 constraints: f44c3ddb) com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544) com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) -com.google.guava:guava:28.0-jre (21 constraints: 88453dad) +com.google.guava:guava:28.0-jre (41 constraints: 0c51e1d2) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) -com.google.inject:guice:3.0 (6 constraints: 6873914c) -com.google.inject.extensions:guice-servlet:3.0 (11 constraints: a9d50a2b) +com.google.inject:guice:3.0 (8 constraints: 2c93366d) +com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce) +com.google.inject.extensions:guice-servlet:3.0 (12 constraints: 01e2ea30) com.google.j2objc:j2objc-annotations:1.3 (1 constraints: b809eda0) -com.google.protobuf:protobuf-java:2.5.0 (16 constraints: 2f1c978f) +com.google.protobuf:protobuf-java:2.5.0 (19 constraints: dc42d583) com.googlecode.javaewah:JavaEWAH:0.3.2 (1 constraints: ea0dfc42) com.jamesmurty.utils:java-xmlbuilder:0.4 (1 constraints: e40aa5ca) com.jcraft:jsch:0.1.42 (1 constraints: bb0ded3c) com.jolbox:bonecp:0.8.0.RELEASE (2 constraints: b22109f9) com.ning:compress-lzf:1.0.3 (1 constraints: 150dba36) -com.sun.jersey:jersey-client:1.9 (4 constraints: 65529ed7) -com.sun.jersey:jersey-core:1.9 (9 constraints: ec8f4404) -com.sun.jersey:jersey-json:1.9 (5 constraints: 945f2f90) -com.sun.jersey:jersey-server:1.9 (4 constraints: ef373c01) +com.sun.jersey:jersey-client:1.9 (6 constraints: 546829c2) +com.sun.jersey:jersey-core:1.9 (10 constraints: 399cd337) +com.sun.jersey:jersey-json:1.9 (7 constraints: 8375d6d3) +com.sun.jersey:jersey-server:1.9 (6 constraints: 9b50d765) com.sun.jersey.contribs:jersey-guice:1.9 (4 constraints: 65529ed7) com.sun.xml.bind:jaxb-impl:2.2.3-1 (1 constraints: 330c2404) +com.tdunning:json:1.8 (2 constraints: 051968bf) com.thoughtworks.paranamer:paranamer:2.8 (3 constraints: 742d4cb1) com.twitter:chill-java:0.9.3 (2 constraints: a716716f) com.twitter:chill_2.11:0.9.3 (2 constraints: 121b92c3) com.twitter:parquet-hadoop-bundle:1.6.0 (2 constraints: 061b4d93) com.univocity:univocity-parsers:2.7.3 (1 constraints: c40ccb27) +com.zaxxer:HikariCP:2.5.1 (1 constraints: 390d7120) commons-beanutils:commons-beanutils:1.7.0 (1 constraints: da0e635f) commons-beanutils:commons-beanutils-core:1.8.0 (1 constraints: 1d134124) -commons-cli:commons-cli:1.2 (9 constraints: f874366c) -commons-codec:commons-codec:1.10 (17 constraints: a8de3870) -commons-collections:commons-collections:3.2.2 (3 constraints: e73a8e36) +commons-cli:commons-cli:1.2 (13 constraints: e4a04168) +commons-codec:commons-codec:1.10 (24 constraints: d0347e53) +commons-collections:commons-collections:3.2.2 (6 constraints: 8e604cdc) commons-configuration:commons-configuration:1.6 (1 constraints: 2d0d5c14) commons-daemon:commons-daemon:1.0.13 (1 constraints: d50c811c) commons-dbcp:commons-dbcp:1.4 (3 constraints: 9029e0e4) commons-digester:commons-digester:1.8 (1 constraints: bf1228fe) commons-el:commons-el:1.0 (2 constraints: ad11e7f0) -commons-httpclient:commons-httpclient:3.1 (4 constraints: e52cc77f) -commons-io:commons-io:2.4 (6 constraints: 4a568049) -commons-lang:commons-lang:2.6 (20 constraints: 401f63f3) -commons-logging:commons-logging:1.2 (20 constraints: 424b646e) +commons-httpclient:commons-httpclient:3.1 (6 constraints: 8545051d) +commons-io:commons-io:2.4 (11 constraints: e6902ece) +commons-lang:commons-lang:2.6 (33 constraints: 12c29bcf) +commons-logging:commons-logging:1.2 (31 constraints: aedcdd2a) commons-net:commons-net:3.1 (3 constraints: 3d222e61) commons-pool:commons-pool:1.6 (4 constraints: e336ab5e) dk.brics.automaton:automaton:1.11-8 (1 constraints: 92088a8d) hsqldb:hsqldb:1.8.0.10 (1 constraints: f008499f) io.airlift:aircompressor:0.15 (1 constraints: 0e0aa4b2) -io.dropwizard.metrics:metrics-core:3.1.5 (8 constraints: 3b8585b8) +io.dropwizard.metrics:metrics-core:3.1.5 (9 constraints: be90b1ee) io.dropwizard.metrics:metrics-graphite:3.1.5 (1 constraints: 1a0dc936) io.dropwizard.metrics:metrics-json:3.1.5 (2 constraints: 03195c12) io.dropwizard.metrics:metrics-jvm:3.1.5 (2 constraints: 03195c12) -io.netty:netty:3.9.9.Final (9 constraints: 9eb0396d) -io.netty:netty-all:4.1.17.Final (3 constraints: d2312526) +io.netty:netty:3.9.9.Final (11 constraints: 51cef3a2) +io.netty:netty-all:4.1.17.Final (6 constraints: 646011b6) io.netty:netty-buffer:4.1.27.Final (1 constraints: 4a0fee77) -javax.activation:activation:1.1.1 (1 constraints: 140dbb36) +it.unimi.dsi:fastutil:6.5.6 (1 constraints: 910b3ce5) +javax.activation:activation:1.1.1 (3 constraints: b02331a0) javax.annotation:javax.annotation-api:1.3.2 (3 constraints: 55341c48) javax.inject:javax.inject:1 (4 constraints: 852d0c1a) javax.jdo:jdo-api:3.0.1 (2 constraints: 4c1dcc1a) +javax.mail:mail:1.4.1 (1 constraints: fc0fe399) javax.servlet:javax.servlet-api:3.1.0 (1 constraints: 150dc436) javax.servlet:jsp-api:2.0 (1 constraints: 0b0aa0a7) -javax.servlet:servlet-api:2.5 (9 constraints: f991a6d2) -javax.servlet.jsp:jsp-api:2.1 (1 constraints: 290d5a14) +javax.servlet:servlet-api:2.5 (12 constraints: 72b75a2c) +javax.servlet.jsp:jsp-api:2.1 (2 constraints: 811985e6) javax.transaction:jta:1.1 (1 constraints: 9f07d96b) +javax.transaction:transaction-api:1.1 (1 constraints: 0a0b64c9) javax.validation:validation-api:1.1.0.Final (1 constraints: 13133130) javax.ws.rs:javax.ws.rs-api:2.0.1 (5 constraints: 6e649355) javax.xml.bind:jaxb-api:2.2.11 (6 constraints: a069fd48) javolution:javolution:5.5.1 (2 constraints: 2b1b2b82) -jline:jline:2.12 (3 constraints: 7c21b2cb) +jline:jline:2.12 (4 constraints: e72bc09c) joda-time:joda-time:2.9.9 (4 constraints: 2326d336) +junit:junit:4.12 (10 constraints: 4a8036b6) log4j:apache-log4j-extras:1.2.17 (1 constraints: 200e1d51) -log4j:log4j:1.2.17 (8 constraints: e7772b11) +log4j:log4j:1.2.17 (17 constraints: fbf6ff1d) net.hydromatic:eigenbase-properties:1.1.5 (1 constraints: 5f0daf2c) net.java.dev.jets3t:jets3t:0.9.0 (2 constraints: ec152b22) net.razorvine:pyrolite:4.13 (1 constraints: eb0cb829) @@ -99,56 +113,68 @@ org.apache.ant:ant-launcher:1.9.1 (1 constraints: 69082485) org.apache.arrow:arrow-format:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-memory:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-vector:0.14.1 (2 constraints: 2012a545) -org.apache.avro:avro:1.9.2 (4 constraints: 3e2e68f4) +org.apache.avro:avro:1.9.2 (8 constraints: 71686591) org.apache.avro:avro-ipc:1.8.2 (1 constraints: f90b5bf4) org.apache.avro:avro-mapred:1.8.2 (2 constraints: 3a1a4787) org.apache.calcite:calcite-avatica:1.2.0-incubating (3 constraints: 4b35b263) org.apache.calcite:calcite-core:1.2.0-incubating (1 constraints: 68119fdf) org.apache.calcite:calcite-linq4j:1.2.0-incubating (1 constraints: ac1147d8) -org.apache.commons:commons-compress:1.19 (6 constraints: 464a0c7f) +org.apache.commons:commons-compress:1.19 (7 constraints: ff569390) org.apache.commons:commons-crypto:1.0.0 (2 constraints: 3a1e5fbf) -org.apache.commons:commons-lang3:3.9 (5 constraints: 503b94b4) -org.apache.commons:commons-math3:3.4.1 (2 constraints: a11af290) -org.apache.curator:curator-client:2.7.1 (2 constraints: 6a1d2734) -org.apache.curator:curator-framework:2.7.1 (4 constraints: 4d37382d) -org.apache.curator:curator-recipes:2.7.1 (2 constraints: a61acc91) +org.apache.commons:commons-lang3:3.9 (8 constraints: 3362d239) +org.apache.commons:commons-math3:3.4.1 (3 constraints: 7c24247c) +org.apache.curator:curator-client:2.7.1 (3 constraints: 272ac6a3) +org.apache.curator:curator-framework:2.7.1 (9 constraints: 797966d8) +org.apache.curator:curator-recipes:2.7.1 (4 constraints: ba337377) org.apache.derby:derby:10.12.1.1 (3 constraints: 9f2cb182) org.apache.directory.api:api-asn1-api:1.0.0-M20 (1 constraints: 3d163b13) org.apache.directory.api:api-util:1.0.0-M20 (1 constraints: 3d163b13) org.apache.directory.server:apacheds-i18n:2.0.0-M15 (1 constraints: 42164713) org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15 (1 constraints: 8f0d3b45) -org.apache.hadoop:hadoop-annotations:2.7.3 (16 constraints: 2c27b38c) -org.apache.hadoop:hadoop-auth:2.7.3 (1 constraints: 900d4d2f) -org.apache.hadoop:hadoop-client:2.7.3 (2 constraints: 2b12043c) -org.apache.hadoop:hadoop-common:2.7.3 (4 constraints: 163dee6b) -org.apache.hadoop:hadoop-hdfs:2.7.3 (4 constraints: b834c025) +org.apache.geronimo.specs:geronimo-annotation_1.0_spec:1.1.1 (1 constraints: f90fda99) +org.apache.geronimo.specs:geronimo-jaspic_1.0_spec:1.0 (1 constraints: 990f187a) +org.apache.geronimo.specs:geronimo-jta_1.1_spec:1.1.1 (1 constraints: f90fda99) +org.apache.hadoop:hadoop-annotations:2.7.3 (23 constraints: 1b8376ec) +org.apache.hadoop:hadoop-auth:2.7.3 (5 constraints: 903fb325) +org.apache.hadoop:hadoop-client:2.7.3 (4 constraints: 912bfbce) +org.apache.hadoop:hadoop-common:2.7.3 (20 constraints: ca218990) +org.apache.hadoop:hadoop-hdfs:2.7.3 (8 constraints: fc69f814) org.apache.hadoop:hadoop-mapreduce-client-app:2.7.3 (3 constraints: ab2f8436) -org.apache.hadoop:hadoop-mapreduce-client-common:2.7.3 (4 constraints: 184f4f66) -org.apache.hadoop:hadoop-mapreduce-client-core:2.7.3 (4 constraints: 66361812) +org.apache.hadoop:hadoop-mapreduce-client-common:2.7.3 (5 constraints: 815ba1d8) +org.apache.hadoop:hadoop-mapreduce-client-core:2.7.3 (11 constraints: 369ec41b) org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.7.3 (2 constraints: 3b1dfa13) org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.7.3 (2 constraints: 2628c449) -org.apache.hadoop:hadoop-yarn-api:2.7.3 (10 constraints: 07b8bd4c) -org.apache.hadoop:hadoop-yarn-client:2.7.3 (1 constraints: 1f14626e) -org.apache.hadoop:hadoop-yarn-common:2.7.3 (9 constraints: b3b2f06f) +org.apache.hadoop:hadoop-yarn-api:2.7.3 (18 constraints: 061eefa9) +org.apache.hadoop:hadoop-yarn-client:2.7.3 (4 constraints: 543421ae) +org.apache.hadoop:hadoop-yarn-common:2.7.3 (16 constraints: 260de1f1) org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice:2.7.3 (1 constraints: f5157dcf) org.apache.hadoop:hadoop-yarn-server-common:2.7.3 (7 constraints: 5192cac0) org.apache.hadoop:hadoop-yarn-server-nodemanager:2.7.3 (2 constraints: 6726468d) org.apache.hadoop:hadoop-yarn-server-resourcemanager:2.7.3 (2 constraints: b22045af) -org.apache.hadoop:hadoop-yarn-server-web-proxy:2.7.3 (2 constraints: cb287679) -org.apache.hive:hive-common:2.3.6 (1 constraints: 7b0bc2e4) -org.apache.hive:hive-metastore:2.3.6 (1 constraints: 0d050436) -org.apache.hive:hive-serde:2.3.6 (1 constraints: 3c0d7020) -org.apache.hive:hive-service-rpc:2.3.6 (1 constraints: 7b0bc2e4) -org.apache.hive:hive-shims:2.3.6 (4 constraints: b22fbcd9) +org.apache.hadoop:hadoop-yarn-server-web-proxy:2.7.3 (3 constraints: aa32e81b) +org.apache.hbase:hbase-annotations:1.1.1 (4 constraints: 0d36c989) +org.apache.hbase:hbase-client:1.1.1 (3 constraints: a427c078) +org.apache.hbase:hbase-common:1.1.1 (5 constraints: 4b433401) +org.apache.hbase:hbase-protocol:1.1.1 (4 constraints: 9d335412) +org.apache.hive:hive-common:2.3.6 (8 constraints: eb682401) +org.apache.hive:hive-metastore:2.3.6 (5 constraints: ef389a58) +org.apache.hive:hive-serde:2.3.6 (7 constraints: 834d1ecf) +org.apache.hive:hive-service-rpc:2.3.6 (3 constraints: cd22ff7b) +org.apache.hive:hive-shims:2.3.6 (7 constraints: 1f53d276) org.apache.hive:hive-storage-api:2.4.0 (1 constraints: ec0b19f3) org.apache.hive.shims:hive-shims-0.23:2.3.6 (1 constraints: 8c0b6ce5) org.apache.hive.shims:hive-shims-common:2.3.6 (3 constraints: 222cfaad) org.apache.hive.shims:hive-shims-scheduler:2.3.6 (1 constraints: 8c0b6ce5) -org.apache.htrace:htrace-core:3.1.0-incubating (2 constraints: cd22cffa) -org.apache.httpcomponents:httpclient:4.5.6 (4 constraints: 573134dd) -org.apache.httpcomponents:httpcore:4.4.10 (3 constraints: d327f763) +org.apache.htrace:htrace-core:3.1.0-incubating (5 constraints: 89553ebc) +org.apache.httpcomponents:httpclient:4.5.6 (7 constraints: 455794a4) +org.apache.httpcomponents:httpcore:4.4.10 (6 constraints: c24d4f67) org.apache.ivy:ivy:2.4.0 (3 constraints: 0826dbf1) -org.apache.orc:orc-core:1.6.2 (3 constraints: ba1d17ad) +org.apache.logging.log4j:log4j-1.2-api:2.6.2 (2 constraints: fb160260) +org.apache.logging.log4j:log4j-api:2.6.2 (4 constraints: 2e3c9f23) +org.apache.logging.log4j:log4j-core:2.6.2 (2 constraints: fd1c2464) +org.apache.logging.log4j:log4j-slf4j-impl:2.6.2 (3 constraints: 8d27cb64) +org.apache.logging.log4j:log4j-web:2.6.2 (1 constraints: f00b21f3) +org.apache.orc:orc-core:1.6.2 (4 constraints: 8b2becc9) org.apache.orc:orc-mapreduce:1.5.5 (1 constraints: c30cc227) org.apache.orc:orc-shims:1.6.2 (1 constraints: 3f0aeabc) org.apache.parquet:parquet-avro:1.11.0 (1 constraints: 35052c3b) @@ -157,6 +183,7 @@ org.apache.parquet:parquet-common:1.11.0 (2 constraints: 4c1e7785) org.apache.parquet:parquet-encoding:1.11.0 (1 constraints: ca0efb64) org.apache.parquet:parquet-format-structures:1.11.0 (3 constraints: 6e2bc836) org.apache.parquet:parquet-hadoop:1.11.0 (2 constraints: de1ac7b3) +org.apache.parquet:parquet-hadoop-bundle:1.8.1 (1 constraints: 7a0bc7e4) org.apache.parquet:parquet-jackson:1.11.0 (1 constraints: b70ee963) org.apache.pig:pig:0.14.0 (1 constraints: 37052f3b) org.apache.spark:spark-avro_2.11:2.4.4 (1 constraints: 0c050536) @@ -171,18 +198,24 @@ org.apache.spark:spark-sketch_2.11:2.4.4 (2 constraints: 981bd4f5) org.apache.spark:spark-sql_2.11:2.4.4 (1 constraints: 1e0d0037) org.apache.spark:spark-tags_2.11:2.4.4 (8 constraints: 036fa69d) org.apache.spark:spark-unsafe_2.11:2.4.4 (2 constraints: f11bc213) -org.apache.thrift:libfb303:0.9.3 (3 constraints: 27289d07) -org.apache.thrift:libthrift:0.9.3 (6 constraints: 3f4f0635) +org.apache.thrift:libfb303:0.9.3 (4 constraints: 8034ebe7) +org.apache.thrift:libthrift:0.9.3 (11 constraints: 908a19e4) +org.apache.twill:twill-api:0.6.0-incubating (2 constraints: e422b4e1) +org.apache.twill:twill-common:0.6.0-incubating (4 constraints: 3d467991) +org.apache.twill:twill-core:0.6.0-incubating (1 constraints: d70f9d7c) +org.apache.twill:twill-discovery-api:0.6.0-incubating (3 constraints: 25345d4c) +org.apache.twill:twill-discovery-core:0.6.0-incubating (2 constraints: 332039f9) +org.apache.twill:twill-zookeeper:0.6.0-incubating (3 constraints: 94341288) org.apache.xbean:xbean-asm6-shaded:4.8 (2 constraints: 2419a30f) org.apache.yetus:audience-annotations:0.11.0 (1 constraints: c40eb364) -org.apache.zookeeper:zookeeper:3.4.6 (11 constraints: 18a71f48) +org.apache.zookeeper:zookeeper:3.4.6 (17 constraints: 12f4f17d) org.checkerframework:checker-qual:2.8.1 (2 constraints: 1a1a3944) -org.codehaus.jackson:jackson-core-asl:1.9.13 (11 constraints: 8091cd06) -org.codehaus.jackson:jackson-jaxrs:1.9.13 (2 constraints: 821bca9d) -org.codehaus.jackson:jackson-mapper-asl:1.9.13 (11 constraints: e18d325f) -org.codehaus.jackson:jackson-xc:1.9.13 (2 constraints: 821bca9d) -org.codehaus.janino:commons-compiler:3.0.9 (2 constraints: a41a546f) -org.codehaus.janino:janino:3.0.9 (1 constraints: d90e817c) +org.codehaus.jackson:jackson-core-asl:1.9.13 (14 constraints: e8bb1763) +org.codehaus.jackson:jackson-jaxrs:1.9.13 (4 constraints: 5235d62f) +org.codehaus.jackson:jackson-mapper-asl:1.9.13 (17 constraints: c3eee844) +org.codehaus.jackson:jackson-xc:1.9.13 (3 constraints: 73286ded) +org.codehaus.janino:commons-compiler:3.0.9 (3 constraints: 0a2837cc) +org.codehaus.janino:janino:3.0.9 (2 constraints: 3f1c6304) org.codehaus.jettison:jettison:1.1 (4 constraints: a84e24a9) org.codehaus.mojo:animal-sniffer-annotations:1.17 (1 constraints: ed09d8aa) org.datanucleus:datanucleus-api-jdo:4.2.4 (2 constraints: 591df91b) @@ -190,6 +223,8 @@ org.datanucleus:datanucleus-core:4.1.17 (5 constraints: 584455e8) org.datanucleus:datanucleus-rdbms:4.1.19 (2 constraints: 911dec32) org.datanucleus:javax.jdo:3.2.0-m3 (1 constraints: 030ea249) org.eclipse.jdt:core:3.1.1 (1 constraints: b40a38d8) +org.eclipse.jetty.aggregate:jetty-all:7.6.0.v20120127 (2 constraints: b31cf79a) +org.eclipse.jetty.orbit:javax.servlet:3.0.0.v201112011016 (1 constraints: dd0e53b1) org.fusesource.leveldbjni:leveldbjni-all:1.8 (9 constraints: 91a69ae7) org.glassfish.hk2:hk2-api:2.4.0-b34 (5 constraints: 9d5608c7) org.glassfish.hk2:hk2-locator:2.4.0-b34 (4 constraints: 3d490865) @@ -204,23 +239,28 @@ org.glassfish.jersey.core:jersey-client:2.22.2 (2 constraints: 791ef7a3) org.glassfish.jersey.core:jersey-common:2.22.2 (6 constraints: 5f747f50) org.glassfish.jersey.core:jersey-server:2.22.2 (3 constraints: 553f5d56) org.glassfish.jersey.media:jersey-media-jaxb:2.22.2 (1 constraints: 3111f1d4) +org.hamcrest:hamcrest-core:1.3 (2 constraints: 7910aeb0) org.iq80.snappy:snappy:0.2 (1 constraints: 890d5927) org.javassist:javassist:3.18.1-GA (1 constraints: 570d4740) org.jetbrains:annotations:17.0.0 (1 constraints: 6e0a64c7) org.jodd:jodd-core:3.5.2 (2 constraints: 0c1bda93) +org.jruby.jcodings:jcodings:1.0.8 (2 constraints: 9d15c301) +org.jruby.joni:joni:2.1.2 (1 constraints: 8f0c160d) org.json4s:json4s-ast_2.11:3.5.3 (1 constraints: 0c0b9ae9) org.json4s:json4s-core_2.11:3.5.3 (1 constraints: 4c0c5316) org.json4s:json4s-jackson_2.11:3.5.3 (1 constraints: 1c0dd336) org.json4s:json4s-scalap_2.11:3.5.3 (1 constraints: 0c0b9ae9) org.lz4:lz4-java:1.4.0 (1 constraints: 160dc336) -org.mortbay.jetty:jetty:6.1.26 (4 constraints: c8369437) -org.mortbay.jetty:jetty-util:6.1.26 (7 constraints: 7e689dae) -org.mortbay.jetty:jsp-2.1:6.1.14 (1 constraints: 9408a38d) -org.mortbay.jetty:jsp-api-2.1:6.1.14 (2 constraints: 7e130c9d) +org.mortbay.jetty:jetty:6.1.26 (8 constraints: b66925ab) +org.mortbay.jetty:jetty-util:6.1.26 (11 constraints: 3799d04e) +org.mortbay.jetty:jsp-2.1:6.1.14 (2 constraints: 71154c11) +org.mortbay.jetty:jsp-api-2.1:6.1.14 (3 constraints: 5b20fbe2) org.mortbay.jetty:servlet-api:2.5-20081211 (1 constraints: 390cbd19) -org.mortbay.jetty:servlet-api-2.5:6.1.14 (2 constraints: e51482f7) +org.mortbay.jetty:servlet-api-2.5:6.1.14 (3 constraints: c221f470) org.objenesis:objenesis:2.5.1 (2 constraints: 19198bcb) -org.roaringbitmap:RoaringBitmap:0.7.45 (1 constraints: 510d1c44) +org.ow2.asm:asm-all:5.0.2 (1 constraints: 0d0ceaf6) +org.pentaho:pentaho-aggdesigner-algorithm:5.1.5-jhyde (1 constraints: a40f6d84) +org.roaringbitmap:RoaringBitmap:0.7.45 (2 constraints: 2e1c26e3) org.roaringbitmap:shims:0.7.45 (1 constraints: 260eb249) org.scala-lang:scala-library:2.11.12 (11 constraints: 5c9bfe44) org.scala-lang:scala-reflect:2.11.12 (1 constraints: 340fb09a) @@ -228,30 +268,77 @@ org.scala-lang.modules:scala-parser-combinators_2.11:1.1.0 (1 constraints: cf0e7 org.scala-lang.modules:scala-xml_2.11:1.0.6 (1 constraints: 080b84e9) org.slf4j:jcl-over-slf4j:1.7.16 (1 constraints: 500d1d44) org.slf4j:jul-to-slf4j:1.7.16 (1 constraints: 500d1d44) -org.slf4j:slf4j-api:1.7.25 (49 constraints: f1d591ce) +org.slf4j:slf4j-api:1.7.25 (76 constraints: 544996f4) org.sonatype.sisu.inject:cglib:2.2.1-v20090111 (1 constraints: aa0cfd36) org.spark-project.hive:hive-exec:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.hive:hive-metastore:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.spark:unused:1.0.0 (12 constraints: 9aab75cf) org.xerial.snappy:snappy-java:1.1.7.3 (2 constraints: 681c5e46) oro:oro:2.0.8 (3 constraints: 3b229337) -stax:stax-api:1.0.1 (2 constraints: ea186edd) -tomcat:jasper-compiler:5.5.23 (2 constraints: 93169c60) -tomcat:jasper-runtime:5.5.23 (2 constraints: 93169c60) +stax:stax-api:1.0.1 (3 constraints: 8d2668e5) +tomcat:jasper-compiler:5.5.23 (4 constraints: ff2fc367) +tomcat:jasper-runtime:5.5.23 (4 constraints: ff2fc367) xerces:xercesImpl:2.9.1 (1 constraints: ac0ccc0f) -xml-apis:xml-apis:1.3.04 (1 constraints: b008af8c) +xml-apis:xml-apis:1.3.04 (2 constraints: a20e0877) xmlenc:xmlenc:0.52 (3 constraints: 05228b2f) [Test dependencies] -junit:junit:4.12 (1 constraints: db04ff30) -org.apache.curator:apache-curator:2.7.1 (1 constraints: 0c0bf8d6) +com.beust:jcommander:1.30 (1 constraints: 8a0c0505) +com.klarna:hiverunner:5.1.1 (1 constraints: 09050836) +com.lmax:disruptor:3.3.0 (1 constraints: a80c770e) +com.ning:async-http-client:1.8.16 (1 constraints: 110f9671) +com.yammer.metrics:metrics-core:2.2.0 (2 constraints: 121c6ce2) +dom4j:dom4j:1.6.1 (1 constraints: 8c0ceb00) +jakarta.jms:jakarta.jms-api:2.0.2 (1 constraints: 5617da29) +javassist:javassist:3.12.1.GA (1 constraints: 710d3035) +net.sf.jpam:jpam:1.1 (1 constraints: f20b40e9) +org.apache.calcite.avatica:avatica:1.8.0 (1 constraints: 0b0bf5d6) +org.apache.calcite.avatica:avatica-metrics:1.8.0 (1 constraints: 960e635d) +org.apache.commons:commons-collections4:4.1 (2 constraints: 09137a51) +org.apache.commons:commons-math:2.2 (3 constraints: 322af180) +org.apache.curator:apache-curator:2.7.1 (2 constraints: c718e2d6) +org.apache.hadoop:hadoop-archives:2.7.3 (1 constraints: f21198ff) org.apache.hadoop:hadoop-mapreduce-client-hs:2.7.3 (1 constraints: b60fac84) org.apache.hadoop:hadoop-minicluster:2.7.3 (1 constraints: 0e050d36) +org.apache.hadoop:hadoop-yarn-registry:2.7.3 (1 constraints: be0ccd11) org.apache.hadoop:hadoop-yarn-server-tests:2.7.3 (1 constraints: b60fac84) -org.apache.hive:hive-exec:2.3.6 (1 constraints: 0d050436) +org.apache.hbase:hbase-hadoop-compat:1.1.1 (4 constraints: 5438a206) +org.apache.hbase:hbase-hadoop2-compat:1.1.1 (3 constraints: e9285c23) +org.apache.hbase:hbase-prefix-tree:1.1.1 (1 constraints: a50c680e) +org.apache.hbase:hbase-procedure:1.1.1 (1 constraints: a50c680e) +org.apache.hbase:hbase-server:1.1.1 (1 constraints: cd0d4a3c) +org.apache.hive:hive-cli:2.3.6 (1 constraints: f21190ff) +org.apache.hive:hive-exec:2.3.6 (6 constraints: fa4f2433) +org.apache.hive:hive-jdbc:2.3.6 (1 constraints: 090a76ad) +org.apache.hive:hive-llap-client:2.3.6 (2 constraints: 651aa157) +org.apache.hive:hive-llap-common:2.3.6 (2 constraints: 911b96aa) +org.apache.hive:hive-llap-server:2.3.6 (1 constraints: 590cd001) +org.apache.hive:hive-llap-tez:2.3.6 (2 constraints: e11855d8) +org.apache.hive:hive-service:2.3.6 (3 constraints: a21fbe56) org.apache.hive:hive-vector-code-gen:2.3.6 (1 constraints: 0d0bf1d6) +org.apache.hive.hcatalog:hive-hcatalog-core:2.3.6 (2 constraints: 8e2bb3f1) +org.apache.hive.hcatalog:hive-hcatalog-server-extensions:2.3.6 (1 constraints: 3214a97c) +org.apache.hive.hcatalog:hive-webhcat-java-client:2.3.6 (1 constraints: 090a76ad) +org.apache.slider:slider-core:0.90.2-incubating (1 constraints: 56124bfd) +org.apache.tez:hadoop-shim:0.9.1 (3 constraints: d3244224) +org.apache.tez:tez-api:0.9.1 (5 constraints: 1740d50b) +org.apache.tez:tez-common:0.9.1 (5 constraints: e13ec822) +org.apache.tez:tez-dag:0.9.1 (1 constraints: 080a79ad) +org.apache.tez:tez-mapreduce:0.9.1 (1 constraints: 080a79ad) +org.apache.tez:tez-runtime-internals:0.9.1 (1 constraints: e1099fb1) +org.apache.tez:tez-runtime-library:0.9.1 (2 constraints: 4b167422) org.apache.velocity:velocity:1.5 (1 constraints: c70e875e) +org.apiguardian:apiguardian-api:1.1.0 (5 constraints: 0654a8a8) org.codehaus.groovy:groovy-all:2.4.4 (1 constraints: 0c0bf2d6) -org.hamcrest:hamcrest-core:1.3 (2 constraints: 7910aeb0) +org.jamon:jamon-runtime:2.3.1 (2 constraints: fb1870e4) +org.junit.jupiter:junit-jupiter:5.6.0 (1 constraints: 090a88ad) +org.junit.jupiter:junit-jupiter-api:5.6.0 (3 constraints: 6a2f2bdb) +org.junit.jupiter:junit-jupiter-engine:5.6.0 (1 constraints: 080ed73b) +org.junit.jupiter:junit-jupiter-params:5.6.0 (1 constraints: 080ed73b) +org.junit.platform:junit-platform-commons:1.6.0 (2 constraints: d520374a) +org.junit.platform:junit-platform-engine:1.6.0 (1 constraints: a7101fb4) org.mockito:mockito-core:1.10.19 (1 constraints: 6e059840) +org.mortbay.jetty:jetty-sslengine:6.1.26 (1 constraints: e10c631b) +org.opentest4j:opentest4j:1.2.0 (2 constraints: cd205b49) +org.reflections:reflections:0.9.8 (1 constraints: 0f0a80ad) org.slf4j:slf4j-simple:1.7.5 (1 constraints: 0f050a36) From d7dfb0e14ebbca8a0e1a07349cdd78e941cda8fb Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Wed, 25 Mar 2020 15:15:37 +0000 Subject: [PATCH 10/51] Fix checkstyle issues --- build.gradle | 1 + .../apache/iceberg/mr/IcebergInputFormat.java | 3 -- .../IcebergObjectInspectorGenerator.java | 37 ++++++++++--------- .../mr/mapred/IcebergSchemaToTypeInfo.java | 25 ++++++------- .../org/iceberg/mr/mapred/IcebergSerDe.java | 34 +++++++++-------- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/build.gradle b/build.gradle index 2d8b66c80996..10b17e34c768 100644 --- a/build.gradle +++ b/build.gradle @@ -698,6 +698,7 @@ project(':iceberg-pig') { testCompile("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' } + testCompile 'junit:junit' } } diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java index d1424cde4fa3..637e14b8c5c8 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java @@ -53,8 +53,6 @@ import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataReader; @@ -62,7 +60,6 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.hadoop.HadoopInputFile; import org.apache.iceberg.hadoop.HadoopTables; -//import org.apache.iceberg.hive.HiveCatalogs; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.orc.ORC; diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java index 750e3498effb..ab2b84224580 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java @@ -1,20 +1,26 @@ -/** - * Copyright (C) 2020 Expedia, Inc. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ + package org.iceberg.mr.mapred; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; @@ -27,9 +33,6 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; -import java.util.ArrayList; -import java.util.List; - class IcebergObjectInspectorGenerator { protected ObjectInspector createObjectInspector(Schema schema) throws Exception { @@ -37,7 +40,7 @@ protected ObjectInspector createObjectInspector(Schema schema) throws Exception List columnTypes = IcebergSchemaToTypeInfo.getColumnTypes(schema); List columnOIs = new ArrayList<>(columnTypes.size()); - for(int i = 0; i < columnTypes.size(); i++) { + for (int i = 0; i < columnTypes.size(); i++) { columnOIs.add(createObjectInspectorWorker(columnTypes.get(i))); } return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs, null); @@ -46,7 +49,7 @@ protected ObjectInspector createObjectInspector(Schema schema) throws Exception protected ObjectInspector createObjectInspectorWorker(TypeInfo typeInfo) throws Exception { ObjectInspector.Category typeCategory = typeInfo.getCategory(); - switch(typeCategory) { + switch (typeCategory) { case PRIMITIVE: PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo; return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti); diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java index d8ca94b4060d..23378f536fc5 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -16,9 +16,12 @@ * specific language governing permissions and limitations * under the License. */ + package org.iceberg.mr.mapred; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; @@ -28,17 +31,12 @@ import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; -import java.util.ArrayList; -import java.util.List; - /** * Class to convert Iceberg types to Hive TypeInfo */ final class IcebergSchemaToTypeInfo { - private IcebergSchemaToTypeInfo() { - - } + private IcebergSchemaToTypeInfo() {} private static final ImmutableMap primitiveTypeToTypeInfo = ImmutableMap.builder() .put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME)) @@ -49,12 +47,13 @@ private IcebergSchemaToTypeInfo() { .put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME)) .put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)) .put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME)) - .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)).build(); + .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)) + .build(); public static List getColumnTypes(Schema schema) throws Exception { List fields = schema.columns(); List types = new ArrayList<>(fields.size()); - for (Types.NestedField field: fields) { + for (Types.NestedField field : fields) { types.add(generateTypeInfo(field.type())); } return types; @@ -64,7 +63,7 @@ private static TypeInfo generateTypeInfo(Type type) throws Exception { if (primitiveTypeToTypeInfo.containsKey(type)) { return (TypeInfo) primitiveTypeToTypeInfo.get(type); } - switch(type.typeId()) { + switch (type.typeId()) { case UUID: return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); case FIXED: @@ -72,7 +71,7 @@ private static TypeInfo generateTypeInfo(Type type) throws Exception { case TIME: return TypeInfoFactory.getPrimitiveTypeInfo("long"); case DECIMAL: - Types.DecimalType dec = (Types.DecimalType)type; + Types.DecimalType dec = (Types.DecimalType) type; int scale = dec.scale(); int precision = dec.precision(); try { @@ -83,11 +82,11 @@ private static TypeInfo generateTypeInfo(Type type) throws Exception { } return TypeInfoFactory.getDecimalTypeInfo(precision, scale); case STRUCT: - return generateStructTypeInfo((Types.StructType)type); + return generateStructTypeInfo((Types.StructType) type); case LIST: - return generateListTypeInfo((Types.ListType)type); + return generateListTypeInfo((Types.ListType) type); case MAP: - return generateMapTypeInfo((Types.MapType)type); + return generateMapTypeInfo((Types.MapType) type); default: throw new SerDeException("Can't map Iceberg type to Hive TypeInfo: '" + type.typeId() + "'"); } diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java index 888e16348c60..e78a18b46d53 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java @@ -1,18 +1,22 @@ -/** - * Copyright (C) 2020 Expedia, Inc. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ + package org.iceberg.mr.mapred; import java.util.ArrayList; @@ -73,11 +77,9 @@ public SerDeStats getSerDeStats() { @Override public Object deserialize(Writable writable) throws SerDeException { IcebergWritable icebergWritable = (IcebergWritable) writable; - Schema schema = icebergWritable.getSchema(); - List fields = schema.columns(); + List fields = icebergWritable.getSchema().columns(); List row = new ArrayList<>(); - - for(Types.NestedField field: fields){ + for (Types.NestedField field : fields) { Object obj = ((IcebergWritable) writable).getRecord().getField(field.name()); row.add(obj); } From 07307ec57b4f04d6b5adb858ccbd35de2484b21d Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 26 Mar 2020 18:04:30 +0000 Subject: [PATCH 11/51] backing up the crazy - need to find out how to get hive-exec:core back on CP --- baseline.gradle | 4 +- build.gradle | 57 +++++++++-- .../org/apache/iceberg/avro/TypeToSchema.java | 2 +- .../mr/mapred/TestIcebergInputFormat.java | 31 +++--- versions.lock | 96 ++++++++----------- versions.props | 1 - 6 files changed, 111 insertions(+), 80 deletions(-) diff --git a/baseline.gradle b/baseline.gradle index b30d6b508873..ce7bdec42ac4 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -33,8 +33,8 @@ subprojects { // ready to enforce linting on. apply plugin: 'org.inferred.processors' if (!project.hasProperty('quick')) { - apply plugin: 'com.palantir.baseline-checkstyle' - apply plugin: 'com.palantir.baseline-error-prone' + //apply plugin: 'com.palantir.baseline-checkstyle' + //apply plugin: 'com.palantir.baseline-error-prone' } apply plugin: 'com.palantir.baseline-scalastyle' apply plugin: 'com.palantir.baseline-class-uniqueness' diff --git a/build.gradle b/build.gradle index 10b17e34c768..92032d9c3907 100644 --- a/build.gradle +++ b/build.gradle @@ -60,6 +60,7 @@ subprojects { configurations { testCompile.extendsFrom compileOnly all { + exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.slf4j', module: 'slf4j-log4j12' } @@ -248,7 +249,6 @@ project(':iceberg-data') { exclude (dependency('org.apache.iceberg:iceberg-common')) exclude (dependency('org.apache.iceberg:iceberg-api')) exclude (dependency('org.apache.iceberg:iceberg-core')) - } zip64 true @@ -279,6 +279,7 @@ project(':iceberg-hive') { compileOnly "org.apache.avro:avro" compileOnly("org.apache.hive:hive-metastore") { + exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -298,7 +299,7 @@ project(':iceberg-hive') { // that's really old. We use the core classifier to be able to override our guava // version. Luckily, hive-exec seems to work okay so far with this version of guava // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context. - testCompile("org.apache.hive:hive-exec::core") { + testCompile("org.apache.hive:hive-exec:2.3.6:core") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -312,6 +313,7 @@ project(':iceberg-hive') { testCompile("org.apache.hive:hive-metastore") { exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency exclude group: 'org.apache.hbase' @@ -341,13 +343,13 @@ project(':iceberg-mr') { compile project(path: ':iceberg-orc', configuration: 'shadow') compile project(path: ':iceberg-parquet', configuration: 'shadow') compile project(path: ':iceberg-data', configuration: 'shadow') - + compileOnly "org.apache.avro:avro" compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' } compileOnly('com.github.ben-manes.caffeine:caffeine') - compileOnly ('org.apache.calcite:calcite-core') + compileOnly('org.apache.calcite:calcite-core:1.10.0') compile("org.apache.hive:hive-serde:2.3.6") { exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' @@ -362,18 +364,61 @@ project(':iceberg-mr') { exclude group: 'org.checkerframework', module: 'checker-qual' exclude group: 'org.codehaus.mojo', module: 'animal-sniffer-annotations' exclude group: 'commons-collections', module: 'commons-collections' + exclude group: 'org.apache.hive', module: 'hive-exec' } - + testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - testCompile("com.klarna:hiverunner:5.1.1") { + testCompile("org.apache.hive:hive-service:2.3.6") { + exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' + exclude group: 'org.apache.avro', module: 'avro' + } + + compile("org.apache.hive:hive-exec:2.3.6:core") { + exclude group: 'stax', module: 'stax-api' + exclude group: 'commons-collections', module: 'commons-collections' + exclude group: 'org.apache.ant', module: '*' + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.pentaho' // missing dependency + exclude group: 'org.apache.hive', module: 'hive-llap-tez' + exclude group: 'org.apache.logging.log4j' + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.calcite' + exclude group: 'org.apache.calcite.avatica' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + + testCompile 'junit:junit' + testCompile("com.klarna:hiverunner:4.1.0") { exclude group: 'com.google.protobuf', module: 'protobuf-java' exclude group: 'org.apache.calcite', module: '*' exclude group: 'org.codehaus.jettison', module: 'jettison' + exclude group: 'javax.jms', module: 'jms' + exclude group: 'org.apache.hive', module: '*' + } + testCompile("org.apache.hive:hive-exec:2.3.6:core") { + exclude group: 'stax', module: 'stax-api' + exclude group: 'commons-collections', module: 'commons-collections' + exclude group: 'org.apache.ant', module: '*' + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.pentaho' // missing dependency + exclude group: 'org.apache.hive', module: 'hive-llap-tez' + exclude group: 'org.apache.logging.log4j' + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.calcite' + exclude group: 'org.apache.calcite.avatica' + exclude group: 'com.google.code.findbugs', module: 'jsr305' } } + task copyToLib(type: Copy) { + into "$buildDir/output/lib" + from configurations.testCompile + } } project(':iceberg-orc') { diff --git a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java index 70ee42352ae8..3c9d9ea7bb24 100644 --- a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java +++ b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java @@ -31,7 +31,7 @@ import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -class TypeToSchema extends TypeUtil.SchemaVisitor { +public class TypeToSchema extends TypeUtil.SchemaVisitor { private static final Schema BOOLEAN_SCHEMA = Schema.create(Schema.Type.BOOLEAN); private static final Schema INTEGER_SCHEMA = Schema.create(Schema.Type.INT); private static final Schema LONG_SCHEMA = Schema.create(Schema.Type.LONG); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 803217f4e980..099830ae80af 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -19,40 +19,34 @@ package org.apache.iceberg.mr.mapred; -import com.google.common.collect.Lists; +import static org.apache.iceberg.types.Types.NestedField.optional; + import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.StandaloneHiveRunner; import com.klarna.hiverunner.annotations.HiveSQL; import java.io.File; import java.io.IOException; -import java.util.List; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.RecordReader; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; -import org.apache.iceberg.data.Record; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.types.Types; -import org.iceberg.mr.mapred.IcebergInputFormat; -import org.iceberg.mr.mapred.IcebergWritable; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; - -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @RunWith(StandaloneHiveRunner.class) public class TestIcebergInputFormat { + private static final Logger LOG = LoggerFactory.getLogger(TestIcebergInputFormat.class); + @HiveSQL(files = {}, autoStart = true) private HiveShell shell; @@ -78,6 +72,17 @@ public void before() throws IOException { table.newAppend().appendFile(fileA).commit(); } + @Test + public void bla() { + try { + LOG.error("YYY: " + org.apache.avro.Schema.class.getProtectionDomain().getCodeSource()); + } catch (Throwable t) { + t.printStackTrace(); + LOG.error("XXX", t); + LOG.error("ZZZ", t.getCause()); + } + } +/* @Test public void testInputFormat() { shell.execute("CREATE DATABASE source_db"); @@ -128,7 +133,7 @@ public void testGetRecordReader() throws IOException { } } assertEquals(3, records.size()); - } + }*/ @After public void after() throws IOException { diff --git a/versions.lock b/versions.lock index c7cf1f751f8c..54780d02e40b 100644 --- a/versions.lock +++ b/versions.lock @@ -23,17 +23,17 @@ com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 (1 constr com.github.luben:zstd-jni:1.3.2-2 (1 constraints: 760d7c51) com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (10 constraints: 078609f3) com.google.code.findbugs:jsr305:3.0.2 (15 constraints: f4c0e31b) -com.google.code.gson:gson:2.2.4 (6 constraints: f44c3ddb) +com.google.code.gson:gson:2.2.4 (5 constraints: eb412e83) com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544) com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) -com.google.guava:guava:28.0-jre (41 constraints: 0c51e1d2) +com.google.guava:guava:28.0-jre (40 constraints: b841d485) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) com.google.inject:guice:3.0 (8 constraints: 2c93366d) com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce) com.google.inject.extensions:guice-servlet:3.0 (12 constraints: 01e2ea30) com.google.j2objc:j2objc-annotations:1.3 (1 constraints: b809eda0) -com.google.protobuf:protobuf-java:2.5.0 (19 constraints: dc42d583) +com.google.protobuf:protobuf-java:2.5.0 (21 constraints: 3e5cd766) com.googlecode.javaewah:JavaEWAH:0.3.2 (1 constraints: ea0dfc42) com.jamesmurty.utils:java-xmlbuilder:0.4 (1 constraints: e40aa5ca) com.jcraft:jsch:0.1.42 (1 constraints: bb0ded3c) @@ -54,17 +54,17 @@ com.univocity:univocity-parsers:2.7.3 (1 constraints: c40ccb27) com.zaxxer:HikariCP:2.5.1 (1 constraints: 390d7120) commons-beanutils:commons-beanutils:1.7.0 (1 constraints: da0e635f) commons-beanutils:commons-beanutils-core:1.8.0 (1 constraints: 1d134124) -commons-cli:commons-cli:1.2 (13 constraints: e4a04168) -commons-codec:commons-codec:1.10 (24 constraints: d0347e53) +commons-cli:commons-cli:1.2 (12 constraints: ab9686c9) +commons-codec:commons-codec:1.10 (23 constraints: 282a59a2) commons-collections:commons-collections:3.2.2 (6 constraints: 8e604cdc) commons-configuration:commons-configuration:1.6 (1 constraints: 2d0d5c14) commons-daemon:commons-daemon:1.0.13 (1 constraints: d50c811c) commons-dbcp:commons-dbcp:1.4 (3 constraints: 9029e0e4) commons-digester:commons-digester:1.8 (1 constraints: bf1228fe) commons-el:commons-el:1.0 (2 constraints: ad11e7f0) -commons-httpclient:commons-httpclient:3.1 (6 constraints: 8545051d) -commons-io:commons-io:2.4 (11 constraints: e6902ece) -commons-lang:commons-lang:2.6 (33 constraints: 12c29bcf) +commons-httpclient:commons-httpclient:3.1 (5 constraints: 803ac296) +commons-io:commons-io:2.4 (10 constraints: 3d86595e) +commons-lang:commons-lang:2.6 (30 constraints: b29c2b23) commons-logging:commons-logging:1.2 (31 constraints: aedcdd2a) commons-net:commons-net:3.1 (3 constraints: 3d222e61) commons-pool:commons-pool:1.6 (4 constraints: e336ab5e) @@ -94,7 +94,7 @@ javax.validation:validation-api:1.1.0.Final (1 constraints: 13133130) javax.ws.rs:javax.ws.rs-api:2.0.1 (5 constraints: 6e649355) javax.xml.bind:jaxb-api:2.2.11 (6 constraints: a069fd48) javolution:javolution:5.5.1 (2 constraints: 2b1b2b82) -jline:jline:2.12 (4 constraints: e72bc09c) +jline:jline:2.12 (3 constraints: 7c21b2cb) joda-time:joda-time:2.9.9 (4 constraints: 2326d336) junit:junit:4.12 (10 constraints: 4a8036b6) log4j:apache-log4j-extras:1.2.17 (1 constraints: 200e1d51) @@ -105,26 +105,26 @@ net.razorvine:pyrolite:4.13 (1 constraints: eb0cb829) net.sf.kosmosfs:kfs:0.3 (1 constraints: fd077074) net.sf.opencsv:opencsv:2.3 (2 constraints: a218daa5) net.sf.py4j:py4j:0.10.7 (1 constraints: 490d0044) -org.antlr:ST4:4.0.4 (3 constraints: 5521e4e4) -org.antlr:antlr-runtime:3.5.2 (6 constraints: 7a43035f) +org.antlr:ST4:4.0.4 (2 constraints: 4c16631f) +org.antlr:antlr-runtime:3.5.2 (5 constraints: 6f380c67) org.antlr:antlr4-runtime:4.7 (1 constraints: 7a0e125f) -org.apache.ant:ant:1.9.1 (3 constraints: 262660e7) +org.apache.ant:ant:1.9.1 (1 constraints: f10b24f3) org.apache.ant:ant-launcher:1.9.1 (1 constraints: 69082485) org.apache.arrow:arrow-format:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-memory:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-vector:0.14.1 (2 constraints: 2012a545) -org.apache.avro:avro:1.9.2 (8 constraints: 71686591) +org.apache.avro:avro:1.9.2 (6 constraints: da4d6402) org.apache.avro:avro-ipc:1.8.2 (1 constraints: f90b5bf4) org.apache.avro:avro-mapred:1.8.2 (2 constraints: 3a1a4787) org.apache.calcite:calcite-avatica:1.2.0-incubating (3 constraints: 4b35b263) org.apache.calcite:calcite-core:1.2.0-incubating (1 constraints: 68119fdf) org.apache.calcite:calcite-linq4j:1.2.0-incubating (1 constraints: ac1147d8) -org.apache.commons:commons-compress:1.19 (7 constraints: ff569390) +org.apache.commons:commons-compress:1.19 (6 constraints: 524cca4d) org.apache.commons:commons-crypto:1.0.0 (2 constraints: 3a1e5fbf) org.apache.commons:commons-lang3:3.9 (8 constraints: 3362d239) org.apache.commons:commons-math3:3.4.1 (3 constraints: 7c24247c) org.apache.curator:curator-client:2.7.1 (3 constraints: 272ac6a3) -org.apache.curator:curator-framework:2.7.1 (9 constraints: 797966d8) +org.apache.curator:curator-framework:2.7.1 (7 constraints: 756381e2) org.apache.curator:curator-recipes:2.7.1 (4 constraints: ba337377) org.apache.derby:derby:10.12.1.1 (3 constraints: 9f2cb182) org.apache.directory.api:api-asn1-api:1.0.0-M20 (1 constraints: 3d163b13) @@ -156,23 +156,23 @@ org.apache.hbase:hbase-annotations:1.1.1 (4 constraints: 0d36c989) org.apache.hbase:hbase-client:1.1.1 (3 constraints: a427c078) org.apache.hbase:hbase-common:1.1.1 (5 constraints: 4b433401) org.apache.hbase:hbase-protocol:1.1.1 (4 constraints: 9d335412) -org.apache.hive:hive-common:2.3.6 (8 constraints: eb682401) -org.apache.hive:hive-metastore:2.3.6 (5 constraints: ef389a58) -org.apache.hive:hive-serde:2.3.6 (7 constraints: 834d1ecf) -org.apache.hive:hive-service-rpc:2.3.6 (3 constraints: cd22ff7b) -org.apache.hive:hive-shims:2.3.6 (7 constraints: 1f53d276) +org.apache.hive:hive-common:2.3.6 (5 constraints: 6141bdc2) +org.apache.hive:hive-metastore:2.3.6 (2 constraints: 651190f2) +org.apache.hive:hive-serde:2.3.6 (4 constraints: e22dbc9a) +org.apache.hive:hive-service-rpc:2.3.6 (2 constraints: d317528f) +org.apache.hive:hive-shims:2.3.6 (4 constraints: 7a329460) org.apache.hive:hive-storage-api:2.4.0 (1 constraints: ec0b19f3) org.apache.hive.shims:hive-shims-0.23:2.3.6 (1 constraints: 8c0b6ce5) org.apache.hive.shims:hive-shims-common:2.3.6 (3 constraints: 222cfaad) org.apache.hive.shims:hive-shims-scheduler:2.3.6 (1 constraints: 8c0b6ce5) org.apache.htrace:htrace-core:3.1.0-incubating (5 constraints: 89553ebc) -org.apache.httpcomponents:httpclient:4.5.6 (7 constraints: 455794a4) -org.apache.httpcomponents:httpcore:4.4.10 (6 constraints: c24d4f67) -org.apache.ivy:ivy:2.4.0 (3 constraints: 0826dbf1) -org.apache.logging.log4j:log4j-1.2-api:2.6.2 (2 constraints: fb160260) +org.apache.httpcomponents:httpclient:4.5.6 (5 constraints: 153ee053) +org.apache.httpcomponents:httpcore:4.4.10 (4 constraints: 91348b59) +org.apache.ivy:ivy:2.4.0 (2 constraints: 011b7392) +org.apache.logging.log4j:log4j-1.2-api:2.6.2 (1 constraints: f00b21f3) org.apache.logging.log4j:log4j-api:2.6.2 (4 constraints: 2e3c9f23) org.apache.logging.log4j:log4j-core:2.6.2 (2 constraints: fd1c2464) -org.apache.logging.log4j:log4j-slf4j-impl:2.6.2 (3 constraints: 8d27cb64) +org.apache.logging.log4j:log4j-slf4j-impl:2.6.2 (2 constraints: 821cbce5) org.apache.logging.log4j:log4j-web:2.6.2 (1 constraints: f00b21f3) org.apache.orc:orc-core:1.6.2 (4 constraints: 8b2becc9) org.apache.orc:orc-mapreduce:1.5.5 (1 constraints: c30cc227) @@ -183,7 +183,6 @@ org.apache.parquet:parquet-common:1.11.0 (2 constraints: 4c1e7785) org.apache.parquet:parquet-encoding:1.11.0 (1 constraints: ca0efb64) org.apache.parquet:parquet-format-structures:1.11.0 (3 constraints: 6e2bc836) org.apache.parquet:parquet-hadoop:1.11.0 (2 constraints: de1ac7b3) -org.apache.parquet:parquet-hadoop-bundle:1.8.1 (1 constraints: 7a0bc7e4) org.apache.parquet:parquet-jackson:1.11.0 (1 constraints: b70ee963) org.apache.pig:pig:0.14.0 (1 constraints: 37052f3b) org.apache.spark:spark-avro_2.11:2.4.4 (1 constraints: 0c050536) @@ -199,7 +198,7 @@ org.apache.spark:spark-sql_2.11:2.4.4 (1 constraints: 1e0d0037) org.apache.spark:spark-tags_2.11:2.4.4 (8 constraints: 036fa69d) org.apache.spark:spark-unsafe_2.11:2.4.4 (2 constraints: f11bc213) org.apache.thrift:libfb303:0.9.3 (4 constraints: 8034ebe7) -org.apache.thrift:libthrift:0.9.3 (11 constraints: 908a19e4) +org.apache.thrift:libthrift:0.9.3 (9 constraints: f574b013) org.apache.twill:twill-api:0.6.0-incubating (2 constraints: e422b4e1) org.apache.twill:twill-common:0.6.0-incubating (4 constraints: 3d467991) org.apache.twill:twill-core:0.6.0-incubating (1 constraints: d70f9d7c) @@ -208,7 +207,7 @@ org.apache.twill:twill-discovery-core:0.6.0-incubating (2 constraints: 332039f9) org.apache.twill:twill-zookeeper:0.6.0-incubating (3 constraints: 94341288) org.apache.xbean:xbean-asm6-shaded:4.8 (2 constraints: 2419a30f) org.apache.yetus:audience-annotations:0.11.0 (1 constraints: c40eb364) -org.apache.zookeeper:zookeeper:3.4.6 (17 constraints: 12f4f17d) +org.apache.zookeeper:zookeeper:3.4.6 (15 constraints: 08de0a12) org.checkerframework:checker-qual:2.8.1 (2 constraints: 1a1a3944) org.codehaus.jackson:jackson-core-asl:1.9.13 (14 constraints: e8bb1763) org.codehaus.jackson:jackson-jaxrs:1.9.13 (4 constraints: 5235d62f) @@ -216,10 +215,10 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13 (17 constraints: c3eee844) org.codehaus.jackson:jackson-xc:1.9.13 (3 constraints: 73286ded) org.codehaus.janino:commons-compiler:3.0.9 (3 constraints: 0a2837cc) org.codehaus.janino:janino:3.0.9 (2 constraints: 3f1c6304) -org.codehaus.jettison:jettison:1.1 (4 constraints: a84e24a9) +org.codehaus.jettison:jettison:1.1 (5 constraints: 155c2ac6) org.codehaus.mojo:animal-sniffer-annotations:1.17 (1 constraints: ed09d8aa) org.datanucleus:datanucleus-api-jdo:4.2.4 (2 constraints: 591df91b) -org.datanucleus:datanucleus-core:4.1.17 (5 constraints: 584455e8) +org.datanucleus:datanucleus-core:4.1.17 (4 constraints: 1a394483) org.datanucleus:datanucleus-rdbms:4.1.19 (2 constraints: 911dec32) org.datanucleus:javax.jdo:3.2.0-m3 (1 constraints: 030ea249) org.eclipse.jdt:core:3.1.1 (1 constraints: b40a38d8) @@ -268,14 +267,14 @@ org.scala-lang.modules:scala-parser-combinators_2.11:1.1.0 (1 constraints: cf0e7 org.scala-lang.modules:scala-xml_2.11:1.0.6 (1 constraints: 080b84e9) org.slf4j:jcl-over-slf4j:1.7.16 (1 constraints: 500d1d44) org.slf4j:jul-to-slf4j:1.7.16 (1 constraints: 500d1d44) -org.slf4j:slf4j-api:1.7.25 (76 constraints: 544996f4) +org.slf4j:slf4j-api:1.7.25 (70 constraints: 0bf8d047) org.sonatype.sisu.inject:cglib:2.2.1-v20090111 (1 constraints: aa0cfd36) org.spark-project.hive:hive-exec:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.hive:hive-metastore:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.spark:unused:1.0.0 (12 constraints: 9aab75cf) org.xerial.snappy:snappy-java:1.1.7.3 (2 constraints: 681c5e46) -oro:oro:2.0.8 (3 constraints: 3b229337) -stax:stax-api:1.0.1 (3 constraints: 8d2668e5) +oro:oro:2.0.8 (2 constraints: 7c156a13) +stax:stax-api:1.0.1 (2 constraints: 8a1b5e9d) tomcat:jasper-compiler:5.5.23 (4 constraints: ff2fc367) tomcat:jasper-runtime:5.5.23 (4 constraints: ff2fc367) xerces:xercesImpl:2.9.1 (1 constraints: ac0ccc0f) @@ -284,19 +283,16 @@ xmlenc:xmlenc:0.52 (3 constraints: 05228b2f) [Test dependencies] com.beust:jcommander:1.30 (1 constraints: 8a0c0505) -com.klarna:hiverunner:5.1.1 (1 constraints: 09050836) +com.klarna:hiverunner:4.1.0 (1 constraints: 07050236) com.lmax:disruptor:3.3.0 (1 constraints: a80c770e) com.ning:async-http-client:1.8.16 (1 constraints: 110f9671) com.yammer.metrics:metrics-core:2.2.0 (2 constraints: 121c6ce2) dom4j:dom4j:1.6.1 (1 constraints: 8c0ceb00) -jakarta.jms:jakarta.jms-api:2.0.2 (1 constraints: 5617da29) javassist:javassist:3.12.1.GA (1 constraints: 710d3035) net.sf.jpam:jpam:1.1 (1 constraints: f20b40e9) -org.apache.calcite.avatica:avatica:1.8.0 (1 constraints: 0b0bf5d6) -org.apache.calcite.avatica:avatica-metrics:1.8.0 (1 constraints: 960e635d) org.apache.commons:commons-collections4:4.1 (2 constraints: 09137a51) org.apache.commons:commons-math:2.2 (3 constraints: 322af180) -org.apache.curator:apache-curator:2.7.1 (2 constraints: c718e2d6) +org.apache.curator:apache-curator:2.7.1 (1 constraints: bc0d093b) org.apache.hadoop:hadoop-archives:2.7.3 (1 constraints: f21198ff) org.apache.hadoop:hadoop-mapreduce-client-hs:2.7.3 (1 constraints: b60fac84) org.apache.hadoop:hadoop-minicluster:2.7.3 (1 constraints: 0e050d36) @@ -307,18 +303,14 @@ org.apache.hbase:hbase-hadoop2-compat:1.1.1 (3 constraints: e9285c23) org.apache.hbase:hbase-prefix-tree:1.1.1 (1 constraints: a50c680e) org.apache.hbase:hbase-procedure:1.1.1 (1 constraints: a50c680e) org.apache.hbase:hbase-server:1.1.1 (1 constraints: cd0d4a3c) -org.apache.hive:hive-cli:2.3.6 (1 constraints: f21190ff) -org.apache.hive:hive-exec:2.3.6 (6 constraints: fa4f2433) -org.apache.hive:hive-jdbc:2.3.6 (1 constraints: 090a76ad) org.apache.hive:hive-llap-client:2.3.6 (2 constraints: 651aa157) org.apache.hive:hive-llap-common:2.3.6 (2 constraints: 911b96aa) org.apache.hive:hive-llap-server:2.3.6 (1 constraints: 590cd001) -org.apache.hive:hive-llap-tez:2.3.6 (2 constraints: e11855d8) -org.apache.hive:hive-service:2.3.6 (3 constraints: a21fbe56) -org.apache.hive:hive-vector-code-gen:2.3.6 (1 constraints: 0d0bf1d6) -org.apache.hive.hcatalog:hive-hcatalog-core:2.3.6 (2 constraints: 8e2bb3f1) -org.apache.hive.hcatalog:hive-hcatalog-server-extensions:2.3.6 (1 constraints: 3214a97c) -org.apache.hive.hcatalog:hive-webhcat-java-client:2.3.6 (1 constraints: 090a76ad) +org.apache.hive:hive-llap-tez:2.3.6 (1 constraints: d50d5a3c) +org.apache.hive:hive-service:2.3.6 (1 constraints: 0d050436) +org.apache.hive.hcatalog:hive-hcatalog-core:2.3.3 (2 constraints: 882bfff0) +org.apache.hive.hcatalog:hive-hcatalog-server-extensions:2.3.3 (1 constraints: 2f14a67c) +org.apache.hive.hcatalog:hive-webhcat-java-client:2.3.3 (1 constraints: 060a73ad) org.apache.slider:slider-core:0.90.2-incubating (1 constraints: 56124bfd) org.apache.tez:hadoop-shim:0.9.1 (3 constraints: d3244224) org.apache.tez:tez-api:0.9.1 (5 constraints: 1740d50b) @@ -327,18 +319,8 @@ org.apache.tez:tez-dag:0.9.1 (1 constraints: 080a79ad) org.apache.tez:tez-mapreduce:0.9.1 (1 constraints: 080a79ad) org.apache.tez:tez-runtime-internals:0.9.1 (1 constraints: e1099fb1) org.apache.tez:tez-runtime-library:0.9.1 (2 constraints: 4b167422) -org.apache.velocity:velocity:1.5 (1 constraints: c70e875e) -org.apiguardian:apiguardian-api:1.1.0 (5 constraints: 0654a8a8) -org.codehaus.groovy:groovy-all:2.4.4 (1 constraints: 0c0bf2d6) org.jamon:jamon-runtime:2.3.1 (2 constraints: fb1870e4) -org.junit.jupiter:junit-jupiter:5.6.0 (1 constraints: 090a88ad) -org.junit.jupiter:junit-jupiter-api:5.6.0 (3 constraints: 6a2f2bdb) -org.junit.jupiter:junit-jupiter-engine:5.6.0 (1 constraints: 080ed73b) -org.junit.jupiter:junit-jupiter-params:5.6.0 (1 constraints: 080ed73b) -org.junit.platform:junit-platform-commons:1.6.0 (2 constraints: d520374a) -org.junit.platform:junit-platform-engine:1.6.0 (1 constraints: a7101fb4) org.mockito:mockito-core:1.10.19 (1 constraints: 6e059840) org.mortbay.jetty:jetty-sslengine:6.1.26 (1 constraints: e10c631b) -org.opentest4j:opentest4j:1.2.0 (2 constraints: cd205b49) org.reflections:reflections:0.9.8 (1 constraints: 0f0a80ad) org.slf4j:slf4j-simple:1.7.5 (1 constraints: 0f050a36) diff --git a/versions.props b/versions.props index d8b2d3922854..c0293473e71e 100644 --- a/versions.props +++ b/versions.props @@ -18,5 +18,4 @@ junit:junit = 4.12 org.slf4j:slf4j-simple = 1.7.5 org.mockito:mockito-core = 1.10.19 joda-time:joda-time = 2.9.9 -org.apache.hive:hive-exec = 2.3.6 org.apache.hive:hive-metastore = 2.3.6 From 73987533d71827946998afdd3ab4c8abedb90718 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 26 Mar 2020 19:11:53 +0000 Subject: [PATCH 12/51] only hive-exec core on mr classpath now --- build.gradle | 7 +++--- versions.lock | 64 +++++++++++++++++++++++++++----------------------- versions.props | 3 +-- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/build.gradle b/build.gradle index 92032d9c3907..81c2fdd207c4 100644 --- a/build.gradle +++ b/build.gradle @@ -60,7 +60,6 @@ subprojects { configurations { testCompile.extendsFrom compileOnly all { - exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.slf4j', module: 'slf4j-log4j12' } @@ -278,7 +277,7 @@ project(':iceberg-hive') { compileOnly "org.apache.avro:avro" - compileOnly("org.apache.hive:hive-metastore") { + compileOnly("org.apache.hive:hive-metastore:2.3.6") { exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' @@ -311,7 +310,7 @@ project(':iceberg-hive') { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testCompile("org.apache.hive:hive-metastore") { + testCompile("org.apache.hive:hive-metastore:2.3.6") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.slf4j', module: 'slf4j-log4j12' @@ -374,6 +373,7 @@ project(':iceberg-mr') { testCompile("org.apache.hive:hive-service:2.3.6") { exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.hive', module: 'hive-exec' } compile("org.apache.hive:hive-exec:2.3.6:core") { @@ -390,7 +390,6 @@ project(':iceberg-mr') { exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testCompile 'junit:junit' testCompile("com.klarna:hiverunner:4.1.0") { diff --git a/versions.lock b/versions.lock index 54780d02e40b..62a01a3001a9 100644 --- a/versions.lock +++ b/versions.lock @@ -13,9 +13,9 @@ com.carrotsearch:hppc:0.7.2 (1 constraints: f70cda14) com.clearspring.analytics:stream:2.7.0 (1 constraints: 1a0dd136) com.esotericsoftware:kryo-shaded:4.0.2 (2 constraints: b71345a6) com.esotericsoftware:minlog:1.3.0 (1 constraints: 670e7c4f) -com.fasterxml.jackson.core:jackson-annotations:2.10.2 (7 constraints: 27711495) -com.fasterxml.jackson.core:jackson-core:2.10.2 (8 constraints: a16ee0bc) -com.fasterxml.jackson.core:jackson-databind:2.10.2 (13 constraints: 62bac5bf) +com.fasterxml.jackson.core:jackson-annotations:2.10.2 (6 constraints: d863d8d4) +com.fasterxml.jackson.core:jackson-core:2.10.2 (7 constraints: 5261f057) +com.fasterxml.jackson.core:jackson-databind:2.10.2 (12 constraints: 13ad1133) com.fasterxml.jackson.module:jackson-module-paranamer:2.10.2 (1 constraints: 03162c16) com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.2 (1 constraints: 7f0da251) com.github.ben-manes.caffeine:caffeine:2.7.0 (1 constraints: 0b050a36) @@ -23,17 +23,17 @@ com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 (1 constr com.github.luben:zstd-jni:1.3.2-2 (1 constraints: 760d7c51) com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (10 constraints: 078609f3) com.google.code.findbugs:jsr305:3.0.2 (15 constraints: f4c0e31b) -com.google.code.gson:gson:2.2.4 (5 constraints: eb412e83) +com.google.code.gson:gson:2.2.4 (6 constraints: f44c3ddb) com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544) com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) -com.google.guava:guava:28.0-jre (40 constraints: b841d485) +com.google.guava:guava:28.0-jre (41 constraints: 56500d9f) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) com.google.inject:guice:3.0 (8 constraints: 2c93366d) com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce) com.google.inject.extensions:guice-servlet:3.0 (12 constraints: 01e2ea30) com.google.j2objc:j2objc-annotations:1.3 (1 constraints: b809eda0) -com.google.protobuf:protobuf-java:2.5.0 (21 constraints: 3e5cd766) +com.google.protobuf:protobuf-java:3.0.0-beta-1 (22 constraints: f46c2ce2) com.googlecode.javaewah:JavaEWAH:0.3.2 (1 constraints: ea0dfc42) com.jamesmurty.utils:java-xmlbuilder:0.4 (1 constraints: e40aa5ca) com.jcraft:jsch:0.1.42 (1 constraints: bb0ded3c) @@ -55,16 +55,16 @@ com.zaxxer:HikariCP:2.5.1 (1 constraints: 390d7120) commons-beanutils:commons-beanutils:1.7.0 (1 constraints: da0e635f) commons-beanutils:commons-beanutils-core:1.8.0 (1 constraints: 1d134124) commons-cli:commons-cli:1.2 (12 constraints: ab9686c9) -commons-codec:commons-codec:1.10 (23 constraints: 282a59a2) +commons-codec:commons-codec:1.10 (24 constraints: d0347e53) commons-collections:commons-collections:3.2.2 (6 constraints: 8e604cdc) commons-configuration:commons-configuration:1.6 (1 constraints: 2d0d5c14) commons-daemon:commons-daemon:1.0.13 (1 constraints: d50c811c) commons-dbcp:commons-dbcp:1.4 (3 constraints: 9029e0e4) commons-digester:commons-digester:1.8 (1 constraints: bf1228fe) commons-el:commons-el:1.0 (2 constraints: ad11e7f0) -commons-httpclient:commons-httpclient:3.1 (5 constraints: 803ac296) -commons-io:commons-io:2.4 (10 constraints: 3d86595e) -commons-lang:commons-lang:2.6 (30 constraints: b29c2b23) +commons-httpclient:commons-httpclient:3.1 (6 constraints: 8545051d) +commons-io:commons-io:2.4 (11 constraints: e6902ece) +commons-lang:commons-lang:2.6 (32 constraints: d4b70019) commons-logging:commons-logging:1.2 (31 constraints: aedcdd2a) commons-net:commons-net:3.1 (3 constraints: 3d222e61) commons-pool:commons-pool:1.6 (4 constraints: e336ab5e) @@ -105,10 +105,10 @@ net.razorvine:pyrolite:4.13 (1 constraints: eb0cb829) net.sf.kosmosfs:kfs:0.3 (1 constraints: fd077074) net.sf.opencsv:opencsv:2.3 (2 constraints: a218daa5) net.sf.py4j:py4j:0.10.7 (1 constraints: 490d0044) -org.antlr:ST4:4.0.4 (2 constraints: 4c16631f) -org.antlr:antlr-runtime:3.5.2 (5 constraints: 6f380c67) +org.antlr:ST4:4.0.4 (3 constraints: 5521e4e4) +org.antlr:antlr-runtime:3.5.2 (6 constraints: 7a43035f) org.antlr:antlr4-runtime:4.7 (1 constraints: 7a0e125f) -org.apache.ant:ant:1.9.1 (1 constraints: f10b24f3) +org.apache.ant:ant:1.9.1 (3 constraints: 262660e7) org.apache.ant:ant-launcher:1.9.1 (1 constraints: 69082485) org.apache.arrow:arrow-format:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-memory:0.14.1 (1 constraints: 240df421) @@ -116,15 +116,18 @@ org.apache.arrow:arrow-vector:0.14.1 (2 constraints: 2012a545) org.apache.avro:avro:1.9.2 (6 constraints: da4d6402) org.apache.avro:avro-ipc:1.8.2 (1 constraints: f90b5bf4) org.apache.avro:avro-mapred:1.8.2 (2 constraints: 3a1a4787) -org.apache.calcite:calcite-avatica:1.2.0-incubating (3 constraints: 4b35b263) -org.apache.calcite:calcite-core:1.2.0-incubating (1 constraints: 68119fdf) -org.apache.calcite:calcite-linq4j:1.2.0-incubating (1 constraints: ac1147d8) -org.apache.commons:commons-compress:1.19 (6 constraints: 524cca4d) +org.apache.calcite:calcite-avatica:1.2.0-incubating (2 constraints: a0237b5d) +org.apache.calcite:calcite-core:1.10.0 (2 constraints: 9b164229) +org.apache.calcite:calcite-linq4j:1.10.0 (1 constraints: 8a0d363a) +org.apache.calcite.avatica:avatica:1.8.0 (1 constraints: 610dbf2c) +org.apache.calcite.avatica:avatica-metrics:1.8.0 (1 constraints: 960e635d) +org.apache.commons:commons-compress:1.19 (7 constraints: ff569390) org.apache.commons:commons-crypto:1.0.0 (2 constraints: 3a1e5fbf) -org.apache.commons:commons-lang3:3.9 (8 constraints: 3362d239) +org.apache.commons:commons-lang3:3.9 (9 constraints: 316fa5f3) org.apache.commons:commons-math3:3.4.1 (3 constraints: 7c24247c) +org.apache.curator:apache-curator:2.7.1 (2 constraints: c718e2d6) org.apache.curator:curator-client:2.7.1 (3 constraints: 272ac6a3) -org.apache.curator:curator-framework:2.7.1 (7 constraints: 756381e2) +org.apache.curator:curator-framework:2.7.1 (8 constraints: 806edda7) org.apache.curator:curator-recipes:2.7.1 (4 constraints: ba337377) org.apache.derby:derby:10.12.1.1 (3 constraints: 9f2cb182) org.apache.directory.api:api-asn1-api:1.0.0-M20 (1 constraints: 3d163b13) @@ -157,18 +160,20 @@ org.apache.hbase:hbase-client:1.1.1 (3 constraints: a427c078) org.apache.hbase:hbase-common:1.1.1 (5 constraints: 4b433401) org.apache.hbase:hbase-protocol:1.1.1 (4 constraints: 9d335412) org.apache.hive:hive-common:2.3.6 (5 constraints: 6141bdc2) +org.apache.hive:hive-exec:2.3.6 (1 constraints: 0d050436) org.apache.hive:hive-metastore:2.3.6 (2 constraints: 651190f2) org.apache.hive:hive-serde:2.3.6 (4 constraints: e22dbc9a) org.apache.hive:hive-service-rpc:2.3.6 (2 constraints: d317528f) -org.apache.hive:hive-shims:2.3.6 (4 constraints: 7a329460) +org.apache.hive:hive-shims:2.3.6 (5 constraints: 863d32c4) org.apache.hive:hive-storage-api:2.4.0 (1 constraints: ec0b19f3) +org.apache.hive:hive-vector-code-gen:2.3.6 (1 constraints: 0d0bf1d6) org.apache.hive.shims:hive-shims-0.23:2.3.6 (1 constraints: 8c0b6ce5) org.apache.hive.shims:hive-shims-common:2.3.6 (3 constraints: 222cfaad) org.apache.hive.shims:hive-shims-scheduler:2.3.6 (1 constraints: 8c0b6ce5) org.apache.htrace:htrace-core:3.1.0-incubating (5 constraints: 89553ebc) -org.apache.httpcomponents:httpclient:4.5.6 (5 constraints: 153ee053) -org.apache.httpcomponents:httpcore:4.4.10 (4 constraints: 91348b59) -org.apache.ivy:ivy:2.4.0 (2 constraints: 011b7392) +org.apache.httpcomponents:httpclient:4.5.6 (6 constraints: ac4c9b4e) +org.apache.httpcomponents:httpcore:4.4.10 (5 constraints: 29432873) +org.apache.ivy:ivy:2.4.0 (3 constraints: 0826dbf1) org.apache.logging.log4j:log4j-1.2-api:2.6.2 (1 constraints: f00b21f3) org.apache.logging.log4j:log4j-api:2.6.2 (4 constraints: 2e3c9f23) org.apache.logging.log4j:log4j-core:2.6.2 (2 constraints: fd1c2464) @@ -205,10 +210,12 @@ org.apache.twill:twill-core:0.6.0-incubating (1 constraints: d70f9d7c) org.apache.twill:twill-discovery-api:0.6.0-incubating (3 constraints: 25345d4c) org.apache.twill:twill-discovery-core:0.6.0-incubating (2 constraints: 332039f9) org.apache.twill:twill-zookeeper:0.6.0-incubating (3 constraints: 94341288) +org.apache.velocity:velocity:1.5 (1 constraints: c70e875e) org.apache.xbean:xbean-asm6-shaded:4.8 (2 constraints: 2419a30f) org.apache.yetus:audience-annotations:0.11.0 (1 constraints: c40eb364) -org.apache.zookeeper:zookeeper:3.4.6 (15 constraints: 08de0a12) +org.apache.zookeeper:zookeeper:3.4.6 (16 constraints: 16e9c913) org.checkerframework:checker-qual:2.8.1 (2 constraints: 1a1a3944) +org.codehaus.groovy:groovy-all:2.4.4 (1 constraints: 0c0bf2d6) org.codehaus.jackson:jackson-core-asl:1.9.13 (14 constraints: e8bb1763) org.codehaus.jackson:jackson-jaxrs:1.9.13 (4 constraints: 5235d62f) org.codehaus.jackson:jackson-mapper-asl:1.9.13 (17 constraints: c3eee844) @@ -218,7 +225,7 @@ org.codehaus.janino:janino:3.0.9 (2 constraints: 3f1c6304) org.codehaus.jettison:jettison:1.1 (5 constraints: 155c2ac6) org.codehaus.mojo:animal-sniffer-annotations:1.17 (1 constraints: ed09d8aa) org.datanucleus:datanucleus-api-jdo:4.2.4 (2 constraints: 591df91b) -org.datanucleus:datanucleus-core:4.1.17 (4 constraints: 1a394483) +org.datanucleus:datanucleus-core:4.1.17 (5 constraints: 584455e8) org.datanucleus:datanucleus-rdbms:4.1.19 (2 constraints: 911dec32) org.datanucleus:javax.jdo:3.2.0-m3 (1 constraints: 030ea249) org.eclipse.jdt:core:3.1.1 (1 constraints: b40a38d8) @@ -267,14 +274,14 @@ org.scala-lang.modules:scala-parser-combinators_2.11:1.1.0 (1 constraints: cf0e7 org.scala-lang.modules:scala-xml_2.11:1.0.6 (1 constraints: 080b84e9) org.slf4j:jcl-over-slf4j:1.7.16 (1 constraints: 500d1d44) org.slf4j:jul-to-slf4j:1.7.16 (1 constraints: 500d1d44) -org.slf4j:slf4j-api:1.7.25 (70 constraints: 0bf8d047) +org.slf4j:slf4j-api:1.7.25 (75 constraints: f240961e) org.sonatype.sisu.inject:cglib:2.2.1-v20090111 (1 constraints: aa0cfd36) org.spark-project.hive:hive-exec:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.hive:hive-metastore:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.spark:unused:1.0.0 (12 constraints: 9aab75cf) org.xerial.snappy:snappy-java:1.1.7.3 (2 constraints: 681c5e46) -oro:oro:2.0.8 (2 constraints: 7c156a13) -stax:stax-api:1.0.1 (2 constraints: 8a1b5e9d) +oro:oro:2.0.8 (3 constraints: 3b229337) +stax:stax-api:1.0.1 (3 constraints: 8d2668e5) tomcat:jasper-compiler:5.5.23 (4 constraints: ff2fc367) tomcat:jasper-runtime:5.5.23 (4 constraints: ff2fc367) xerces:xercesImpl:2.9.1 (1 constraints: ac0ccc0f) @@ -292,7 +299,6 @@ javassist:javassist:3.12.1.GA (1 constraints: 710d3035) net.sf.jpam:jpam:1.1 (1 constraints: f20b40e9) org.apache.commons:commons-collections4:4.1 (2 constraints: 09137a51) org.apache.commons:commons-math:2.2 (3 constraints: 322af180) -org.apache.curator:apache-curator:2.7.1 (1 constraints: bc0d093b) org.apache.hadoop:hadoop-archives:2.7.3 (1 constraints: f21198ff) org.apache.hadoop:hadoop-mapreduce-client-hs:2.7.3 (1 constraints: b60fac84) org.apache.hadoop:hadoop-minicluster:2.7.3 (1 constraints: 0e050d36) diff --git a/versions.props b/versions.props index c0293473e71e..5c8a81b86d08 100644 --- a/versions.props +++ b/versions.props @@ -2,7 +2,6 @@ org.slf4j:slf4j-api = 1.7.22 com.google.guava:guava = 28.0-jre org.apache.avro:avro = 1.9.2 org.apache.hadoop:* = 2.7.3 -org.apache.hive:hive-metastore = 2.3.6 org.apache.orc:orc-core = 1.6.2 org.apache.parquet:parquet-avro = 1.11.0 org.apache.spark:spark-hive_2.11 = 2.4.4 @@ -18,4 +17,4 @@ junit:junit = 4.12 org.slf4j:slf4j-simple = 1.7.5 org.mockito:mockito-core = 1.10.19 joda-time:joda-time = 2.9.9 -org.apache.hive:hive-metastore = 2.3.6 + From e3c26de6e19a3aeb32935dbaf3be1da8398157a4 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 26 Mar 2020 19:28:01 +0000 Subject: [PATCH 13/51] brutal attempt at overriding guava version --- baseline.gradle | 2 +- build.gradle | 17 +++++++++-------- versions.props | 1 - 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/baseline.gradle b/baseline.gradle index ce7bdec42ac4..0a21db74f113 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -37,7 +37,7 @@ subprojects { //apply plugin: 'com.palantir.baseline-error-prone' } apply plugin: 'com.palantir.baseline-scalastyle' - apply plugin: 'com.palantir.baseline-class-uniqueness' + //apply plugin: 'com.palantir.baseline-class-uniqueness' apply plugin: 'com.palantir.baseline-reproducibility' apply plugin: 'com.palantir.baseline-exact-dependencies' apply plugin: 'com.palantir.baseline-release-compatibility' diff --git a/build.gradle b/build.gradle index 81c2fdd207c4..870935d3a075 100644 --- a/build.gradle +++ b/build.gradle @@ -89,7 +89,7 @@ apply from: 'jmh.gradle' project(':iceberg-api') { dependencies { - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -108,7 +108,7 @@ project(':iceberg-api') { project(':iceberg-common') { dependencies { - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -132,7 +132,7 @@ project(':iceberg-core') { compile project(':iceberg-api') compile project(':iceberg-common') - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -201,7 +201,7 @@ project(':iceberg-data') { compileOnly project(':iceberg-parquet') compileOnly project(':iceberg-orc') - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -338,6 +338,7 @@ project(':iceberg-hive') { project(':iceberg-mr') { dependencies { + compile('com.google.guava:guava:11.0.2') compile project(path: ':iceberg-core', configuration: 'shadow') compile project(path: ':iceberg-orc', configuration: 'shadow') compile project(path: ':iceberg-parquet', configuration: 'shadow') @@ -431,7 +432,7 @@ project(':iceberg-orc') { compile project(':iceberg-api') compile project(':iceberg-core') - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -517,7 +518,7 @@ project(':iceberg-parquet') { compile project(':iceberg-api') compile project(':iceberg-core') - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -642,7 +643,7 @@ project(':iceberg-arrow') { compile project(':iceberg-api') compile project(':iceberg-parquet') - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -675,7 +676,7 @@ project(':iceberg-spark') { compile project(':iceberg-arrow') compile project(':iceberg-hive') - compile('com.google.guava:guava') { + compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } diff --git a/versions.props b/versions.props index 5c8a81b86d08..af2e21763a2d 100644 --- a/versions.props +++ b/versions.props @@ -1,5 +1,4 @@ org.slf4j:slf4j-api = 1.7.22 -com.google.guava:guava = 28.0-jre org.apache.avro:avro = 1.9.2 org.apache.hadoop:* = 2.7.3 org.apache.orc:orc-core = 1.6.2 From 5d728e5c3119c3ce172bbe4107936c576ee9cc75 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Fri, 27 Mar 2020 16:10:10 +0000 Subject: [PATCH 14/51] Shade all the guava --- build.gradle | 215 ++++++++++++++++++++++++++++++++++++++++++++------ versions.lock | 2 +- 2 files changed, 191 insertions(+), 26 deletions(-) diff --git a/build.gradle b/build.gradle index 870935d3a075..0ee2063cb52b 100644 --- a/build.gradle +++ b/build.gradle @@ -88,6 +88,12 @@ apply from: 'tasks.gradle' apply from: 'jmh.gradle' project(':iceberg-api') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -103,9 +109,29 @@ project(':iceberg-api') { testCompile "org.apache.avro:avro" testCompile 'joda-time:joda-time' } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + } } project(':iceberg-common') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar dependencies { compile('com.google.guava:guava:28.0-jre') { @@ -119,6 +145,21 @@ project(':iceberg-common') { testCompile 'org.slf4j:slf4j-simple' testCompile 'org.mockito:mockito-core' } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + } } project(':iceberg-core') { @@ -129,8 +170,8 @@ project(':iceberg-core') { tasks.javadocJar.dependsOn tasks.shadowJar dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-common', configuration: 'shadow') compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -196,10 +237,10 @@ project(':iceberg-data') { tasks.javadocJar.dependsOn tasks.shadowJar dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') - compileOnly project(':iceberg-parquet') - compileOnly project(':iceberg-orc') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-core', configuration: 'shadow') + compileOnly project(path: ':iceberg-parquet', configuration: 'shadow') + compileOnly project(path: ':iceberg-orc', configuration: 'shadow') compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -217,6 +258,23 @@ project(':iceberg-data') { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' } + compile("org.apache.orc:orc-core::nohive") { + exclude group: 'org.apache.hadoop' + exclude group: 'commons-lang' + // These artifacts are shaded and included in the orc-core fat jar + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.hive', module: 'hive-storage-api' + } + + compile("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } + + compileOnly "org.apache.avro:avro" + compileOnly('com.github.ben-manes.caffeine:caffeine') testCompile("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' @@ -265,8 +323,14 @@ project(':iceberg-data') { } project(':iceberg-hive') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { - compile project(':iceberg-core') + compile project(path: ':iceberg-core', configuration: 'shadow') compileOnly 'org.slf4j:slf4j-api' compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' @@ -277,6 +341,8 @@ project(':iceberg-hive') { compileOnly "org.apache.avro:avro" + compileOnly('com.github.ben-manes.caffeine:caffeine') + compileOnly("org.apache.hive:hive-metastore:2.3.6") { exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.apache.avro', module: 'avro' @@ -334,6 +400,21 @@ project(':iceberg-hive') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + } } project(':iceberg-mr') { @@ -346,10 +427,13 @@ project(':iceberg-mr') { compileOnly "org.apache.avro:avro" compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'com.google.guava', module: 'guava' } compileOnly('com.github.ben-manes.caffeine:caffeine') - compileOnly('org.apache.calcite:calcite-core:1.10.0') + compileOnly('org.apache.calcite:calcite-core:1.10.0') { + exclude group: 'com.google.guava', module: 'guava' + } compile("org.apache.hive:hive-serde:2.3.6") { exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' @@ -365,6 +449,7 @@ project(':iceberg-mr') { exclude group: 'org.codehaus.mojo', module: 'animal-sniffer-annotations' exclude group: 'commons-collections', module: 'commons-collections' exclude group: 'org.apache.hive', module: 'hive-exec' + exclude group: 'com.google.guava', module: 'guava' } testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') @@ -375,6 +460,7 @@ project(':iceberg-mr') { exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.hive', module: 'hive-exec' + exclude group: 'com.google.guava', module: 'guava' } compile("org.apache.hive:hive-exec:2.3.6:core") { @@ -390,6 +476,7 @@ project(':iceberg-mr') { exclude group: 'org.apache.calcite' exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'com.google.guava', module: 'guava' } testCompile 'junit:junit' @@ -399,6 +486,7 @@ project(':iceberg-mr') { exclude group: 'org.codehaus.jettison', module: 'jettison' exclude group: 'javax.jms', module: 'jms' exclude group: 'org.apache.hive', module: '*' + exclude group: 'com.google.guava', module: 'guava' } testCompile("org.apache.hive:hive-exec:2.3.6:core") { exclude group: 'stax', module: 'stax-api' @@ -413,6 +501,7 @@ project(':iceberg-mr') { exclude group: 'org.apache.calcite' exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'com.google.guava', module: 'guava' } } task copyToLib(type: Copy) { @@ -429,8 +518,8 @@ project(':iceberg-orc') { tasks.javadocJar.dependsOn tasks.shadowJar dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-core', configuration: 'shadow') compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -515,8 +604,8 @@ project(':iceberg-parquet') { tasks.javadocJar.dependsOn tasks.shadowJar dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-core', configuration: 'shadow') compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -524,6 +613,7 @@ project(':iceberg-parquet') { } compileOnly 'org.slf4j:slf4j-api' compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + compileOnly('com.github.ben-manes.caffeine:caffeine') testCompile 'junit:junit' testCompile 'org.slf4j:slf4j-simple' @@ -639,9 +729,15 @@ project(':iceberg-parquet') { } project(':iceberg-arrow') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { - compile project(':iceberg-api') - compile project(':iceberg-parquet') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-parquet', configuration: 'shadow') compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -661,20 +757,47 @@ project(':iceberg-arrow') { compile("org.apache.arrow:arrow-memory") { exclude group: 'io.netty', module: 'netty-common' } + compile("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } + } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' } } project(':iceberg-spark') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + apply plugin: 'scala' dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-arrow') - compile project(':iceberg-hive') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-common', configuration: 'shadow') + compile project(path: ':iceberg-core', configuration: 'shadow') + compile project(path: ':iceberg-orc', configuration: 'shadow') + compile project(path: ':iceberg-parquet', configuration: 'shadow') + compile project(path: ':iceberg-arrow', configuration: 'shadow') + compile project(path: ':iceberg-hive', configuration: 'shadow') compile('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead @@ -719,14 +842,35 @@ project(':iceberg-spark') { exclude group: 'org.apache.avro', module: 'avro' } } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + } } project(':iceberg-pig') { + apply plugin: 'com.github.johnrengelman.shadow' + + tasks.assemble.dependsOn tasks.shadowJar + tasks.install.dependsOn tasks.shadowJar + tasks.javadocJar.dependsOn tasks.shadowJar + dependencies { - compile project(':iceberg-api') - compile project(':iceberg-common') - compile project(':iceberg-core') - compile project(':iceberg-parquet') + compile project(path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-common', configuration: 'shadow') + compile project(path: ':iceberg-core', configuration: 'shadow') + compile project(path: ':iceberg-parquet', configuration: 'shadow') compile "org.apache.commons:commons-lang3" @@ -737,6 +881,12 @@ project(':iceberg-pig') { compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' } + compile("org.apache.parquet:parquet-avro") { + exclude group: 'org.apache.avro', module: 'avro' + // already shaded by Parquet + exclude group: 'it.unimi.dsi' + exclude group: 'org.codehaus.jackson' + } testCompile "org.apache.hadoop:hadoop-hdfs::tests" testCompile "org.apache.hadoop:hadoop-common::tests" @@ -745,6 +895,21 @@ project(':iceberg-pig') { } testCompile 'junit:junit' } + + shadowJar { + // shade compileOnly dependencies to avoid including in transitive dependencies + configurations = [project.configurations.compile] + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + } } // the runtime jar is a self-contained artifact for testing in a notebook diff --git a/versions.lock b/versions.lock index 62a01a3001a9..ff2edcdb73d5 100644 --- a/versions.lock +++ b/versions.lock @@ -27,7 +27,7 @@ com.google.code.gson:gson:2.2.4 (6 constraints: f44c3ddb) com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544) com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) -com.google.guava:guava:28.0-jre (41 constraints: 56500d9f) +com.google.guava:guava:28.0-jre (26 constraints: c387b8b2) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) com.google.inject:guice:3.0 (8 constraints: 2c93366d) com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce) From fff5e0e8443e3891b08693e241b5c6342bf490dd Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 30 Mar 2020 13:05:07 +0100 Subject: [PATCH 15/51] Fix the guava --- build.gradle | 60 +++++++------------ .../apache/iceberg/mr/IcebergInputFormat.java | 15 ++--- .../iceberg/mr/mapred/IcebergInputFormat.java | 9 ++- .../mr/mapred/IcebergReaderFactory.java | 3 +- .../mr/mapred/IcebergSchemaToTypeInfo.java | 31 ++++++---- .../mr/mapred/TestIcebergInputFormat.java | 15 ++++- versions.lock | 5 +- 7 files changed, 72 insertions(+), 66 deletions(-) diff --git a/build.gradle b/build.gradle index 0ee2063cb52b..60ddc2a50e40 100644 --- a/build.gradle +++ b/build.gradle @@ -173,7 +173,7 @@ project(':iceberg-core') { compile project(path: ':iceberg-api', configuration: 'shadow') compile project(path: ':iceberg-common', configuration: 'shadow') - compile('com.google.guava:guava:28.0-jre') { + compileOnly('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -242,7 +242,7 @@ project(':iceberg-data') { compileOnly project(path: ':iceberg-parquet', configuration: 'shadow') compileOnly project(path: ':iceberg-orc', configuration: 'shadow') - compile('com.google.guava:guava:28.0-jre') { + compileOnly('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -331,6 +331,8 @@ project(':iceberg-hive') { dependencies { compile project(path: ':iceberg-core', configuration: 'shadow') + compile project (path: ':iceberg-api', configuration: 'shadow') + compile project (path: ':iceberg-common', configuration: 'shadow') compileOnly 'org.slf4j:slf4j-api' compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' @@ -419,7 +421,7 @@ project(':iceberg-hive') { project(':iceberg-mr') { dependencies { - compile('com.google.guava:guava:11.0.2') + compile project (path: ':iceberg-api', configuration: 'shadow') compile project(path: ':iceberg-core', configuration: 'shadow') compile project(path: ':iceberg-orc', configuration: 'shadow') compile project(path: ':iceberg-parquet', configuration: 'shadow') @@ -429,6 +431,8 @@ project(':iceberg-mr') { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'com.google.guava', module: 'guava' } + compile group: 'com.esotericsoftware.kryo', name: 'kryo', version: '2.24.0' + compileOnly('com.github.ben-manes.caffeine:caffeine') compileOnly('org.apache.calcite:calcite-core:1.10.0') { @@ -476,7 +480,7 @@ project(':iceberg-mr') { exclude group: 'org.apache.calcite' exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' - exclude group: 'com.google.guava', module: 'guava' + } testCompile 'junit:junit' @@ -521,7 +525,7 @@ project(':iceberg-orc') { compile project(path: ':iceberg-api', configuration: 'shadow') compile project(path: ':iceberg-core', configuration: 'shadow') - compile('com.google.guava:guava:28.0-jre') { + compileOnly('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -606,8 +610,9 @@ project(':iceberg-parquet') { dependencies { compile project(path: ':iceberg-api', configuration: 'shadow') compile project(path: ':iceberg-core', configuration: 'shadow') + compile project(path: ':iceberg-common', configuration: 'shadow') - compile('com.google.guava:guava:28.0-jre') { + compileOnly('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -642,7 +647,7 @@ project(':iceberg-parquet') { shadowJar { // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compileOnly] + configurations = [project.configurations.compile] dependencies { exclude (dependency('commons-collections:commons-collections:3.2.2')) @@ -739,7 +744,7 @@ project(':iceberg-arrow') { compile project(path: ':iceberg-api', configuration: 'shadow') compile project(path: ':iceberg-parquet', configuration: 'shadow') - compile('com.google.guava:guava:28.0-jre') { + compileOnly('com.google.guava:guava:28.0-jre') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } @@ -782,27 +787,17 @@ project(':iceberg-arrow') { } project(':iceberg-spark') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - apply plugin: 'scala' dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-common', configuration: 'shadow') - compile project(path: ':iceberg-core', configuration: 'shadow') - compile project(path: ':iceberg-orc', configuration: 'shadow') - compile project(path: ':iceberg-parquet', configuration: 'shadow') - compile project(path: ':iceberg-arrow', configuration: 'shadow') - compile project(path: ':iceberg-hive', configuration: 'shadow') + compile project(':iceberg-api') + compile project(':iceberg-common') + compile project(':iceberg-core') + compile project(':iceberg-orc') + compile project(':iceberg-parquet') + compile project(':iceberg-arrow') + compile project(':iceberg-hive') - compile('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } compile 'org.slf4j:slf4j-api' compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' @@ -842,21 +837,6 @@ project(':iceberg-spark') { exclude group: 'org.apache.avro', module: 'avro' } } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - } } project(':iceberg-pig') { diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java index 637e14b8c5c8..8e7ff3d5b164 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java @@ -19,9 +19,6 @@ package org.apache.iceberg.mr; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import java.io.Closeable; import java.io.DataInput; import java.io.DataOutput; @@ -30,6 +27,9 @@ import java.util.Iterator; import java.util.List; import java.util.Set; + +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.compress.utils.Sets; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileSystem; @@ -287,9 +287,10 @@ private Iterator open(FileScanTask currentTask) { if (hasJoinedPartitionColumns) { readSchema = TypeUtil.selectNot(tableSchema, idColumns); Schema identityPartitionSchema = TypeUtil.select(tableSchema, idColumns); - return Iterators.transform( - open(currentTask, readSchema), - row -> withPartitionColumns(row, identityPartitionSchema, spec, file.partition())); + //return Iterators.transform( + //open(currentTask, readSchema), + //row -> withPartitionColumns(row, identityPartitionSchema, spec, file.partition())); + return (open( currentTask, readSchema)); } else { return open(currentTask, readSchema); } @@ -333,7 +334,7 @@ private T withPartitionColumns(T row, Schema identityPartitionSchema, PartitionS private static Record icebergRecordWithPartitionsColumns( Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { - List fields = Lists.newArrayList(record.struct().fields()); + List fields = Lists.newArrayList(); fields.addAll(identityPartitionSchema.asStruct().fields()); GenericRecord row = GenericRecord.create(Types.StructType.of(fields)); int size = record.struct().fields().size(); diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java index 567f0c52f3e7..9739fd5c000e 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java @@ -19,7 +19,6 @@ package org.iceberg.mr.mapred; -import com.google.common.collect.Lists; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -27,6 +26,9 @@ import java.net.URISyntaxException; import java.util.Iterator; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -64,7 +66,10 @@ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { } table = tables.load(location.getPath()); - List tasks = Lists.newArrayList(table.newScan().planTasks()); + CloseableIterable taskIterable = table.newScan().planTasks(); + List tasks = (List) StreamSupport + .stream(taskIterable.spliterator(), false) + .collect(Collectors.toList()); return createSplits(tasks); } diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java index f49b2ae411be..1b77d27e5dd7 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java @@ -25,6 +25,7 @@ import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataReader; +import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.orc.ORC; @@ -79,7 +80,7 @@ private CloseableIterable buildOrcReader(FileScanTask task, InputFile file, Sche private CloseableIterable buildParquetReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { Parquet.ReadBuilder builder = Parquet.read(file) - //.createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) + .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) .project(schema) .split(task.start(), task.length()); diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java index 23378f536fc5..91d06495f5e6 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -19,9 +19,13 @@ package org.iceberg.mr.mapred; -import com.google.common.collect.ImmutableMap; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Hashtable; import java.util.List; +import java.util.Map; + import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; @@ -38,17 +42,20 @@ final class IcebergSchemaToTypeInfo { private IcebergSchemaToTypeInfo() {} - private static final ImmutableMap primitiveTypeToTypeInfo = ImmutableMap.builder() - .put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BOOLEAN_TYPE_NAME)) - .put(Types.IntegerType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.INT_TYPE_NAME)) - .put(Types.LongType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BIGINT_TYPE_NAME)) - .put(Types.FloatType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.FLOAT_TYPE_NAME)) - .put(Types.DoubleType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DOUBLE_TYPE_NAME)) - .put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.BINARY_TYPE_NAME)) - .put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME)) - .put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.DATE_TYPE_NAME)) - .put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.TIMESTAMP_TYPE_NAME)) - .build(); + private static final Map primitiveTypeToTypeInfo = initTypeMap(); + private static Map initTypeMap() { + Map theMap = new Hashtable(); + theMap.put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo("boolean")); + theMap.put(Types.IntegerType.get(), TypeInfoFactory.getPrimitiveTypeInfo("int")); + theMap.put(Types.LongType.get(), TypeInfoFactory.getPrimitiveTypeInfo("bigint")); + theMap.put(Types.FloatType.get(), TypeInfoFactory.getPrimitiveTypeInfo("float")); + theMap.put(Types.DoubleType.get(), TypeInfoFactory.getPrimitiveTypeInfo("double")); + theMap.put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo("binary")); + theMap.put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo("string")); + theMap.put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo("date")); + theMap.put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo("timestamp")); + return Collections.unmodifiableMap(theMap); + } public static List getColumnTypes(Schema schema) throws Exception { List fields = schema.columns(); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 099830ae80af..104a13882413 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -20,20 +20,31 @@ package org.apache.iceberg.mr.mapred; import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.StandaloneHiveRunner; import com.klarna.hiverunner.annotations.HiveSQL; import java.io.File; import java.io.IOException; +import java.util.List; + +import org.apache.commons.compress.utils.Lists; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.data.Record; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.types.Types; +import org.iceberg.mr.mapred.IcebergInputFormat; +import org.iceberg.mr.mapred.IcebergWritable; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -82,7 +93,7 @@ public void bla() { LOG.error("ZZZ", t.getCause()); } } -/* + @Test public void testInputFormat() { shell.execute("CREATE DATABASE source_db"); @@ -133,7 +144,7 @@ public void testGetRecordReader() throws IOException { } } assertEquals(3, records.size()); - }*/ + } @After public void after() throws IOException { diff --git a/versions.lock b/versions.lock index ff2edcdb73d5..fc3aeb883c3d 100644 --- a/versions.lock +++ b/versions.lock @@ -13,6 +13,8 @@ com.carrotsearch:hppc:0.7.2 (1 constraints: f70cda14) com.clearspring.analytics:stream:2.7.0 (1 constraints: 1a0dd136) com.esotericsoftware:kryo-shaded:4.0.2 (2 constraints: b71345a6) com.esotericsoftware:minlog:1.3.0 (1 constraints: 670e7c4f) +com.esotericsoftware.kryo:kryo:2.24.0 (1 constraints: 3a053f3b) +com.esotericsoftware.minlog:minlog:1.2 (1 constraints: 650d3615) com.fasterxml.jackson.core:jackson-annotations:2.10.2 (6 constraints: d863d8d4) com.fasterxml.jackson.core:jackson-core:2.10.2 (7 constraints: 5261f057) com.fasterxml.jackson.core:jackson-databind:2.10.2 (12 constraints: 13ad1133) @@ -27,7 +29,6 @@ com.google.code.gson:gson:2.2.4 (6 constraints: f44c3ddb) com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544) com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) -com.google.guava:guava:28.0-jre (26 constraints: c387b8b2) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) com.google.inject:guice:3.0 (8 constraints: 2c93366d) com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce) @@ -263,7 +264,7 @@ org.mortbay.jetty:jsp-2.1:6.1.14 (2 constraints: 71154c11) org.mortbay.jetty:jsp-api-2.1:6.1.14 (3 constraints: 5b20fbe2) org.mortbay.jetty:servlet-api:2.5-20081211 (1 constraints: 390cbd19) org.mortbay.jetty:servlet-api-2.5:6.1.14 (3 constraints: c221f470) -org.objenesis:objenesis:2.5.1 (2 constraints: 19198bcb) +org.objenesis:objenesis:2.5.1 (3 constraints: 7d266d95) org.ow2.asm:asm-all:5.0.2 (1 constraints: 0d0ceaf6) org.pentaho:pentaho-aggdesigner-algorithm:5.1.5-jhyde (1 constraints: a40f6d84) org.roaringbitmap:RoaringBitmap:0.7.45 (2 constraints: 2e1c26e3) From ee213adead49adee6c8a5bfac4eaef4f29bbc890 Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 30 Mar 2020 14:25:30 +0100 Subject: [PATCH 16/51] Nuke jackson dependencies --- versions.lock | 5 ----- versions.props | 1 - 2 files changed, 6 deletions(-) diff --git a/versions.lock b/versions.lock index fc3aeb883c3d..ddc8ccace0b8 100644 --- a/versions.lock +++ b/versions.lock @@ -15,11 +15,6 @@ com.esotericsoftware:kryo-shaded:4.0.2 (2 constraints: b71345a6) com.esotericsoftware:minlog:1.3.0 (1 constraints: 670e7c4f) com.esotericsoftware.kryo:kryo:2.24.0 (1 constraints: 3a053f3b) com.esotericsoftware.minlog:minlog:1.2 (1 constraints: 650d3615) -com.fasterxml.jackson.core:jackson-annotations:2.10.2 (6 constraints: d863d8d4) -com.fasterxml.jackson.core:jackson-core:2.10.2 (7 constraints: 5261f057) -com.fasterxml.jackson.core:jackson-databind:2.10.2 (12 constraints: 13ad1133) -com.fasterxml.jackson.module:jackson-module-paranamer:2.10.2 (1 constraints: 03162c16) -com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.2 (1 constraints: 7f0da251) com.github.ben-manes.caffeine:caffeine:2.7.0 (1 constraints: 0b050a36) com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 (1 constraints: e90b08f3) com.github.luben:zstd-jni:1.3.2-2 (1 constraints: 760d7c51) diff --git a/versions.props b/versions.props index af2e21763a2d..cf0bf897dd6e 100644 --- a/versions.props +++ b/versions.props @@ -7,7 +7,6 @@ org.apache.spark:spark-hive_2.11 = 2.4.4 org.apache.spark:spark-avro_2.11 = 2.4.4 org.apache.pig:pig = 0.14.0 org.apache.commons:commons-lang3 = 3.9 -com.fasterxml.jackson.*:* = 2.10.0 com.github.ben-manes.caffeine:caffeine = 2.7.0 org.apache.arrow:arrow-vector = 0.14.1 From a0d07a7e98f5ac98b8ec4757eea3fd30527d2761 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 30 Mar 2020 15:33:48 +0100 Subject: [PATCH 17/51] remove test method --- .../iceberg/mr/mapred/TestIcebergInputFormat.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 104a13882413..a6e698e060b5 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -29,7 +29,6 @@ import java.io.File; import java.io.IOException; import java.util.List; - import org.apache.commons.compress.utils.Lists; import org.apache.commons.io.FileUtils; import org.apache.hadoop.mapred.InputSplit; @@ -83,17 +82,6 @@ public void before() throws IOException { table.newAppend().appendFile(fileA).commit(); } - @Test - public void bla() { - try { - LOG.error("YYY: " + org.apache.avro.Schema.class.getProtectionDomain().getCodeSource()); - } catch (Throwable t) { - t.printStackTrace(); - LOG.error("XXX", t); - LOG.error("ZZZ", t.getCause()); - } - } - @Test public void testInputFormat() { shell.execute("CREATE DATABASE source_db"); From 11296c334d133eb3720d64427b4b142744d1f2f3 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 30 Mar 2020 16:02:47 +0100 Subject: [PATCH 18/51] tidy up checkstyle --- baseline.gradle | 2 +- .../java/org/iceberg/mr/mapred/IcebergInputFormat.java | 1 - .../org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java | 5 ++--- .../apache/iceberg/mr/mapred/TestIcebergInputFormat.java | 7 +++---- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/baseline.gradle b/baseline.gradle index 0a21db74f113..ac9015f9cf75 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -33,7 +33,7 @@ subprojects { // ready to enforce linting on. apply plugin: 'org.inferred.processors' if (!project.hasProperty('quick')) { - //apply plugin: 'com.palantir.baseline-checkstyle' + apply plugin: 'com.palantir.baseline-checkstyle' //apply plugin: 'com.palantir.baseline-error-prone' } apply plugin: 'com.palantir.baseline-scalastyle' diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java index 9739fd5c000e..0b36c4a5ac26 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java @@ -28,7 +28,6 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; - import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java index 91d06495f5e6..554be063e449 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java @@ -21,11 +21,9 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; - import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; @@ -43,6 +41,7 @@ final class IcebergSchemaToTypeInfo { private IcebergSchemaToTypeInfo() {} private static final Map primitiveTypeToTypeInfo = initTypeMap(); + private static Map initTypeMap() { Map theMap = new Hashtable(); theMap.put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo("boolean")); @@ -68,7 +67,7 @@ public static List getColumnTypes(Schema schema) throws Exception { private static TypeInfo generateTypeInfo(Type type) throws Exception { if (primitiveTypeToTypeInfo.containsKey(type)) { - return (TypeInfo) primitiveTypeToTypeInfo.get(type); + return primitiveTypeToTypeInfo.get(type); } switch (type.typeId()) { case UUID: diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index a6e698e060b5..756964ff0b44 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -19,10 +19,6 @@ package org.apache.iceberg.mr.mapred; -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.StandaloneHiveRunner; import com.klarna.hiverunner.annotations.HiveSQL; @@ -51,6 +47,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; @RunWith(StandaloneHiveRunner.class) public class TestIcebergInputFormat { From 69cef2b2b4507fd8271eda1d9c20a7829a7660dd Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 30 Mar 2020 16:03:10 +0100 Subject: [PATCH 19/51] remove classes from mapreduce inputformat branch --- .../apache/iceberg/mr/IcebergInputFormat.java | 492 ------------------ .../apache/iceberg/mr/SerializationUtil.java | 79 --- .../iceberg/mr/TestIcebergInputFormat.java | 133 ----- 3 files changed, 704 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java delete mode 100644 mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java deleted file mode 100644 index 8e7ff3d5b164..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergInputFormat.java +++ /dev/null @@ -1,492 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr; - -import java.io.Closeable; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import org.apache.commons.compress.utils.Lists; -import org.apache.commons.compress.utils.Sets; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.iceberg.CombinedScanTask; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.FileScanTask; -import org.apache.iceberg.PartitionField; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.SchemaParser; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.TableScan; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.avro.DataReader; -import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.hadoop.HadoopInputFile; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.types.TypeUtil; -import org.apache.iceberg.types.Types; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class IcebergInputFormat extends InputFormat { - private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); - - static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; - static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; - static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; - static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; - static final String READ_SCHEMA = "iceberg.mr.read.schema"; - static final String REUSE_CONTAINERS = "iceberg.mr.case.sensitive"; - static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; - static final String SPLIT_SIZE = "iceberg.mr.split.size"; - static final String TABLE_PATH = "iceberg.mr.table.path"; - static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - private static final String LOCALITY = "iceberg.mr.locality"; - - private transient List splits; - - public enum InMemoryDataModel { - PIG, - HIVE, - DEFAULT // Default data model is of Iceberg Generics - } - - /** - * Configures the {@code Job} to use the {@code IcebergInputFormat} and - * returns a helper to add further configuration. - * - * @param job the {@code Job} to configure - */ - public static ConfigBuilder configure(Job job) { - job.setInputFormatClass(IcebergInputFormat.class); - return new ConfigBuilder(job.getConfiguration()); - } - - public static class ConfigBuilder { - private final Configuration conf; - - public ConfigBuilder(Configuration conf) { - this.conf = conf; - } - - public ConfigBuilder readFrom(String path) { - conf.set(TABLE_PATH, path); - Table table = getTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - return this; - } - - public ConfigBuilder filter(Expression expression) { - conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); - return this; - } - - public ConfigBuilder project(Schema schema) { - conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); - return this; - } - - public ConfigBuilder reuseContainers(boolean reuse) { - conf.setBoolean(REUSE_CONTAINERS, reuse); - return this; - } - - public ConfigBuilder caseSensitive(boolean caseSensitive) { - conf.setBoolean(CASE_SENSITIVE, caseSensitive); - return this; - } - - public ConfigBuilder snapshotId(long snapshotId) { - conf.setLong(SNAPSHOT_ID, snapshotId); - return this; - } - - public ConfigBuilder asOfTime(long asOfTime) { - conf.setLong(AS_OF_TIMESTAMP, asOfTime); - return this; - } - - public ConfigBuilder splitSize(long splitSize) { - conf.setLong(SPLIT_SIZE, splitSize); - return this; - } - - public ConfigBuilder locality(boolean localityPreferred) { - conf.setBoolean(LOCALITY, localityPreferred); - return this; - } - - public ConfigBuilder inMemoryDataModel(InMemoryDataModel inMemoryDataModel) { - conf.set(IN_MEMORY_DATA_MODEL, inMemoryDataModel.name()); - return this; - } - } - - @Override - public List getSplits(JobContext context) { - if (splits != null) { - LOG.info("Returning cached splits: {}", splits.size()); - return splits; - } - - Configuration conf = context.getConfiguration(); - Table table = getTable(conf); - TableScan scan = table.newScan() - .caseSensitive(conf.getBoolean(CASE_SENSITIVE, true)); - long snapshotId = conf.getLong(SNAPSHOT_ID, -1); - if (snapshotId != -1) { - scan = scan.useSnapshot(snapshotId); - } - long asOfTime = conf.getLong(AS_OF_TIMESTAMP, -1); - if (asOfTime != -1) { - scan = scan.asOfTime(asOfTime); - } - long splitSize = conf.getLong(SPLIT_SIZE, -1); - if (splitSize != -1) { - scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); - } - String schemaStr = conf.get(READ_SCHEMA); - if (schemaStr != null) { - scan.project(SchemaParser.fromJson(schemaStr)); - } - - // TODO add a filter parser to get rid of Serialization - Expression filterExpression = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); - if (filterExpression != null) { - scan = scan.filter(filterExpression); - } - - splits = Lists.newArrayList(); - try (CloseableIterable tasksIterable = scan.planTasks()) { - tasksIterable.forEach(task -> splits.add(new IcebergSplit(conf, task))); - } catch (IOException e) { - throw new RuntimeIOException(e, "Failed to close table scan: %s", scan); - } - - return splits; - } - - @Override - public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { - return new IcebergRecordReader<>(); - } - - private static final class IcebergRecordReader extends RecordReader { - private TaskAttemptContext context; - private Iterator tasks; - private Iterator currentIterator; - private T currentRow; - private Schema expectedSchema; - private Schema tableSchema; - private InMemoryDataModel inMemoryDataModel; - private Closeable currentCloseable; - private boolean reuseContainers; - private boolean caseSensitive; - - @Override - public void initialize(InputSplit split, TaskAttemptContext newContext) { - Configuration conf = newContext.getConfiguration(); - CombinedScanTask task = ((IcebergSplit) split).task; - this.context = newContext; - this.tasks = task.files().iterator(); - this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); - String readSchemaStr = conf.get(READ_SCHEMA); - if (readSchemaStr != null) { - this.expectedSchema = SchemaParser.fromJson(readSchemaStr); - } - this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); - this.caseSensitive = conf.getBoolean(CASE_SENSITIVE, true); - this.inMemoryDataModel = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.DEFAULT); - this.currentIterator = open(tasks.next()); - } - - @Override - public boolean nextKeyValue() throws IOException { - while (true) { - if (currentIterator.hasNext()) { - currentRow = currentIterator.next(); - return true; - } else if (tasks.hasNext()) { - currentCloseable.close(); - currentIterator = open(tasks.next()); - } else { - return false; - } - } - } - - @Override - public Void getCurrentKey() { - return null; - } - - @Override - public T getCurrentValue() { - return currentRow; - } - - @Override - public float getProgress() { - return context.getProgress(); - } - - @Override - public void close() throws IOException { - currentCloseable.close(); - } - - private Iterator open(FileScanTask currentTask) { - DataFile file = currentTask.file(); - // schema of rows returned by readers - PartitionSpec spec = currentTask.spec(); - Set idColumns = spec.identitySourceIds(); - Schema readSchema = expectedSchema != null ? expectedSchema : tableSchema; - boolean hasJoinedPartitionColumns = !idColumns.isEmpty(); - if (hasJoinedPartitionColumns) { - readSchema = TypeUtil.selectNot(tableSchema, idColumns); - Schema identityPartitionSchema = TypeUtil.select(tableSchema, idColumns); - //return Iterators.transform( - //open(currentTask, readSchema), - //row -> withPartitionColumns(row, identityPartitionSchema, spec, file.partition())); - return (open( currentTask, readSchema)); - } else { - return open(currentTask, readSchema); - } - } - - private Iterator open(FileScanTask currentTask, Schema readSchema) { - DataFile file = currentTask.file(); - // TODO should we somehow make use of FileIO to create inputFile? - InputFile inputFile = HadoopInputFile.fromLocation(file.path(), context.getConfiguration()); - CloseableIterable iterable; - switch (file.format()) { - case AVRO: - iterable = newAvroIterable(inputFile, currentTask, readSchema); - break; - case ORC: - iterable = newOrcIterable(inputFile, currentTask, readSchema); - break; - case PARQUET: - iterable = newParquetIterable(inputFile, currentTask, readSchema); - break; - default: - throw new UnsupportedOperationException( - String.format("Cannot read %s file: %s", file.format().name(), file.path())); - } - currentCloseable = iterable; - return iterable.iterator(); - } - - @SuppressWarnings("unchecked") - private T withPartitionColumns(T row, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { - switch (inMemoryDataModel) { - case PIG: - case HIVE: - // TODO implement adding partition columns to records for Pig and Hive - throw new UnsupportedOperationException(); - case DEFAULT: - return (T) icebergRecordWithPartitionsColumns((Record) row, identityPartitionSchema, spec, partition); - } - return row; - } - - private static Record icebergRecordWithPartitionsColumns( - Record record, Schema identityPartitionSchema, PartitionSpec spec, StructLike partition) { - List fields = Lists.newArrayList(); - fields.addAll(identityPartitionSchema.asStruct().fields()); - GenericRecord row = GenericRecord.create(Types.StructType.of(fields)); - int size = record.struct().fields().size(); - for (int i = 0; i < size; i++) { - row.set(i, record.get(i)); - } - List partitionFields = spec.fields(); - List identityColumns = identityPartitionSchema.columns(); - for (int i = 0; i < identityColumns.size(); i++) { - Types.NestedField identityColumn = identityColumns.get(i); - - for (int j = 0; j < partitionFields.size(); j++) { - PartitionField partitionField = partitionFields.get(j); - if (identityColumn.fieldId() == partitionField.sourceId() && - "identity".equals(partitionField.transform().toString())) { - row.set(size + i, partition.get(j, spec.javaClasses()[i])); - } - } - } - return row; - } - - private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { - Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) - .project(readSchema) - .split(task.start(), task.length()); - - if (reuseContainers) { - avroReadBuilder.reuseContainers(); - } - - switch (inMemoryDataModel) { - case PIG: - case HIVE: - //TODO implement value readers for Pig and Hive - throw new UnsupportedOperationException(); - case DEFAULT: - avroReadBuilder.createReaderFunc(DataReader::create); - } - return avroReadBuilder.build(); - } - - private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { - Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) - .project(readSchema) - .filter(task.residual()) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); - if (reuseContainers) { - parquetReadBuilder.reuseContainers(); - } - - switch (inMemoryDataModel) { - case PIG: - case HIVE: - //TODO implement value readers for Pig and Hive - throw new UnsupportedOperationException(); - case DEFAULT: - //parquetReadBuilder.createReaderFunc( - //fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); - } - return parquetReadBuilder.build(); - } - - private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { - ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) - .schema(readSchema) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); - // ORC does not support reuse containers yet - switch (inMemoryDataModel) { - case PIG: - case HIVE: - //TODO implement value readers for Pig and Hive - throw new UnsupportedOperationException(); - case DEFAULT: - //TODO: We do not have support for Iceberg generics for ORC - throw new UnsupportedOperationException(); - } - - return orcReadBuilder.build(); - } - } - - private static Table getTable(Configuration conf) { - String path = conf.get(TABLE_PATH); - if (path.contains("/")) { - HadoopTables tables = new HadoopTables(conf); - return tables.load(path); - } else { - //Catalog catalog = HiveCatalogs.loadCatalog(conf); - //TableIdentifier tableIdentifier = TableIdentifier.parse(path); - //return catalog.loadTable(tableIdentifier); - return null; - } - } - - private static class IcebergSplit extends InputSplit implements Writable { - private static final String[] ANYWHERE = new String[]{"*"}; - private CombinedScanTask task; - private transient String[] locations; - private transient Configuration conf; - - IcebergSplit(Configuration conf, CombinedScanTask task) { - this.task = task; - this.conf = conf; - } - - @Override - public long getLength() { - return task.files().stream().mapToLong(FileScanTask::length).sum(); - } - - @Override - public String[] getLocations() { - boolean localityPreferred = conf.getBoolean(LOCALITY, false); - if (!localityPreferred) { - return ANYWHERE; - } - if (locations != null) { - return locations; - } - - Set locationSets = Sets.newHashSet(); - for (FileScanTask f : task.files()) { - Path path = new Path(f.file().path().toString()); - try { - FileSystem fs = path.getFileSystem(conf); - for (BlockLocation b : fs.getFileBlockLocations(path, f.start(), f.length())) { - locationSets.addAll(Arrays.asList(b.getHosts())); - } - } catch (IOException ioe) { - LOG.warn("Failed to get block locations for path {}", path, ioe); - } - } - - locations = locationSets.toArray(new String[0]); - return locations; - } - - @Override - public void write(DataOutput out) throws IOException { - byte[] data = SerializationUtil.serializeToBytes(this.task); - out.writeInt(data.length); - out.write(data); - } - - @Override - public void readFields(DataInput in) throws IOException { - byte[] data = new byte[in.readInt()]; - in.readFully(data); - this.task = SerializationUtil.deserializeFromBytes(data); - } - } -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java b/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java deleted file mode 100644 index af31eb26789f..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/SerializationUtil.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; -import org.apache.iceberg.exceptions.RuntimeIOException; - - -public class SerializationUtil { - - private SerializationUtil() { - } - - public static byte[] serializeToBytes(Object obj) { - try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); - GZIPOutputStream gos = new GZIPOutputStream(baos); - ObjectOutputStream oos = new ObjectOutputStream(gos)) { - oos.writeObject(obj); - return baos.toByteArray(); - } catch (IOException e) { - throw new RuntimeIOException("Failed to serialize object", e); - } - } - - @SuppressWarnings("unchecked") - public static T deserializeFromBytes(byte[] bytes) { - if (bytes == null) { - return null; - } - - try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - GZIPInputStream gis = new GZIPInputStream(bais); - ObjectInputStream ois = new ObjectInputStream(gis)) { - return (T) ois.readObject(); - } catch (IOException e) { - throw new RuntimeIOException("Failed to deserialize object", e); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Could not read object ", e); - } - } - - public static String serializeToBase64(Object obj) { - byte[] bytes = serializeToBytes(obj); - return new String(Base64.getMimeEncoder().encode(bytes), StandardCharsets.UTF_8); - } - - public static T deserializeFromBase64(String base64) { - if (base64 == null) { - return null; - } - byte[] bytes = Base64.getMimeDecoder().decode(base64.getBytes(StandardCharsets.UTF_8)); - return deserializeFromBytes(bytes); - } -} diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java deleted file mode 100644 index d1c389176e2e..000000000000 --- a/mr/src/test/java/org/apache/iceberg/mr/TestIcebergInputFormat.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import org.apache.avro.generic.GenericData; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; -import org.apache.iceberg.Files; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.Table; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.avro.RandomAvroData; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.apache.iceberg.types.Types.NestedField.required; - - -public class TestIcebergInputFormat { - private static final Configuration CONF = new Configuration(); - private static final HadoopTables TABLES = new HadoopTables(CONF); - - private File tableLocation; - - private static final Schema SCHEMA = new Schema( - required(1, "id", Types.LongType.get()), - optional(2, "data", Types.StringType.get()), - required(3, "date", Types.StringType.get())); - - private static final PartitionSpec PARTITION_BY_DATE = PartitionSpec - .builderFor(SCHEMA) - .identity("date") - .build(); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - IcebergInputFormat icebergInputFormat; - - @Test - public void test() throws IOException, InterruptedException { - tableLocation = new File(temp.newFolder(), "table"); - Table table = TABLES.create(SCHEMA, PARTITION_BY_DATE, tableLocation.toString()); - List records = RandomAvroData.generate(SCHEMA, 5, 0L); - File file = temp.newFile(); - Assert.assertTrue(file.delete()); - try (FileAppender appender = Avro.write(Files.localOutput(file)) - .schema(SCHEMA) - .named("avro") - .build()) { - appender.addAll(records); - } - - DataFile dataFile = DataFiles.builder(PARTITION_BY_DATE) - .withPartition(partitionData("2020-03-15")) - .withRecordCount(records.size()) - .withFileSizeInBytes(file.length()) - .withPath(file.toString()) - .withFormat("avro") - .build(); - - table.newAppend().appendFile(dataFile).commit(); - - Job job = Job.getInstance(new Configuration()); - IcebergInputFormat - .configure(job) - .readFrom(tableLocation.getAbsolutePath()); - - TaskAttemptContext context = new TaskAttemptContextImpl(new JobConf(job.getConfiguration()), new TaskAttemptID()); - icebergInputFormat = new IcebergInputFormat<>(); - List splits = icebergInputFormat.getSplits(context); - final RecordReader recordReader = - icebergInputFormat.createRecordReader(splits.get(0), context); - recordReader.initialize(splits.get(0), context); - while (recordReader.nextKeyValue()) { - System.out.println(recordReader.getCurrentValue()); - } - } - - private StructLike partitionData(String date) { - return new StructLike() { - - @Override - public int size() { - return 1; - } - - @Override - public T get(int pos, Class javaClass) { - return (T) date; - } - - @Override - public void set(int pos, T value) { - } - }; - } -} From a08c087469419be7ae99aefaa1ef17ff92cd8287 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 30 Mar 2020 16:42:29 +0100 Subject: [PATCH 20/51] revert baseline plugin version (no idea why it was failing earlier) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 60ddc2a50e40..6bc0a5c57bfe 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ buildscript { } dependencies { classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0' - classpath 'com.palantir.baseline:gradle-baseline-java:0.58.0' + classpath 'com.palantir.baseline:gradle-baseline-java:0.55.0' classpath 'com.diffplug.spotless:spotless-plugin-gradle:3.14.0' classpath 'gradle.plugin.org.inferred:gradle-processors:2.1.0' classpath 'me.champeau.gradle:jmh-gradle-plugin:0.4.8' From d4cff1c4855993818d510c65181302d3d0be6805 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 30 Mar 2020 16:52:49 +0100 Subject: [PATCH 21/51] re-enable error-prone plugin --- baseline.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baseline.gradle b/baseline.gradle index ac9015f9cf75..0e755958591e 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -34,7 +34,7 @@ subprojects { apply plugin: 'org.inferred.processors' if (!project.hasProperty('quick')) { apply plugin: 'com.palantir.baseline-checkstyle' - //apply plugin: 'com.palantir.baseline-error-prone' + apply plugin: 'com.palantir.baseline-error-prone' } apply plugin: 'com.palantir.baseline-scalastyle' //apply plugin: 'com.palantir.baseline-class-uniqueness' From 72a04a8b9b7605da64685f76cf801f8abf4acdb1 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 30 Mar 2020 17:16:15 +0100 Subject: [PATCH 22/51] tidy up dependency scopes --- build.gradle | 49 ++++++++++++++----------------------------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/build.gradle b/build.gradle index 6bc0a5c57bfe..b2a722f76367 100644 --- a/build.gradle +++ b/build.gradle @@ -433,13 +433,12 @@ project(':iceberg-mr') { } compile group: 'com.esotericsoftware.kryo', name: 'kryo', version: '2.24.0' - compileOnly('com.github.ben-manes.caffeine:caffeine') compileOnly('org.apache.calcite:calcite-core:1.10.0') { exclude group: 'com.google.guava', module: 'guava' } - compile("org.apache.hive:hive-serde:2.3.6") { + compileOnly("org.apache.hive:hive-serde:2.3.6") { exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' exclude group: 'org.apache.ant', module: '*' exclude group: 'javax.servlet', module: 'jsp-api' @@ -456,18 +455,7 @@ project(':iceberg-mr') { exclude group: 'com.google.guava', module: 'guava' } - testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - - testCompile("org.apache.hive:hive-service:2.3.6") { - exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.hive', module: 'hive-exec' - exclude group: 'com.google.guava', module: 'guava' - } - - compile("org.apache.hive:hive-exec:2.3.6:core") { + compileOnly("org.apache.hive:hive-exec:2.3.6:core") { exclude group: 'stax', module: 'stax-api' exclude group: 'commons-collections', module: 'commons-collections' exclude group: 'org.apache.ant', module: '*' @@ -480,9 +468,19 @@ project(':iceberg-mr') { exclude group: 'org.apache.calcite' exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' - } + + testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') + testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') + testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') + testCompile("org.apache.hive:hive-service:2.3.6") { + exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.hive', module: 'hive-exec' + exclude group: 'com.google.guava', module: 'guava' + } + testCompile 'junit:junit' testCompile("com.klarna:hiverunner:4.1.0") { exclude group: 'com.google.protobuf', module: 'protobuf-java' @@ -491,26 +489,7 @@ project(':iceberg-mr') { exclude group: 'javax.jms', module: 'jms' exclude group: 'org.apache.hive', module: '*' exclude group: 'com.google.guava', module: 'guava' - } - testCompile("org.apache.hive:hive-exec:2.3.6:core") { - exclude group: 'stax', module: 'stax-api' - exclude group: 'commons-collections', module: 'commons-collections' - exclude group: 'org.apache.ant', module: '*' - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.pentaho' // missing dependency - exclude group: 'org.apache.hive', module: 'hive-llap-tez' - exclude group: 'org.apache.logging.log4j' - exclude group: 'com.google.protobuf', module: 'protobuf-java' - exclude group: 'org.apache.calcite' - exclude group: 'org.apache.calcite.avatica' - exclude group: 'com.google.code.findbugs', module: 'jsr305' - exclude group: 'com.google.guava', module: 'guava' - } - } - task copyToLib(type: Copy) { - into "$buildDir/output/lib" - from configurations.testCompile + } } } From d57037ab2c84746c6e1b28a448094b4031cb4d71 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 8 Apr 2020 14:42:55 +0100 Subject: [PATCH 23/51] fix build after merge --- build.gradle | 24 +++---------------- .../mr/mapred/IcebergReaderFactory.java | 2 +- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index ff7e72ba1d85..c3f453f68d8d 100644 --- a/build.gradle +++ b/build.gradle @@ -422,7 +422,7 @@ project(':iceberg-hive') { project(':iceberg-mr') { dependencies { - compile project (path: ':iceberg-api', configuration: 'shadow') + compile project(path: ':iceberg-api', configuration: 'shadow') compile project(path: ':iceberg-core', configuration: 'shadow') compile project(path: ':iceberg-orc', configuration: 'shadow') compile project(path: ':iceberg-parquet', configuration: 'shadow') @@ -470,8 +470,8 @@ project(':iceberg-mr') { exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' } - - testCompile project(path: ':iceberg-hive', configuration: 'testArtifacts') + + testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') @@ -494,24 +494,6 @@ project(':iceberg-mr') { } } -project(':iceberg-mr') { - dependencies { - compile project(':iceberg-api') - compile project(':iceberg-core') - compile project(':iceberg-orc') - compile project(':iceberg-parquet') - compile project(':iceberg-data') - - compileOnly("org.apache.hadoop:hadoop-client") { - exclude group: 'org.apache.avro', module: 'avro' - } - - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') - testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - } -} - project(':iceberg-orc') { apply plugin: 'com.github.johnrengelman.shadow' diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java index 1b77d27e5dd7..e653787c9ed3 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java @@ -70,7 +70,7 @@ private CloseableIterable buildAvroReader(FileScanTask task, InputFile file, Sch private CloseableIterable buildOrcReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { ORC.ReadBuilder builder = ORC.read(file) // .createReaderFunc() // FIXME: implement - .schema(schema) + .project(schema) .split(task.start(), task.length()); return builder.build(); From f135b78465eec5ff1b3d07a63f150c0c02b438c5 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 9 Apr 2020 12:29:47 +0100 Subject: [PATCH 24/51] trim down to non-hive related classes --- baseline.gradle | 2 +- build.gradle | 579 ++---------------- .../iceberg/mr/mapred/IcebergInputFormat.java | 2 +- .../mr/mapred/IcebergReaderFactory.java | 2 +- .../iceberg/mr/mapred/IcebergWritable.java | 2 +- .../IcebergObjectInspectorGenerator.java | 86 --- .../mr/mapred/IcebergSchemaToTypeInfo.java | 122 ---- .../org/iceberg/mr/mapred/IcebergSerDe.java | 93 --- .../mr/mapred/TestIcebergInputFormat.java | 36 +- versions.lock | 248 +++----- versions.props | 5 +- 11 files changed, 135 insertions(+), 1042 deletions(-) rename mr/src/main/java/org/{ => apache}/iceberg/mr/mapred/IcebergInputFormat.java (99%) rename mr/src/main/java/org/{ => apache}/iceberg/mr/mapred/IcebergReaderFactory.java (98%) rename mr/src/main/java/org/{ => apache}/iceberg/mr/mapred/IcebergWritable.java (97%) delete mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java delete mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java delete mode 100644 mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java diff --git a/baseline.gradle b/baseline.gradle index 0e755958591e..b30d6b508873 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -37,7 +37,7 @@ subprojects { apply plugin: 'com.palantir.baseline-error-prone' } apply plugin: 'com.palantir.baseline-scalastyle' - //apply plugin: 'com.palantir.baseline-class-uniqueness' + apply plugin: 'com.palantir.baseline-class-uniqueness' apply plugin: 'com.palantir.baseline-reproducibility' apply plugin: 'com.palantir.baseline-exact-dependencies' apply plugin: 'com.palantir.baseline-release-compatibility' diff --git a/build.gradle b/build.gradle index c3f453f68d8d..3820e1677cb6 100644 --- a/build.gradle +++ b/build.gradle @@ -81,109 +81,39 @@ subprojects { sourceCompatibility = '1.8' targetCompatibility = '1.8' -} - -apply from: 'baseline.gradle' -apply from: 'deploy.gradle' -apply from: 'tasks.gradle' -apply from: 'jmh.gradle' - -project(':iceberg-api') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar dependencies { - compile('com.google.guava:guava:28.0-jre') { + compile 'org.slf4j:slf4j-api' + compile('com.google.guava:guava') { // may be LGPL - use ALv2 findbugs-annotations instead exclude group: 'com.google.code.findbugs' } - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' + compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' testCompile 'junit:junit' testCompile 'org.slf4j:slf4j-simple' testCompile 'org.mockito:mockito-core' - - testCompile "org.apache.avro:avro" - testCompile 'joda-time:joda-time' - } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' } } -project(':iceberg-common') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar +apply from: 'baseline.gradle' +apply from: 'deploy.gradle' +apply from: 'tasks.gradle' +apply from: 'jmh.gradle' +project(':iceberg-api') { dependencies { - compile('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' - } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + testCompile "org.apache.avro:avro" + testCompile 'joda-time:joda-time' } } -project(':iceberg-core') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar +project(':iceberg-common') {} +project(':iceberg-core') { dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-common', configuration: 'shadow') - - compileOnly('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' + compile project(':iceberg-api') + compile project(':iceberg-common') compile("org.apache.avro:avro") { exclude group: 'org.tukaani' // xz compression is not supported @@ -199,83 +129,19 @@ project(':iceberg-core') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - - dependencies { - exclude (dependency('org.apache.avro:avro')) - exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) - exclude (dependency('com.fasterxml.jackson.core:jackson-core')) - exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) - exclude (dependency('org.checkerframework:checker-qual')) - exclude (dependency('com.github.ben-manes.caffeine:caffeine')) - exclude (dependency('org.slf4j:slf4j-api')) - exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) - exclude (dependency('org.apache.commons:commons-compress')) - } - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' - } } project(':iceberg-data') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-core', configuration: 'shadow') - compileOnly project(path: ':iceberg-parquet', configuration: 'shadow') - compileOnly project(path: ':iceberg-orc', configuration: 'shadow') - - compileOnly('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' - + compile project(':iceberg-api') + compile project(':iceberg-core') + compileOnly project(':iceberg-parquet') + compileOnly project(':iceberg-orc') compileOnly("org.apache.hadoop:hadoop-common") { exclude group: 'commons-beanutils' exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' } - compile("org.apache.orc:orc-core::nohive") { - exclude group: 'org.apache.hadoop' - exclude group: 'commons-lang' - // These artifacts are shaded and included in the orc-core fat jar - exclude group: 'com.google.protobuf', module: 'protobuf-java' - exclude group: 'org.apache.hive', module: 'hive-storage-api' - } - - compile("org.apache.parquet:parquet-avro") { - exclude group: 'org.apache.avro', module: 'avro' - // already shaded by Parquet - exclude group: 'it.unimi.dsi' - exclude group: 'org.codehaus.jackson' - } - - compileOnly "org.apache.avro:avro" - compileOnly('com.github.ben-manes.caffeine:caffeine') testCompile("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' @@ -289,65 +155,15 @@ project(':iceberg-data') { // Only for TestSplitScan as of Gradle 5.0+ maxHeapSize '1500m' } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - - dependencies { - exclude (dependency('org.apache.avro:avro')) - exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) - exclude (dependency('com.fasterxml.jackson.core:jackson-core')) - exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) - exclude (dependency('org.checkerframework:checker-qual')) - exclude (dependency('com.github.ben-manes.caffeine:caffeine')) - exclude (dependency('org.slf4j:slf4j-api')) - exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) - exclude (dependency('org.apache.commons:commons-compress')) - exclude (dependency('org.apache.iceberg:iceberg-common')) - exclude (dependency('org.apache.iceberg:iceberg-api')) - exclude (dependency('org.apache.iceberg:iceberg-core')) - } - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' - } } project(':iceberg-hive') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - dependencies { - compile project(path: ':iceberg-core', configuration: 'shadow') - compile project (path: ':iceberg-api', configuration: 'shadow') - compile project (path: ':iceberg-common', configuration: 'shadow') - - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' + compile project(':iceberg-core') compileOnly "org.apache.avro:avro" - compileOnly('com.github.ben-manes.caffeine:caffeine') - - compileOnly("org.apache.hive:hive-metastore:2.3.6") { - exclude group: 'org.apache.hive', module: 'hive-exec' + compileOnly("org.apache.hive:hive-metastore") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -367,7 +183,7 @@ project(':iceberg-hive') { // that's really old. We use the core classifier to be able to override our guava // version. Luckily, hive-exec seems to work okay so far with this version of guava // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context. - testCompile("org.apache.hive:hive-exec:2.3.6:core") { + testCompile("org.apache.hive:hive-exec::core") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency @@ -379,9 +195,8 @@ project(':iceberg-hive') { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testCompile("org.apache.hive:hive-metastore:2.3.6") { + testCompile("org.apache.hive:hive-metastore") { exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.pentaho' // missing dependency exclude group: 'org.apache.hbase' @@ -403,118 +218,30 @@ project(':iceberg-hive') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - } } project(':iceberg-mr') { dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-core', configuration: 'shadow') - compile project(path: ':iceberg-orc', configuration: 'shadow') - compile project(path: ':iceberg-parquet', configuration: 'shadow') - compile project(path: ':iceberg-data', configuration: 'shadow') - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.hadoop:hadoop-client") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'com.google.guava', module: 'guava' - } - compile group: 'com.esotericsoftware.kryo', name: 'kryo', version: '2.24.0' - - compileOnly('com.github.ben-manes.caffeine:caffeine') - compileOnly('org.apache.calcite:calcite-core:1.10.0') { - exclude group: 'com.google.guava', module: 'guava' - } + compile project(':iceberg-api') + compile project(':iceberg-core') + compile project(':iceberg-orc') + compile project(':iceberg-parquet') + compile project(':iceberg-data') - compileOnly("org.apache.hive:hive-serde:2.3.6") { - exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' - exclude group: 'org.apache.ant', module: '*' - exclude group: 'javax.servlet', module: 'jsp-api' - exclude group: 'commons-beanutils', module: 'commons-beanutils-core' - exclude group: 'javax.annotation', module: '*' - exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet' - exclude group: 'org.apache.logging.log4j', module: 'log4j-1.2-api' - exclude group: 'commons-beanutils', module: 'commons-beanutils' - exclude group: 'org.apache.geronimo.specs', module: 'geronimo-annotation_1.0_spec' - exclude group: 'org.checkerframework', module: 'checker-qual' - exclude group: 'org.codehaus.mojo', module: 'animal-sniffer-annotations' - exclude group: 'commons-collections', module: 'commons-collections' - exclude group: 'org.apache.hive', module: 'hive-exec' - exclude group: 'com.google.guava', module: 'guava' - } - - compileOnly("org.apache.hive:hive-exec:2.3.6:core") { - exclude group: 'stax', module: 'stax-api' - exclude group: 'commons-collections', module: 'commons-collections' - exclude group: 'org.apache.ant', module: '*' + compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.pentaho' // missing dependency - exclude group: 'org.apache.hive', module: 'hive-llap-tez' - exclude group: 'org.apache.logging.log4j' - exclude group: 'com.google.protobuf', module: 'protobuf-java' - exclude group: 'org.apache.calcite' - exclude group: 'org.apache.calcite.avatica' - exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') + testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') - - testCompile("org.apache.hive:hive-service:2.3.6") { - exclude group: 'org.apache.parquet', module:'parquet-hadoop-bundle' - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.hive', module: 'hive-exec' - exclude group: 'com.google.guava', module: 'guava' - } - - testCompile 'junit:junit' - testCompile("com.klarna:hiverunner:4.1.0") { - exclude group: 'com.google.protobuf', module: 'protobuf-java' - exclude group: 'org.apache.calcite', module: '*' - exclude group: 'org.codehaus.jettison', module: 'jettison' - exclude group: 'javax.jms', module: 'jms' - exclude group: 'org.apache.hive', module: '*' - exclude group: 'com.google.guava', module: 'guava' - } } } project(':iceberg-orc') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-core', configuration: 'shadow') - - compileOnly('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' + compile project(':iceberg-api') + compile project(':iceberg-core') compile("org.apache.orc:orc-core::nohive") { exclude group: 'org.apache.hadoop' @@ -535,74 +262,12 @@ project(':iceberg-orc') { testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') } - - configurations { - compile { - exclude group: 'javax.xml.bind' - } - } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - - zip64 true - - dependencies { - exclude (dependency('com.fasterxml.jackson.core:jackson-core')) - exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) - exclude (dependency('org.apache.iceberg:iceberg-common')) - exclude (dependency('org.apache.avro:avro')) - exclude (dependency('org.jetbrains:annotations')) - exclude (dependency('org.checkerframework:checker-qual')) - exclude (dependency('io.airlift:aircompressor')) - exclude (dependency('com.github.ben-manes.caffeine:caffeine')) - exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) - exclude (dependency('org.apache.commons:commons-compress')) - exclude (dependency('org.apache.orc:orc-shims')) - exclude (dependency('org.apache.iceberg:iceberg-core')) - exclude (dependency('org.slf4j:slf4j-api')) - exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) - exclude (dependency('org.apache.orc:orc-core')) - exclude (dependency('org.apache.iceberg:iceberg-api')) - - } - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' - } } project(':iceberg-parquet') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-core', configuration: 'shadow') - compile project(path: ':iceberg-common', configuration: 'shadow') - - compileOnly('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } - compileOnly 'org.slf4j:slf4j-api' - compileOnly 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - compileOnly('com.github.ben-manes.caffeine:caffeine') - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' + compile project(':iceberg-api') + compile project(':iceberg-core') compile("org.apache.parquet:parquet-avro") { exclude group: 'org.apache.avro', module: 'avro' @@ -618,122 +283,12 @@ project(':iceberg-parquet') { testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') } - - configurations { - compile { - exclude group: 'javax.xml.bind' - } - } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - - dependencies { - exclude (dependency('commons-collections:commons-collections:3.2.2')) - exclude (dependency('org.apache.hadoop:hadoop-yarn-api')) - exclude (dependency('org.apache.htrace:htrace-core')) - exclude (dependency('com.sun.xml.bind:jaxb-impl')) - exclude (dependency('org.apache.directory.server:apacheds-i18n')) - exclude (dependency('com.fasterxml.jackson.core:jackson-core')) - exclude (dependency('xmlenc:xmlenc')) - exclude (dependency('org.codehaus.jackson:jackson-mapper-asl')) - exclude (dependency('org.sonatype.sisu.inject:cglib')) - exclude (dependency('commons-codec:commons-codec')) - exclude (dependency('org.slf4j:slf4j-api')) - exclude (dependency('com.sun.jersey:jersey-json')) - exclude (dependency('org.apache.directory.api:api-util')) - exclude (dependency('com.fasterxml.jackson.core:jackson-databind')) - exclude (dependency('org.apache.commons:commons-math3')) - exclude (dependency('org.codehaus.jackson:jackson-xc')) - exclude (dependency('org.apache.directory.api:api-asn1-api')) - exclude (dependency('org.apache.hadoop:hadoop-yarn-common')) - exclude (dependency('commons-configuration:commons-configuration')) - exclude (dependency('org.apache.commons:commons-compress')) - exclude (dependency('javax.xml.bind:jaxb-api')) - exclude (dependency('org.apache.hadoop:hadoop-auth')) - exclude (dependency('commons-lang:commons-lang')) - exclude (dependency('org.apache.curator:curator-client')) - exclude (dependency('org.apache.hadoop:hadoop-common')) - exclude (dependency('com.sun.jersey:jersey-client')) - exclude (dependency('com.sun.jersey:jersey-core')) - exclude (dependency('javax.servlet:servlet-api')) - exclude (dependency('org.checkerframework:checker-qual')) - exclude (dependency('io.netty:netty')) - exclude (dependency(' org.apache.httpcomponents:httpcore')) - exclude (dependency('org.apache.avro:avro')) - exclude (dependency('javax.inject:javax.inject')) - exclude (dependency('log4j:log4j')) - exclude (dependency('org.codehaus.jackson:jackson-jaxrs')) - exclude (dependency('jline:jline')) - exclude (dependency('org.apache.directory.server:apacheds-kerberos-codec')) - exclude (dependency('aopalliance:aopalliance')) - exclude (dependency('asm:asm')) - exclude (dependency('commons-httpclient:commons-httpclient')) - exclude (dependency('commons-collections:commons-collections')) - exclude (dependency('commons-io:commons-io')) - exclude (dependency('com.fasterxml.jackson.core:jackson-annotations')) - exclude (dependency('org.apache.hadoop:hadoop-annotations')) - exclude (dependency('org.apache.zookeeper:zookeeper')) - exclude (dependency('org.codehaus.jackson:jackson-core-asl')) - exclude (dependency('org.apache.httpcomponents:httpcore')) - exclude (dependency('org.codehaus.mojo:animal-sniffer-annotations')) - exclude (dependency('org.fusesource.leveldbjni:leveldbjni-all')) - exclude (dependency('com.sun.jersey:jersey-server')) - exclude (dependency('commons-logging:commons-logging')) - exclude (dependency('javax.activation:activation')) - exclude (dependency('org.apache.httpcomponents:httpclient')) - exclude (dependency('org.mortbay.jetty:jetty-util')) - exclude (dependency('org.apache.curator:curator-recipes')) - exclude (dependency('commons-cli:commons-cli')) - exclude (dependency('com.sun.jersey.contribs:jersey-guice')) - exclude (dependency('commons-collections:commons-collections')) - exclude (dependency('com.google.code.findbugs:jsr305')) - exclude (dependency('commons-digester:commons-digester')) - exclude (dependency('jline:jline')) - exclude (dependency('org.codehaus.jettison:jettison')) - exclude (dependency('org.apache.hadoop:hadoop-yarn-server-common')) - exclude (dependency('commons-net:commons-net')) - exclude (dependency('javax.servlet.jsp:jsp-api')) - exclude (dependency('org.apache.curator:curator-framework')) - } - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'org.apache.calcite', 'org.apache.iceberg.shaded.org.apache.calcite' - relocate 'commons-collections', 'org.apache.iceberg.shaded.commons-collections' - } } project(':iceberg-arrow') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-parquet', configuration: 'shadow') - - compileOnly('com.google.guava:guava:28.0-jre') { - // may be LGPL - use ALv2 findbugs-annotations instead - exclude group: 'com.google.code.findbugs' - } - compile 'org.slf4j:slf4j-api' - compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' + compile project(':iceberg-api') + compile project(':iceberg-parquet') compile("org.apache.arrow:arrow-vector") { exclude group: 'io.netty', module: 'netty-buffer' @@ -742,27 +297,6 @@ project(':iceberg-arrow') { compile("org.apache.arrow:arrow-memory") { exclude group: 'io.netty', module: 'netty-common' } - compile("org.apache.parquet:parquet-avro") { - exclude group: 'org.apache.avro', module: 'avro' - // already shaded by Parquet - exclude group: 'it.unimi.dsi' - exclude group: 'org.codehaus.jackson' - } - } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' } } @@ -778,13 +312,6 @@ project(':iceberg-spark') { compile project(':iceberg-arrow') compile project(':iceberg-hive') - compile 'org.slf4j:slf4j-api' - compile 'com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1' - - testCompile 'junit:junit' - testCompile 'org.slf4j:slf4j-simple' - testCompile 'org.mockito:mockito-core' - compileOnly "org.apache.avro:avro" compileOnly("org.apache.spark:spark-hive_2.11") { exclude group: 'org.apache.avro', module: 'avro' @@ -820,17 +347,11 @@ project(':iceberg-spark') { } project(':iceberg-pig') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.assemble.dependsOn tasks.shadowJar - tasks.install.dependsOn tasks.shadowJar - tasks.javadocJar.dependsOn tasks.shadowJar - dependencies { - compile project(path: ':iceberg-api', configuration: 'shadow') - compile project(path: ':iceberg-common', configuration: 'shadow') - compile project(path: ':iceberg-core', configuration: 'shadow') - compile project(path: ':iceberg-parquet', configuration: 'shadow') + compile project(':iceberg-api') + compile project(':iceberg-common') + compile project(':iceberg-core') + compile project(':iceberg-parquet') compile "org.apache.commons:commons-lang3" @@ -841,34 +362,12 @@ project(':iceberg-pig') { compileOnly("org.apache.hadoop:hadoop-client") { exclude group: 'org.apache.avro', module: 'avro' } - compile("org.apache.parquet:parquet-avro") { - exclude group: 'org.apache.avro', module: 'avro' - // already shaded by Parquet - exclude group: 'it.unimi.dsi' - exclude group: 'org.codehaus.jackson' - } testCompile "org.apache.hadoop:hadoop-hdfs::tests" testCompile "org.apache.hadoop:hadoop-common::tests" testCompile("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' } - testCompile 'junit:junit' - } - - shadowJar { - // shade compileOnly dependencies to avoid including in transitive dependencies - configurations = [project.configurations.compile] - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' } } diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java similarity index 99% rename from mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java rename to mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 0b36c4a5ac26..3718f8f544b1 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -17,7 +17,7 @@ * under the License. */ -package org.iceberg.mr.mapred; +package org.apache.iceberg.mr.mapred; import java.io.DataInput; import java.io.DataOutput; diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java similarity index 98% rename from mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java rename to mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java index e653787c9ed3..5fbfd7f62c0c 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java @@ -17,7 +17,7 @@ * under the License. */ -package org.iceberg.mr.mapred; +package org.apache.iceberg.mr.mapred; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java similarity index 97% rename from mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java rename to mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java index 6a73b88aefa0..8b0eb79fbe73 100644 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergWritable.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergWritable.java @@ -17,7 +17,7 @@ * under the License. */ -package org.iceberg.mr.mapred; +package org.apache.iceberg.mr.mapred; import java.io.DataInput; import java.io.DataOutput; diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java deleted file mode 100644 index ab2b84224580..000000000000 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergObjectInspectorGenerator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.iceberg.mr.mapred; - -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.Types; - -class IcebergObjectInspectorGenerator { - - protected ObjectInspector createObjectInspector(Schema schema) throws Exception { - List columnNames = setColumnNames(schema); - List columnTypes = IcebergSchemaToTypeInfo.getColumnTypes(schema); - - List columnOIs = new ArrayList<>(columnTypes.size()); - for (int i = 0; i < columnTypes.size(); i++) { - columnOIs.add(createObjectInspectorWorker(columnTypes.get(i))); - } - return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs, null); - } - - protected ObjectInspector createObjectInspectorWorker(TypeInfo typeInfo) throws Exception { - ObjectInspector.Category typeCategory = typeInfo.getCategory(); - - switch (typeCategory) { - case PRIMITIVE: - PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo; - return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti); - case LIST: - ListTypeInfo ati = (ListTypeInfo) typeInfo; - return ObjectInspectorFactory - .getStandardListObjectInspector(createObjectInspectorWorker(ati.getListElementTypeInfo())); - case MAP: - MapTypeInfo mti = (MapTypeInfo) typeInfo; - return ObjectInspectorFactory.getStandardMapObjectInspector( - createObjectInspectorWorker(mti.getMapKeyTypeInfo()), - createObjectInspectorWorker(mti.getMapValueTypeInfo())); - case STRUCT: - StructTypeInfo sti = (StructTypeInfo) typeInfo; - List ois = new ArrayList<>(sti.getAllStructFieldTypeInfos().size()); - for (TypeInfo structTypeInfos : sti.getAllStructFieldTypeInfos()) { - ois.add(createObjectInspectorWorker(structTypeInfos)); - } - return ObjectInspectorFactory.getStandardStructObjectInspector(sti.getAllStructFieldNames(), ois); - default: - throw new SerDeException("Couldn't create Object Inspector for category: '" + typeCategory + "'"); - } - } - - protected List setColumnNames(Schema schema) { - List fields = schema.columns(); - List fieldsList = new ArrayList<>(fields.size()); - for (Types.NestedField field : fields) { - fieldsList.add(field.name()); - } - return fieldsList; - } - -} diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java deleted file mode 100644 index 554be063e449..000000000000 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSchemaToTypeInfo.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.iceberg.mr.mapred; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Hashtable; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -/** - * Class to convert Iceberg types to Hive TypeInfo - */ -final class IcebergSchemaToTypeInfo { - - private IcebergSchemaToTypeInfo() {} - - private static final Map primitiveTypeToTypeInfo = initTypeMap(); - - private static Map initTypeMap() { - Map theMap = new Hashtable(); - theMap.put(Types.BooleanType.get(), TypeInfoFactory.getPrimitiveTypeInfo("boolean")); - theMap.put(Types.IntegerType.get(), TypeInfoFactory.getPrimitiveTypeInfo("int")); - theMap.put(Types.LongType.get(), TypeInfoFactory.getPrimitiveTypeInfo("bigint")); - theMap.put(Types.FloatType.get(), TypeInfoFactory.getPrimitiveTypeInfo("float")); - theMap.put(Types.DoubleType.get(), TypeInfoFactory.getPrimitiveTypeInfo("double")); - theMap.put(Types.BinaryType.get(), TypeInfoFactory.getPrimitiveTypeInfo("binary")); - theMap.put(Types.StringType.get(), TypeInfoFactory.getPrimitiveTypeInfo("string")); - theMap.put(Types.DateType.get(), TypeInfoFactory.getPrimitiveTypeInfo("date")); - theMap.put(Types.TimestampType.withoutZone(), TypeInfoFactory.getPrimitiveTypeInfo("timestamp")); - return Collections.unmodifiableMap(theMap); - } - - public static List getColumnTypes(Schema schema) throws Exception { - List fields = schema.columns(); - List types = new ArrayList<>(fields.size()); - for (Types.NestedField field : fields) { - types.add(generateTypeInfo(field.type())); - } - return types; - } - - private static TypeInfo generateTypeInfo(Type type) throws Exception { - if (primitiveTypeToTypeInfo.containsKey(type)) { - return primitiveTypeToTypeInfo.get(type); - } - switch (type.typeId()) { - case UUID: - return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); - case FIXED: - return TypeInfoFactory.getPrimitiveTypeInfo("binary"); - case TIME: - return TypeInfoFactory.getPrimitiveTypeInfo("long"); - case DECIMAL: - Types.DecimalType dec = (Types.DecimalType) type; - int scale = dec.scale(); - int precision = dec.precision(); - try { - HiveDecimalUtils.validateParameter(precision, scale); - } catch (Exception e) { - //TODO Log that precision / scale isn't valid - throw e; - } - return TypeInfoFactory.getDecimalTypeInfo(precision, scale); - case STRUCT: - return generateStructTypeInfo((Types.StructType) type); - case LIST: - return generateListTypeInfo((Types.ListType) type); - case MAP: - return generateMapTypeInfo((Types.MapType) type); - default: - throw new SerDeException("Can't map Iceberg type to Hive TypeInfo: '" + type.typeId() + "'"); - } - } - - private static TypeInfo generateMapTypeInfo(Types.MapType type) throws Exception { - Type keyType = type.keyType(); - Type valueType = type.valueType(); - return TypeInfoFactory.getMapTypeInfo(generateTypeInfo(keyType), generateTypeInfo(valueType)); - } - - private static TypeInfo generateStructTypeInfo(Types.StructType type) throws Exception { - List fields = type.fields(); - List fieldNames = new ArrayList<>(fields.size()); - List typeInfos = new ArrayList<>(fields.size()); - - for (Types.NestedField field : fields) { - fieldNames.add(field.name()); - typeInfos.add(generateTypeInfo(field.type())); - } - return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); - } - - private static TypeInfo generateListTypeInfo(Types.ListType type) throws Exception { - return TypeInfoFactory.getListTypeInfo(generateTypeInfo(type.elementType())); - } -} diff --git a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java b/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java deleted file mode 100644 index e78a18b46d53..000000000000 --- a/mr/src/main/java/org/iceberg/mr/mapred/IcebergSerDe.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.iceberg.mr.mapred; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import javax.annotation.Nullable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.AbstractSerDe; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.SerDeStats; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.io.Writable; -import org.apache.iceberg.Schema; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; -import org.apache.iceberg.hadoop.HadoopFileIO; -import org.apache.iceberg.types.Types; - -public class IcebergSerDe extends AbstractSerDe { - - private Schema schema; - private TableMetadata metadata; - private ObjectInspector inspector; - private List columnNames; - private List columnTypes; - - @Override - public void initialize(@Nullable Configuration configuration, Properties properties) throws SerDeException { - //TODO Add methods to dynamically find most recent metadata - String tableDir = properties.getProperty("location") + "/metadata/v2.metadata.json"; - this.metadata = TableMetadataParser.read(new HadoopFileIO(configuration), tableDir); - this.schema = metadata.schema(); - - try { - this.inspector = new IcebergObjectInspectorGenerator().createObjectInspector(schema); - } catch (Exception e) { - throw new SerDeException(e); - } - } - - @Override - public Class getSerializedClass() { - return null; - } - - @Override - public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException { - return null; - } - - @Override - public SerDeStats getSerDeStats() { - return null; - } - - @Override - public Object deserialize(Writable writable) throws SerDeException { - IcebergWritable icebergWritable = (IcebergWritable) writable; - List fields = icebergWritable.getSchema().columns(); - List row = new ArrayList<>(); - for (Types.NestedField field : fields) { - Object obj = ((IcebergWritable) writable).getRecord().getField(field.name()); - row.add(obj); - } - return Collections.unmodifiableList(row); - } - - @Override - public ObjectInspector getObjectInspector() throws SerDeException { - return inspector; - } -} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 756964ff0b44..1e228fc3300c 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -19,9 +19,6 @@ package org.apache.iceberg.mr.mapred; -import com.klarna.hiverunner.HiveShell; -import com.klarna.hiverunner.StandaloneHiveRunner; -import com.klarna.hiverunner.annotations.HiveSQL; import java.io.File; import java.io.IOException; import java.util.List; @@ -38,30 +35,25 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.types.Types; -import org.iceberg.mr.mapred.IcebergInputFormat; -import org.iceberg.mr.mapred.IcebergWritable; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -@RunWith(StandaloneHiveRunner.class) public class TestIcebergInputFormat { private static final Logger LOG = LoggerFactory.getLogger(TestIcebergInputFormat.class); - @HiveSQL(files = {}, autoStart = true) - private HiveShell shell; - private File tableLocation; private Table table; + //TODO flesh out with more tests of the IF itself + //TODO: do we still need the table data etc. if we're not testing from Hive? + @Before public void before() throws IOException { tableLocation = java.nio.file.Files.createTempDirectory("temp").toFile(); @@ -81,28 +73,6 @@ public void before() throws IOException { table.newAppend().appendFile(fileA).commit(); } - @Test - public void testInputFormat() { - shell.execute("CREATE DATABASE source_db"); - shell.execute(new StringBuilder() - .append("CREATE TABLE source_db.table_a ") - .append("ROW FORMAT SERDE 'org.iceberg.mr.mapred.IcebergSerDe' ") - .append("STORED AS ") - .append("INPUTFORMAT 'org.iceberg.mr.mapred.IcebergInputFormat' ") - .append("OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' ") - .append("LOCATION '") - .append(tableLocation.getAbsolutePath()) - .append("'") - .toString()); - - List result = shell.executeStatement("SELECT * FROM source_db.table_a"); - - assertEquals(3, result.size()); - assertArrayEquals(new Object[]{"Michael", 3000L}, result.get(0)); - assertArrayEquals(new Object[]{"Andy", 3000L}, result.get(1)); - assertArrayEquals(new Object[]{"Berta", 4000L}, result.get(2)); - } - @Test public void testGetSplits() throws IOException { IcebergInputFormat format = new IcebergInputFormat(); diff --git a/versions.lock b/versions.lock index 3d70a73d25b2..93ec82fc973d 100644 --- a/versions.lock +++ b/versions.lock @@ -1,100 +1,90 @@ # Run ./gradlew --write-locks to regenerate this file -ant:ant:1.6.5 (2 constraints: 2d1539e1) +ant:ant:1.6.5 (1 constraints: 730a54bc) aopalliance:aopalliance:1.0 (1 constraints: 170a83ac) -asm:asm:3.1 (3 constraints: 251fd7ad) -asm:asm-commons:3.1 (1 constraints: 9c0f1f7a) -asm:asm-tree:3.1 (1 constraints: 2307035c) -ch.qos.logback:logback-classic:1.0.9 (3 constraints: d3258a88) -ch.qos.logback:logback-core:1.0.9 (4 constraints: dd32435a) -co.cask.tephra:tephra-api:0.6.0 (3 constraints: 0828ded1) -co.cask.tephra:tephra-core:0.6.0 (2 constraints: 831cd90d) -co.cask.tephra:tephra-hbase-compat-1.0:0.6.0 (1 constraints: 370d6920) +asm:asm:3.1 (2 constraints: 4f19c3c6) com.carrotsearch:hppc:0.7.2 (1 constraints: f70cda14) com.clearspring.analytics:stream:2.7.0 (1 constraints: 1a0dd136) com.esotericsoftware:kryo-shaded:4.0.2 (2 constraints: b71345a6) com.esotericsoftware:minlog:1.3.0 (1 constraints: 670e7c4f) -com.esotericsoftware.kryo:kryo:2.24.0 (1 constraints: 3a053f3b) -com.esotericsoftware.minlog:minlog:1.2 (1 constraints: 650d3615) +com.fasterxml.jackson.core:jackson-annotations:2.10.2 (5 constraints: 4155160f) +com.fasterxml.jackson.core:jackson-core:2.10.2 (6 constraints: bb52b302) +com.fasterxml.jackson.core:jackson-databind:2.10.2 (11 constraints: 7c9eca9a) +com.fasterxml.jackson.module:jackson-module-paranamer:2.10.2 (1 constraints: 03162c16) +com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.2 (1 constraints: 7f0da251) com.github.ben-manes.caffeine:caffeine:2.7.0 (1 constraints: 0b050a36) com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter:0.1.2 (1 constraints: e90b08f3) com.github.luben:zstd-jni:1.3.2-2 (1 constraints: 760d7c51) -com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (10 constraints: 078609f3) -com.google.code.findbugs:jsr305:3.0.2 (15 constraints: f4c0e31b) -com.google.code.gson:gson:2.2.4 (6 constraints: f44c3ddb) +com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1 (1 constraints: 6d05ab40) +com.google.code.findbugs:jsr305:3.0.2 (7 constraints: fc5db58f) +com.google.code.gson:gson:2.2.4 (2 constraints: 9518bfd2) com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 161a2544) com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714) com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4) +com.google.guava:guava:28.0-jre (21 constraints: 88453dad) com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava (1 constraints: bd17c918) -com.google.inject:guice:3.0 (8 constraints: 2c93366d) -com.google.inject.extensions:guice-assistedinject:3.0 (1 constraints: 250b42ce) -com.google.inject.extensions:guice-servlet:3.0 (12 constraints: 01e2ea30) +com.google.inject:guice:3.0 (6 constraints: 6873914c) +com.google.inject.extensions:guice-servlet:3.0 (11 constraints: a9d50a2b) com.google.j2objc:j2objc-annotations:1.3 (1 constraints: b809eda0) -com.google.protobuf:protobuf-java:3.0.0-beta-1 (22 constraints: f46c2ce2) +com.google.protobuf:protobuf-java:2.5.0 (16 constraints: 2f1c978f) com.googlecode.javaewah:JavaEWAH:0.3.2 (1 constraints: ea0dfc42) com.jamesmurty.utils:java-xmlbuilder:0.4 (1 constraints: e40aa5ca) com.jcraft:jsch:0.1.42 (1 constraints: bb0ded3c) com.jolbox:bonecp:0.8.0.RELEASE (2 constraints: b22109f9) com.ning:compress-lzf:1.0.3 (1 constraints: 150dba36) -com.sun.jersey:jersey-client:1.9 (6 constraints: 546829c2) -com.sun.jersey:jersey-core:1.9 (10 constraints: 399cd337) -com.sun.jersey:jersey-json:1.9 (7 constraints: 8375d6d3) -com.sun.jersey:jersey-server:1.9 (6 constraints: 9b50d765) +com.sun.jersey:jersey-client:1.9 (4 constraints: 65529ed7) +com.sun.jersey:jersey-core:1.9 (9 constraints: ec8f4404) +com.sun.jersey:jersey-json:1.9 (5 constraints: 945f2f90) +com.sun.jersey:jersey-server:1.9 (4 constraints: ef373c01) com.sun.jersey.contribs:jersey-guice:1.9 (4 constraints: 65529ed7) com.sun.xml.bind:jaxb-impl:2.2.3-1 (1 constraints: 330c2404) -com.tdunning:json:1.8 (2 constraints: 051968bf) com.thoughtworks.paranamer:paranamer:2.8 (3 constraints: 742d4cb1) com.twitter:chill-java:0.9.3 (2 constraints: a716716f) com.twitter:chill_2.11:0.9.3 (2 constraints: 121b92c3) com.twitter:parquet-hadoop-bundle:1.6.0 (2 constraints: 061b4d93) com.univocity:univocity-parsers:2.7.3 (1 constraints: c40ccb27) -com.zaxxer:HikariCP:2.5.1 (1 constraints: 390d7120) commons-beanutils:commons-beanutils:1.7.0 (1 constraints: da0e635f) commons-beanutils:commons-beanutils-core:1.8.0 (1 constraints: 1d134124) -commons-cli:commons-cli:1.2 (12 constraints: ab9686c9) -commons-codec:commons-codec:1.10 (24 constraints: d0347e53) -commons-collections:commons-collections:3.2.2 (6 constraints: 8e604cdc) +commons-cli:commons-cli:1.2 (9 constraints: f874366c) +commons-codec:commons-codec:1.10 (17 constraints: a8de3870) +commons-collections:commons-collections:3.2.2 (3 constraints: e73a8e36) commons-configuration:commons-configuration:1.6 (1 constraints: 2d0d5c14) commons-daemon:commons-daemon:1.0.13 (1 constraints: d50c811c) commons-dbcp:commons-dbcp:1.4 (3 constraints: 9029e0e4) commons-digester:commons-digester:1.8 (1 constraints: bf1228fe) commons-el:commons-el:1.0 (2 constraints: ad11e7f0) -commons-httpclient:commons-httpclient:3.1 (6 constraints: 8545051d) -commons-io:commons-io:2.4 (11 constraints: e6902ece) -commons-lang:commons-lang:2.6 (32 constraints: d4b70019) -commons-logging:commons-logging:1.2 (31 constraints: aedcdd2a) +commons-httpclient:commons-httpclient:3.1 (4 constraints: e52cc77f) +commons-io:commons-io:2.4 (6 constraints: 4a568049) +commons-lang:commons-lang:2.6 (20 constraints: 401f63f3) +commons-logging:commons-logging:1.2 (20 constraints: 424b646e) commons-net:commons-net:3.1 (3 constraints: 3d222e61) commons-pool:commons-pool:1.6 (4 constraints: e336ab5e) dk.brics.automaton:automaton:1.11-8 (1 constraints: 92088a8d) hsqldb:hsqldb:1.8.0.10 (1 constraints: f008499f) io.airlift:aircompressor:0.15 (1 constraints: 0e0aa4b2) -io.dropwizard.metrics:metrics-core:3.1.5 (9 constraints: be90b1ee) +io.dropwizard.metrics:metrics-core:3.1.5 (8 constraints: 3b8585b8) io.dropwizard.metrics:metrics-graphite:3.1.5 (1 constraints: 1a0dc936) io.dropwizard.metrics:metrics-json:3.1.5 (2 constraints: 03195c12) io.dropwizard.metrics:metrics-jvm:3.1.5 (2 constraints: 03195c12) -io.netty:netty:3.9.9.Final (11 constraints: 51cef3a2) -io.netty:netty-all:4.1.17.Final (6 constraints: 646011b6) +io.netty:netty:3.9.9.Final (9 constraints: 9eb0396d) +io.netty:netty-all:4.1.17.Final (3 constraints: d2312526) io.netty:netty-buffer:4.1.27.Final (1 constraints: 4a0fee77) -it.unimi.dsi:fastutil:6.5.6 (1 constraints: 910b3ce5) -javax.activation:activation:1.1.1 (3 constraints: b02331a0) +javax.activation:activation:1.1.1 (1 constraints: 140dbb36) javax.annotation:javax.annotation-api:1.3.2 (3 constraints: 55341c48) javax.inject:javax.inject:1 (4 constraints: 852d0c1a) javax.jdo:jdo-api:3.0.1 (2 constraints: 4c1dcc1a) -javax.mail:mail:1.4.1 (1 constraints: fc0fe399) javax.servlet:javax.servlet-api:3.1.0 (1 constraints: 150dc436) javax.servlet:jsp-api:2.0 (1 constraints: 0b0aa0a7) -javax.servlet:servlet-api:2.5 (12 constraints: 72b75a2c) -javax.servlet.jsp:jsp-api:2.1 (2 constraints: 811985e6) +javax.servlet:servlet-api:2.5 (9 constraints: f991a6d2) +javax.servlet.jsp:jsp-api:2.1 (1 constraints: 290d5a14) javax.transaction:jta:1.1 (1 constraints: 9f07d96b) -javax.transaction:transaction-api:1.1 (1 constraints: 0a0b64c9) javax.validation:validation-api:1.1.0.Final (1 constraints: 13133130) javax.ws.rs:javax.ws.rs-api:2.0.1 (5 constraints: 6e649355) javax.xml.bind:jaxb-api:2.2.11 (6 constraints: a069fd48) javolution:javolution:5.5.1 (2 constraints: 2b1b2b82) jline:jline:2.12 (3 constraints: 7c21b2cb) joda-time:joda-time:2.9.9 (4 constraints: 2326d336) -junit:junit:4.12 (10 constraints: 4a8036b6) log4j:apache-log4j-extras:1.2.17 (1 constraints: 200e1d51) -log4j:log4j:1.2.17 (17 constraints: fbf6ff1d) +log4j:log4j:1.2.17 (8 constraints: e7772b11) net.hydromatic:eigenbase-properties:1.1.5 (1 constraints: 5f0daf2c) net.java.dev.jets3t:jets3t:0.9.0 (2 constraints: ec152b22) net.razorvine:pyrolite:4.13 (1 constraints: eb0cb829) @@ -109,72 +99,55 @@ org.apache.ant:ant-launcher:1.9.1 (1 constraints: 69082485) org.apache.arrow:arrow-format:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-memory:0.14.1 (1 constraints: 240df421) org.apache.arrow:arrow-vector:0.14.1 (2 constraints: 2012a545) -org.apache.avro:avro:1.9.2 (6 constraints: da4d6402) +org.apache.avro:avro:1.9.2 (4 constraints: 3e2e68f4) org.apache.avro:avro-ipc:1.8.2 (1 constraints: f90b5bf4) org.apache.avro:avro-mapred:1.8.2 (2 constraints: 3a1a4787) -org.apache.calcite:calcite-avatica:1.2.0-incubating (2 constraints: a0237b5d) -org.apache.calcite:calcite-core:1.10.0 (2 constraints: 9b164229) -org.apache.calcite:calcite-linq4j:1.10.0 (1 constraints: 8a0d363a) -org.apache.calcite.avatica:avatica:1.8.0 (1 constraints: 610dbf2c) -org.apache.calcite.avatica:avatica-metrics:1.8.0 (1 constraints: 960e635d) -org.apache.commons:commons-compress:1.19 (7 constraints: ff569390) +org.apache.calcite:calcite-avatica:1.2.0-incubating (3 constraints: 4b35b263) +org.apache.calcite:calcite-core:1.2.0-incubating (1 constraints: 68119fdf) +org.apache.calcite:calcite-linq4j:1.2.0-incubating (1 constraints: ac1147d8) +org.apache.commons:commons-compress:1.19 (6 constraints: 464a0c7f) org.apache.commons:commons-crypto:1.0.0 (2 constraints: 3a1e5fbf) -org.apache.commons:commons-lang3:3.9 (9 constraints: 316fa5f3) -org.apache.commons:commons-math3:3.4.1 (3 constraints: 7c24247c) -org.apache.curator:apache-curator:2.7.1 (2 constraints: c718e2d6) -org.apache.curator:curator-client:2.7.1 (3 constraints: 272ac6a3) -org.apache.curator:curator-framework:2.7.1 (8 constraints: 806edda7) -org.apache.curator:curator-recipes:2.7.1 (4 constraints: ba337377) +org.apache.commons:commons-lang3:3.9 (5 constraints: 503b94b4) +org.apache.commons:commons-math3:3.4.1 (2 constraints: a11af290) +org.apache.curator:curator-client:2.7.1 (2 constraints: 6a1d2734) +org.apache.curator:curator-framework:2.7.1 (4 constraints: 4d37382d) +org.apache.curator:curator-recipes:2.7.1 (2 constraints: a61acc91) org.apache.derby:derby:10.12.1.1 (3 constraints: 9f2cb182) org.apache.directory.api:api-asn1-api:1.0.0-M20 (1 constraints: 3d163b13) org.apache.directory.api:api-util:1.0.0-M20 (1 constraints: 3d163b13) org.apache.directory.server:apacheds-i18n:2.0.0-M15 (1 constraints: 42164713) org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15 (1 constraints: 8f0d3b45) -org.apache.geronimo.specs:geronimo-annotation_1.0_spec:1.1.1 (1 constraints: f90fda99) -org.apache.geronimo.specs:geronimo-jaspic_1.0_spec:1.0 (1 constraints: 990f187a) -org.apache.geronimo.specs:geronimo-jta_1.1_spec:1.1.1 (1 constraints: f90fda99) -org.apache.hadoop:hadoop-annotations:2.7.3 (23 constraints: 1b8376ec) -org.apache.hadoop:hadoop-auth:2.7.3 (5 constraints: 903fb325) -org.apache.hadoop:hadoop-client:2.7.3 (4 constraints: 912bfbce) -org.apache.hadoop:hadoop-common:2.7.3 (20 constraints: ca218990) -org.apache.hadoop:hadoop-hdfs:2.7.3 (8 constraints: fc69f814) +org.apache.hadoop:hadoop-annotations:2.7.3 (16 constraints: 2c27b38c) +org.apache.hadoop:hadoop-auth:2.7.3 (1 constraints: 900d4d2f) +org.apache.hadoop:hadoop-client:2.7.3 (2 constraints: 2b12043c) +org.apache.hadoop:hadoop-common:2.7.3 (4 constraints: 163dee6b) +org.apache.hadoop:hadoop-hdfs:2.7.3 (4 constraints: b834c025) org.apache.hadoop:hadoop-mapreduce-client-app:2.7.3 (3 constraints: ab2f8436) -org.apache.hadoop:hadoop-mapreduce-client-common:2.7.3 (5 constraints: 815ba1d8) -org.apache.hadoop:hadoop-mapreduce-client-core:2.7.3 (11 constraints: 369ec41b) +org.apache.hadoop:hadoop-mapreduce-client-common:2.7.3 (4 constraints: 184f4f66) +org.apache.hadoop:hadoop-mapreduce-client-core:2.7.3 (4 constraints: 66361812) org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.7.3 (2 constraints: 3b1dfa13) org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.7.3 (2 constraints: 2628c449) -org.apache.hadoop:hadoop-yarn-api:2.7.3 (18 constraints: 061eefa9) -org.apache.hadoop:hadoop-yarn-client:2.7.3 (4 constraints: 543421ae) -org.apache.hadoop:hadoop-yarn-common:2.7.3 (16 constraints: 260de1f1) +org.apache.hadoop:hadoop-yarn-api:2.7.3 (10 constraints: 07b8bd4c) +org.apache.hadoop:hadoop-yarn-client:2.7.3 (1 constraints: 1f14626e) +org.apache.hadoop:hadoop-yarn-common:2.7.3 (9 constraints: b3b2f06f) org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice:2.7.3 (1 constraints: f5157dcf) org.apache.hadoop:hadoop-yarn-server-common:2.7.3 (7 constraints: 5192cac0) org.apache.hadoop:hadoop-yarn-server-nodemanager:2.7.3 (2 constraints: 6726468d) org.apache.hadoop:hadoop-yarn-server-resourcemanager:2.7.3 (2 constraints: b22045af) -org.apache.hadoop:hadoop-yarn-server-web-proxy:2.7.3 (3 constraints: aa32e81b) -org.apache.hbase:hbase-annotations:1.1.1 (4 constraints: 0d36c989) -org.apache.hbase:hbase-client:1.1.1 (3 constraints: a427c078) -org.apache.hbase:hbase-common:1.1.1 (5 constraints: 4b433401) -org.apache.hbase:hbase-protocol:1.1.1 (4 constraints: 9d335412) -org.apache.hive:hive-common:2.3.6 (5 constraints: 6141bdc2) -org.apache.hive:hive-exec:2.3.6 (1 constraints: 0d050436) -org.apache.hive:hive-metastore:2.3.6 (2 constraints: 651190f2) -org.apache.hive:hive-serde:2.3.6 (4 constraints: e22dbc9a) -org.apache.hive:hive-service-rpc:2.3.6 (2 constraints: d317528f) -org.apache.hive:hive-shims:2.3.6 (5 constraints: 863d32c4) +org.apache.hadoop:hadoop-yarn-server-web-proxy:2.7.3 (2 constraints: cb287679) +org.apache.hive:hive-common:2.3.6 (1 constraints: 7b0bc2e4) +org.apache.hive:hive-metastore:2.3.6 (1 constraints: 0d050436) +org.apache.hive:hive-serde:2.3.6 (1 constraints: 3c0d7020) +org.apache.hive:hive-service-rpc:2.3.6 (1 constraints: 7b0bc2e4) +org.apache.hive:hive-shims:2.3.6 (4 constraints: b22fbcd9) org.apache.hive:hive-storage-api:2.4.0 (1 constraints: ec0b19f3) -org.apache.hive:hive-vector-code-gen:2.3.6 (1 constraints: 0d0bf1d6) org.apache.hive.shims:hive-shims-0.23:2.3.6 (1 constraints: 8c0b6ce5) org.apache.hive.shims:hive-shims-common:2.3.6 (3 constraints: 222cfaad) org.apache.hive.shims:hive-shims-scheduler:2.3.6 (1 constraints: 8c0b6ce5) -org.apache.htrace:htrace-core:3.1.0-incubating (5 constraints: 89553ebc) -org.apache.httpcomponents:httpclient:4.5.6 (6 constraints: ac4c9b4e) -org.apache.httpcomponents:httpcore:4.4.10 (5 constraints: 29432873) +org.apache.htrace:htrace-core:3.1.0-incubating (2 constraints: cd22cffa) +org.apache.httpcomponents:httpclient:4.5.6 (4 constraints: 573134dd) +org.apache.httpcomponents:httpcore:4.4.10 (3 constraints: d327f763) org.apache.ivy:ivy:2.4.0 (3 constraints: 0826dbf1) -org.apache.logging.log4j:log4j-1.2-api:2.6.2 (1 constraints: f00b21f3) -org.apache.logging.log4j:log4j-api:2.6.2 (4 constraints: 2e3c9f23) -org.apache.logging.log4j:log4j-core:2.6.2 (2 constraints: fd1c2464) -org.apache.logging.log4j:log4j-slf4j-impl:2.6.2 (2 constraints: 821cbce5) -org.apache.logging.log4j:log4j-web:2.6.2 (1 constraints: f00b21f3) org.apache.orc:orc-core:1.6.2 (3 constraints: ba1d17ad) org.apache.orc:orc-mapreduce:1.6.2 (1 constraints: c30cc227) org.apache.orc:orc-shims:1.6.2 (1 constraints: 3f0aeabc) @@ -198,35 +171,24 @@ org.apache.spark:spark-sketch_2.11:2.4.4 (2 constraints: 981bd4f5) org.apache.spark:spark-sql_2.11:2.4.4 (1 constraints: 1e0d0037) org.apache.spark:spark-tags_2.11:2.4.4 (8 constraints: 036fa69d) org.apache.spark:spark-unsafe_2.11:2.4.4 (2 constraints: f11bc213) -org.apache.thrift:libfb303:0.9.3 (4 constraints: 8034ebe7) -org.apache.thrift:libthrift:0.9.3 (9 constraints: f574b013) -org.apache.twill:twill-api:0.6.0-incubating (2 constraints: e422b4e1) -org.apache.twill:twill-common:0.6.0-incubating (4 constraints: 3d467991) -org.apache.twill:twill-core:0.6.0-incubating (1 constraints: d70f9d7c) -org.apache.twill:twill-discovery-api:0.6.0-incubating (3 constraints: 25345d4c) -org.apache.twill:twill-discovery-core:0.6.0-incubating (2 constraints: 332039f9) -org.apache.twill:twill-zookeeper:0.6.0-incubating (3 constraints: 94341288) -org.apache.velocity:velocity:1.5 (1 constraints: c70e875e) +org.apache.thrift:libfb303:0.9.3 (3 constraints: 27289d07) +org.apache.thrift:libthrift:0.9.3 (6 constraints: 3f4f0635) org.apache.xbean:xbean-asm6-shaded:4.8 (2 constraints: 2419a30f) org.apache.yetus:audience-annotations:0.11.0 (1 constraints: c40eb364) -org.apache.zookeeper:zookeeper:3.4.6 (16 constraints: 16e9c913) +org.apache.zookeeper:zookeeper:3.4.6 (11 constraints: 18a71f48) org.checkerframework:checker-qual:2.8.1 (2 constraints: 1a1a3944) -org.codehaus.groovy:groovy-all:2.4.4 (1 constraints: 0c0bf2d6) -org.codehaus.jackson:jackson-core-asl:1.9.13 (14 constraints: e8bb1763) -org.codehaus.jackson:jackson-jaxrs:1.9.13 (4 constraints: 5235d62f) -org.codehaus.jackson:jackson-mapper-asl:1.9.13 (17 constraints: c3eee844) -org.codehaus.jackson:jackson-xc:1.9.13 (3 constraints: 73286ded) -org.codehaus.janino:commons-compiler:3.0.9 (3 constraints: 0a2837cc) -org.codehaus.janino:janino:3.0.9 (2 constraints: 3f1c6304) -org.codehaus.jettison:jettison:1.1 (5 constraints: 155c2ac6) +org.codehaus.jackson:jackson-core-asl:1.9.13 (11 constraints: 8091cd06) +org.codehaus.jackson:jackson-jaxrs:1.9.13 (2 constraints: 821bca9d) +org.codehaus.jackson:jackson-mapper-asl:1.9.13 (11 constraints: e18d325f) +org.codehaus.jackson:jackson-xc:1.9.13 (2 constraints: 821bca9d) +org.codehaus.janino:commons-compiler:3.0.9 (2 constraints: a41a546f) +org.codehaus.janino:janino:3.0.9 (1 constraints: d90e817c) +org.codehaus.jettison:jettison:1.1 (4 constraints: a84e24a9) org.codehaus.mojo:animal-sniffer-annotations:1.17 (1 constraints: ed09d8aa) org.datanucleus:datanucleus-api-jdo:4.2.4 (2 constraints: 591df91b) org.datanucleus:datanucleus-core:4.1.17 (5 constraints: 584455e8) org.datanucleus:datanucleus-rdbms:4.1.19 (2 constraints: 911dec32) org.datanucleus:javax.jdo:3.2.0-m3 (1 constraints: 030ea249) -org.eclipse.jdt:core:3.1.1 (1 constraints: b40a38d8) -org.eclipse.jetty.aggregate:jetty-all:7.6.0.v20120127 (2 constraints: b31cf79a) -org.eclipse.jetty.orbit:javax.servlet:3.0.0.v201112011016 (1 constraints: dd0e53b1) org.fusesource.leveldbjni:leveldbjni-all:1.8 (9 constraints: 91a69ae7) org.glassfish.hk2:hk2-api:2.4.0-b34 (5 constraints: 9d5608c7) org.glassfish.hk2:hk2-locator:2.4.0-b34 (4 constraints: 3d490865) @@ -241,28 +203,17 @@ org.glassfish.jersey.core:jersey-client:2.22.2 (2 constraints: 791ef7a3) org.glassfish.jersey.core:jersey-common:2.22.2 (6 constraints: 5f747f50) org.glassfish.jersey.core:jersey-server:2.22.2 (3 constraints: 553f5d56) org.glassfish.jersey.media:jersey-media-jaxb:2.22.2 (1 constraints: 3111f1d4) -org.hamcrest:hamcrest-core:1.3 (2 constraints: 7910aeb0) org.iq80.snappy:snappy:0.2 (1 constraints: 890d5927) org.javassist:javassist:3.18.1-GA (1 constraints: 570d4740) org.jetbrains:annotations:17.0.0 (1 constraints: 6e0a64c7) org.jodd:jodd-core:3.5.2 (2 constraints: 0c1bda93) -org.jruby.jcodings:jcodings:1.0.8 (2 constraints: 9d15c301) -org.jruby.joni:joni:2.1.2 (1 constraints: 8f0c160d) org.json4s:json4s-ast_2.11:3.5.3 (1 constraints: 0c0b9ae9) org.json4s:json4s-core_2.11:3.5.3 (1 constraints: 4c0c5316) org.json4s:json4s-jackson_2.11:3.5.3 (1 constraints: 1c0dd336) org.json4s:json4s-scalap_2.11:3.5.3 (1 constraints: 0c0b9ae9) org.lz4:lz4-java:1.4.0 (1 constraints: 160dc336) -org.mortbay.jetty:jetty:6.1.26 (8 constraints: b66925ab) -org.mortbay.jetty:jetty-util:6.1.26 (11 constraints: 3799d04e) -org.mortbay.jetty:jsp-2.1:6.1.14 (2 constraints: 71154c11) -org.mortbay.jetty:jsp-api-2.1:6.1.14 (3 constraints: 5b20fbe2) -org.mortbay.jetty:servlet-api:2.5-20081211 (1 constraints: 390cbd19) -org.mortbay.jetty:servlet-api-2.5:6.1.14 (3 constraints: c221f470) -org.objenesis:objenesis:2.5.1 (3 constraints: 7d266d95) -org.ow2.asm:asm-all:5.0.2 (1 constraints: 0d0ceaf6) -org.pentaho:pentaho-aggdesigner-algorithm:5.1.5-jhyde (1 constraints: a40f6d84) -org.roaringbitmap:RoaringBitmap:0.7.45 (2 constraints: 2e1c26e3) +org.objenesis:objenesis:2.5.1 (2 constraints: 19198bcb) +org.roaringbitmap:RoaringBitmap:0.7.45 (1 constraints: 510d1c44) org.roaringbitmap:shims:0.7.45 (1 constraints: 260eb249) org.scala-lang:scala-library:2.11.12 (11 constraints: 5c9bfe44) org.scala-lang:scala-reflect:2.11.12 (1 constraints: 340fb09a) @@ -270,59 +221,30 @@ org.scala-lang.modules:scala-parser-combinators_2.11:1.1.0 (1 constraints: cf0e7 org.scala-lang.modules:scala-xml_2.11:1.0.6 (1 constraints: 080b84e9) org.slf4j:jcl-over-slf4j:1.7.16 (1 constraints: 500d1d44) org.slf4j:jul-to-slf4j:1.7.16 (1 constraints: 500d1d44) -org.slf4j:slf4j-api:1.7.25 (75 constraints: f240961e) +org.slf4j:slf4j-api:1.7.25 (49 constraints: f1d591ce) org.sonatype.sisu.inject:cglib:2.2.1-v20090111 (1 constraints: aa0cfd36) org.spark-project.hive:hive-exec:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.hive:hive-metastore:1.2.1.spark2 (1 constraints: 990fa09c) org.spark-project.spark:unused:1.0.0 (12 constraints: 9aab75cf) org.xerial.snappy:snappy-java:1.1.7.3 (2 constraints: 681c5e46) oro:oro:2.0.8 (3 constraints: 3b229337) -stax:stax-api:1.0.1 (3 constraints: 8d2668e5) -tomcat:jasper-compiler:5.5.23 (4 constraints: ff2fc367) -tomcat:jasper-runtime:5.5.23 (4 constraints: ff2fc367) +stax:stax-api:1.0.1 (2 constraints: ea186edd) +tomcat:jasper-compiler:5.5.23 (2 constraints: 93169c60) +tomcat:jasper-runtime:5.5.23 (2 constraints: 93169c60) xerces:xercesImpl:2.9.1 (1 constraints: ac0ccc0f) -xml-apis:xml-apis:1.3.04 (2 constraints: a20e0877) +xml-apis:xml-apis:1.3.04 (1 constraints: b008af8c) xmlenc:xmlenc:0.52 (3 constraints: 05228b2f) [Test dependencies] -com.beust:jcommander:1.30 (1 constraints: 8a0c0505) -com.klarna:hiverunner:4.1.0 (1 constraints: 07050236) -com.lmax:disruptor:3.3.0 (1 constraints: a80c770e) -com.ning:async-http-client:1.8.16 (1 constraints: 110f9671) -com.yammer.metrics:metrics-core:2.2.0 (2 constraints: 121c6ce2) -dom4j:dom4j:1.6.1 (1 constraints: 8c0ceb00) -javassist:javassist:3.12.1.GA (1 constraints: 710d3035) -net.sf.jpam:jpam:1.1 (1 constraints: f20b40e9) -org.apache.commons:commons-collections4:4.1 (2 constraints: 09137a51) -org.apache.commons:commons-math:2.2 (3 constraints: 322af180) -org.apache.hadoop:hadoop-archives:2.7.3 (1 constraints: f21198ff) +junit:junit:4.12 (1 constraints: db04ff30) +org.apache.curator:apache-curator:2.7.1 (1 constraints: 0c0bf8d6) org.apache.hadoop:hadoop-mapreduce-client-hs:2.7.3 (1 constraints: b60fac84) org.apache.hadoop:hadoop-minicluster:2.7.3 (1 constraints: 0e050d36) -org.apache.hadoop:hadoop-yarn-registry:2.7.3 (1 constraints: be0ccd11) org.apache.hadoop:hadoop-yarn-server-tests:2.7.3 (1 constraints: b60fac84) -org.apache.hbase:hbase-hadoop-compat:1.1.1 (4 constraints: 5438a206) -org.apache.hbase:hbase-hadoop2-compat:1.1.1 (3 constraints: e9285c23) -org.apache.hbase:hbase-prefix-tree:1.1.1 (1 constraints: a50c680e) -org.apache.hbase:hbase-procedure:1.1.1 (1 constraints: a50c680e) -org.apache.hbase:hbase-server:1.1.1 (1 constraints: cd0d4a3c) -org.apache.hive:hive-llap-client:2.3.6 (2 constraints: 651aa157) -org.apache.hive:hive-llap-common:2.3.6 (2 constraints: 911b96aa) -org.apache.hive:hive-llap-server:2.3.6 (1 constraints: 590cd001) -org.apache.hive:hive-llap-tez:2.3.6 (1 constraints: d50d5a3c) -org.apache.hive:hive-service:2.3.6 (1 constraints: 0d050436) -org.apache.hive.hcatalog:hive-hcatalog-core:2.3.3 (2 constraints: 882bfff0) -org.apache.hive.hcatalog:hive-hcatalog-server-extensions:2.3.3 (1 constraints: 2f14a67c) -org.apache.hive.hcatalog:hive-webhcat-java-client:2.3.3 (1 constraints: 060a73ad) -org.apache.slider:slider-core:0.90.2-incubating (1 constraints: 56124bfd) -org.apache.tez:hadoop-shim:0.9.1 (3 constraints: d3244224) -org.apache.tez:tez-api:0.9.1 (5 constraints: 1740d50b) -org.apache.tez:tez-common:0.9.1 (5 constraints: e13ec822) -org.apache.tez:tez-dag:0.9.1 (1 constraints: 080a79ad) -org.apache.tez:tez-mapreduce:0.9.1 (1 constraints: 080a79ad) -org.apache.tez:tez-runtime-internals:0.9.1 (1 constraints: e1099fb1) -org.apache.tez:tez-runtime-library:0.9.1 (2 constraints: 4b167422) -org.jamon:jamon-runtime:2.3.1 (2 constraints: fb1870e4) +org.apache.hive:hive-exec:2.3.6 (1 constraints: 0d050436) +org.apache.hive:hive-vector-code-gen:2.3.6 (1 constraints: 0d0bf1d6) +org.apache.velocity:velocity:1.5 (1 constraints: c70e875e) +org.codehaus.groovy:groovy-all:2.4.4 (1 constraints: 0c0bf2d6) +org.hamcrest:hamcrest-core:1.3 (2 constraints: 7910aeb0) org.mockito:mockito-core:1.10.19 (1 constraints: 6e059840) -org.mortbay.jetty:jetty-sslengine:6.1.26 (1 constraints: e10c631b) -org.reflections:reflections:0.9.8 (1 constraints: 0f0a80ad) org.slf4j:slf4j-simple:1.7.5 (1 constraints: 0f050a36) diff --git a/versions.props b/versions.props index a707737cdc11..a22c05f973ee 100644 --- a/versions.props +++ b/versions.props @@ -1,4 +1,5 @@ org.slf4j:slf4j-api = 1.7.22 +com.google.guava:guava = 28.0-jre org.apache.avro:avro = 1.9.2 org.apache.hadoop:* = 2.7.3 org.apache.hive:hive-metastore = 2.3.6 @@ -8,6 +9,7 @@ org.apache.spark:spark-hive_2.11 = 2.4.4 org.apache.spark:spark-avro_2.11 = 2.4.4 org.apache.pig:pig = 0.14.0 org.apache.commons:commons-lang3 = 3.9 +com.fasterxml.jackson.*:* = 2.10.0 com.github.ben-manes.caffeine:caffeine = 2.7.0 org.apache.arrow:arrow-vector = 0.14.1 @@ -16,4 +18,5 @@ junit:junit = 4.12 org.slf4j:slf4j-simple = 1.7.5 org.mockito:mockito-core = 1.10.19 joda-time:joda-time = 2.9.9 - +org.apache.hive:hive-exec = 2.3.6 +org.apache.hive:hive-metastore = 2.3.6 From 190fb37fb41c4a29d9227dc5b8effe7a1841c7b3 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 15 Apr 2020 16:04:32 +0100 Subject: [PATCH 25/51] tidy up, add tests, incorporate some code from upstream --- .../iceberg/mr/mapred/IcebergInputFormat.java | 65 ++++++++++++------- .../mr/mapred/IcebergReaderFactory.java | 26 ++++---- .../mr/mapred/TestIcebergInputFormat.java | 29 +++++---- .../test-table/metadata/v1.metadata.json | 2 +- .../test-table/metadata/v2.metadata.json | 4 +- 5 files changed, 73 insertions(+), 53 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 3718f8f544b1..3bbd3b014ba7 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -43,33 +43,41 @@ import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.mr.SerializationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class IcebergInputFormat implements InputFormat { +public class IcebergInputFormat implements InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); + static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; + private Table table; @Override - public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { - //TODO: Change this to use whichever Catalog the table was made with i.e. HiveCatalog instead etc. - HadoopTables tables = new HadoopTables(job); - String tableDir = job.get("location"); + public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { + table = findTable(conf); + CloseableIterable taskIterable = table.newScan().planTasks(); + List tasks = (List) StreamSupport + .stream(taskIterable.spliterator(), false) + .collect(Collectors.toList()); + return createSplits(tasks); + } + private Table findTable(JobConf conf) throws IOException { + HadoopTables tables = new HadoopTables(conf); + String tableDir = conf.get("location"); + if (tableDir == null) { + throw new IllegalArgumentException("Table 'location' not set in JobConf"); + } URI location = null; try { location = new URI(tableDir); } catch (URISyntaxException e) { - throw new IOException("Unable to create URI for table location: '" + tableDir + "'"); + throw new IOException("Unable to create URI for table location: '" + tableDir + "'", e); } table = tables.load(location.getPath()); - - CloseableIterable taskIterable = table.newScan().planTasks(); - List tasks = (List) StreamSupport - .stream(taskIterable.spliterator(), false) - .collect(Collectors.toList()); - return createSplits(tasks); + return table; } private InputSplit[] createSplits(List tasks) { @@ -86,17 +94,19 @@ public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter repo } public class IcebergRecordReader implements RecordReader { - private JobConf context; + private JobConf conf; private IcebergSplit split; private Iterator tasks; private CloseableIterable reader; private Iterator recordIterator; private Record currentRecord; + private boolean reuseContainers; public IcebergRecordReader(InputSplit split, JobConf conf) throws IOException { this.split = (IcebergSplit) split; - this.context = conf; + this.conf = conf; + this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); initialise(); } @@ -108,12 +118,10 @@ private void initialise() { private void nextTask() { FileScanTask currentTask = tasks.next(); DataFile file = currentTask.file(); - InputFile inputFile = HadoopInputFile.fromLocation(file.path(), context); + InputFile inputFile = HadoopInputFile.fromLocation(file.path(), conf); Schema tableSchema = table.schema(); - boolean reuseContainers = true; // FIXME: read from config - IcebergReaderFactory readerFactory = new IcebergReaderFactory(); - reader = readerFactory.createReader(file, currentTask, inputFile, tableSchema, reuseContainers); + reader = IcebergReaderFactory.createReader(file, currentTask, inputFile, tableSchema, reuseContainers); recordIterator = reader.iterator(); } @@ -126,6 +134,11 @@ public boolean next(Void key, IcebergWritable value) { } if (tasks.hasNext()) { + try { + reader.close(); + } catch (IOException e) { + LOG.error("Error closing reader", e); + } nextTask(); currentRecord = recordIterator.next(); value.setRecord(currentRecord); @@ -154,7 +167,7 @@ public long getPos() throws IOException { @Override public void close() throws IOException { - + reader.close(); } @Override @@ -165,6 +178,8 @@ public float getProgress() throws IOException { private static class IcebergSplit implements InputSplit { + private static final String[] ANYWHERE = new String[]{"*"}; + private CombinedScanTask task; IcebergSplit(CombinedScanTask task) { @@ -173,22 +188,26 @@ private static class IcebergSplit implements InputSplit { @Override public long getLength() throws IOException { - return 0; + return task.files().stream().mapToLong(FileScanTask::length).sum(); } @Override public String[] getLocations() throws IOException { - return new String[0]; + return ANYWHERE; } @Override public void write(DataOutput out) throws IOException { - + byte[] data = SerializationUtil.serializeToBytes(this.task); + out.writeInt(data.length); + out.write(data); } @Override public void readFields(DataInput in) throws IOException { - + byte[] data = new byte[in.readInt()]; + in.readFully(data); + this.task = SerializationUtil.deserializeFromBytes(data); } public CombinedScanTask getTask() { diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java index 5fbfd7f62c0c..37bb40f54bba 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java @@ -25,6 +25,7 @@ import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataReader; +import org.apache.iceberg.data.orc.GenericOrcReader; import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.InputFile; @@ -33,10 +34,10 @@ class IcebergReaderFactory { - IcebergReaderFactory() { + private IcebergReaderFactory() { } - public CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, + public static CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, Schema tableSchema, boolean reuseContainers) { switch (file.format()) { case AVRO: @@ -52,9 +53,9 @@ public CloseableIterable createReader(DataFile file, FileScanTask curren } } - // FIXME: use generic reader function - private CloseableIterable buildAvroReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { - Avro.ReadBuilder builder = Avro.read(file) + private static CloseableIterable buildAvroReader(FileScanTask task, InputFile inputFile, Schema schema, + boolean reuseContainers) { + Avro.ReadBuilder builder = Avro.read(inputFile) .createReaderFunc(DataReader::create) .project(schema) .split(task.start(), task.length()); @@ -66,21 +67,20 @@ private CloseableIterable buildAvroReader(FileScanTask task, InputFile file, Sch return builder.build(); } - // FIXME: use generic reader function - private CloseableIterable buildOrcReader(FileScanTask task, InputFile file, Schema schema, boolean reuseContainers) { - ORC.ReadBuilder builder = ORC.read(file) -// .createReaderFunc() // FIXME: implement + private static CloseableIterable buildOrcReader(FileScanTask task, InputFile inputFile, Schema schema, + boolean reuseContainers) { + ORC.ReadBuilder builder = ORC.read(inputFile) + .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema)) .project(schema) .split(task.start(), task.length()); return builder.build(); } - // FIXME: use generic reader function - private CloseableIterable buildParquetReader(FileScanTask task, InputFile file, Schema schema, + private static CloseableIterable buildParquetReader(FileScanTask task, InputFile inputFile, Schema schema, boolean reuseContainers) { - Parquet.ReadBuilder builder = Parquet.read(file) - .createReaderFunc(messageType -> GenericParquetReaders.buildReader(schema, messageType)) + Parquet.ReadBuilder builder = Parquet.read(inputFile) + .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema)) .project(schema) .split(task.start(), task.length()); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 1e228fc3300c..cf1fb878a838 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -50,10 +50,9 @@ public class TestIcebergInputFormat { private File tableLocation; private Table table; + private IcebergInputFormat format = new IcebergInputFormat(); + private JobConf conf = new JobConf(); - //TODO flesh out with more tests of the IF itself - //TODO: do we still need the table data etc. if we're not testing from Hive? - @Before public void before() throws IOException { tableLocation = java.nio.file.Files.createTempDirectory("temp").toFile(); @@ -75,30 +74,32 @@ public void before() throws IOException { @Test public void testGetSplits() throws IOException { - IcebergInputFormat format = new IcebergInputFormat(); - JobConf conf = new JobConf(); conf.set("location", "file:" + tableLocation); InputSplit[] splits = format.getSplits(conf, 1); assertEquals(splits.length, 1); } + @Test(expected = IllegalArgumentException.class) + public void testGetSplitsNoLocation() throws IOException { + format.getSplits(conf, 1); + } + + @Test(expected = IOException.class) + public void testGetSplitsInvalidLocationUri() throws IOException { + conf.set("location", "http:"); + format.getSplits(conf, 1); + } + @Test public void testGetRecordReader() throws IOException { - IcebergInputFormat format = new IcebergInputFormat(); - JobConf conf = new JobConf(); conf.set("location", "file:" + tableLocation); InputSplit[] splits = format.getSplits(conf, 1); RecordReader reader = format.getRecordReader(splits[0], conf, null); IcebergWritable value = (IcebergWritable) reader.createValue(); List records = Lists.newArrayList(); - boolean unfinished = true; - while (unfinished) { - if (reader.next(null, value)) { - records.add(value.getRecord().copy()); - } else { - unfinished = false; - } + while (reader.next(null, value)) { + records.add(value.getRecord().copy()); } assertEquals(3, records.size()); } diff --git a/mr/src/test/resources/test-table/metadata/v1.metadata.json b/mr/src/test/resources/test-table/metadata/v1.metadata.json index d14ac4529e3f..0c08d1732619 100644 --- a/mr/src/test/resources/test-table/metadata/v1.metadata.json +++ b/mr/src/test/resources/test-table/metadata/v1.metadata.json @@ -1,7 +1,7 @@ { "format-version" : 1, "table-uuid" : "5ec03633-03bc-4c4b-8ef9-f799c143e3e7", - "location" : "/Users/cmathiesen/projects/opensource/forks/eg-iceberg-fork/incubator-iceberg/mr/src/test/resources/test-table", + "location" : "mr/src/test/resources/test-table", "last-updated-ms" : 1582645440292, "last-column-id" : 2, "schema" : { diff --git a/mr/src/test/resources/test-table/metadata/v2.metadata.json b/mr/src/test/resources/test-table/metadata/v2.metadata.json index ea938f9a0dbe..80a84f69913f 100644 --- a/mr/src/test/resources/test-table/metadata/v2.metadata.json +++ b/mr/src/test/resources/test-table/metadata/v2.metadata.json @@ -1,7 +1,7 @@ { "format-version" : 1, "table-uuid" : "5ec03633-03bc-4c4b-8ef9-f799c143e3e7", - "location" : "/Users/cmathiesen/projects/opensource/forks/eg-iceberg-fork/incubator-iceberg/mr/src/test/resources/test-table", + "location" : "mr/src/test/resources/test-table", "last-updated-ms" : 1582645443979, "last-column-id" : 2, "schema" : { @@ -38,7 +38,7 @@ "total-records" : "3", "total-data-files" : "1" }, - "manifest-list" : "/Users/cmathiesen/projects/opensource/forks/eg-iceberg-fork/incubator-iceberg/mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro" + "manifest-list" : "mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro" } ], "snapshot-log" : [ { "timestamp-ms" : 1582645443979, From dc8f6a1ee00d942d0629fe25eaaadf093e31acc3 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 16 Apr 2020 15:03:52 +0100 Subject: [PATCH 26/51] revert public access --- core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java index 4ca36b22fc4a..f3586840854a 100644 --- a/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java +++ b/core/src/main/java/org/apache/iceberg/avro/TypeToSchema.java @@ -31,7 +31,7 @@ import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -public class TypeToSchema extends TypeUtil.SchemaVisitor { +class TypeToSchema extends TypeUtil.SchemaVisitor { private static final Schema BOOLEAN_SCHEMA = Schema.create(Schema.Type.BOOLEAN); private static final Schema INTEGER_SCHEMA = Schema.create(Schema.Type.INT); private static final Schema LONG_SCHEMA = Schema.create(Schema.Type.LONG); From ce4e88cc7e731bdd4792c089883b9280e400f023 Mon Sep 17 00:00:00 2001 From: cmathiesen Date: Mon, 20 Apr 2020 14:13:55 +0100 Subject: [PATCH 27/51] Fix mapred serialization bug (#6) * Fix mapred serialization bug --- build.gradle | 13 +++++ .../iceberg/mr/mapred/IcebergInputFormat.java | 55 ++++++++++++++++--- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index bbba9160174c..b8cd487b44d9 100644 --- a/build.gradle +++ b/build.gradle @@ -232,6 +232,19 @@ project(':iceberg-mr') { exclude group: 'org.apache.avro', module: 'avro' } + compileOnly("org.apache.hive:hive-exec::core") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.pentaho' // missing dependency + exclude group: 'org.apache.hive', module: 'hive-llap-tez' + exclude group: 'org.apache.logging.log4j' + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.calcite' + exclude group: 'org.apache.calcite.avatica' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'com.google.guava' + } + testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 3bbd3b014ba7..82ca2bc9bfe7 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -28,6 +28,10 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -47,7 +51,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class IcebergInputFormat implements InputFormat { +/** + * CombineHiveInputFormat.AvoidSplitCombination is implemented to correctly delegate InputSplit + * creation to this class. See: https://stackoverflow.com/questions/29133275/ + * custom-inputformat-getsplits-never-called-in-hive + */ +public class IcebergInputFormat implements InputFormat, CombineHiveInputFormat.AvoidSplitCombination { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; @@ -61,7 +70,7 @@ public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { List tasks = (List) StreamSupport .stream(taskIterable.spliterator(), false) .collect(Collectors.toList()); - return createSplits(tasks); + return createSplits(tasks, table.location()); } private Table findTable(JobConf conf) throws IOException { @@ -80,10 +89,10 @@ private Table findTable(JobConf conf) throws IOException { return table; } - private InputSplit[] createSplits(List tasks) { + private InputSplit[] createSplits(List tasks, String location) { InputSplit[] splits = new InputSplit[tasks.size()]; for (int i = 0; i < tasks.size(); i++) { - splits[i] = new IcebergSplit(tasks.get(i)); + splits[i] = new IcebergSplit(tasks.get(i), location); } return splits; } @@ -93,6 +102,11 @@ public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter repo return new IcebergRecordReader(split, job); } + @Override + public boolean shouldSkipCombine(Path path, Configuration conf) throws IOException { + return true; + } + public class IcebergRecordReader implements RecordReader { private JobConf conf; private IcebergSplit split; @@ -176,18 +190,27 @@ public float getProgress() throws IOException { } } - private static class IcebergSplit implements InputSplit { + /** + * FileSplit is extended rather than implementing the InputSplit interface due to Hive's HiveInputFormat + * expecting a split which is an instance of FileSplit. + */ + private static class IcebergSplit extends FileSplit { private static final String[] ANYWHERE = new String[]{"*"}; private CombinedScanTask task; + private String partitionLocation; - IcebergSplit(CombinedScanTask task) { + IcebergSplit() { + } + + IcebergSplit(CombinedScanTask task, String partitionLocation) { this.task = task; + this.partitionLocation = partitionLocation; } @Override - public long getLength() throws IOException { + public long getLength() { return task.files().stream().mapToLong(FileScanTask::length).sum(); } @@ -196,11 +219,25 @@ public String[] getLocations() throws IOException { return ANYWHERE; } + @Override + public Path getPath() { + return new Path(partitionLocation); + } + + @Override + public long getStart() { + return 0L; + } + @Override public void write(DataOutput out) throws IOException { byte[] data = SerializationUtil.serializeToBytes(this.task); out.writeInt(data.length); out.write(data); + + byte[] tableLocation = SerializationUtil.serializeToBytes(this.partitionLocation); + out.writeInt(tableLocation.length); + out.write(tableLocation); } @Override @@ -208,6 +245,10 @@ public void readFields(DataInput in) throws IOException { byte[] data = new byte[in.readInt()]; in.readFully(data); this.task = SerializationUtil.deserializeFromBytes(data); + + byte[] location = new byte[in.readInt()]; + in.readFully(location); + this.partitionLocation = SerializationUtil.deserializeFromBytes(location); } public CombinedScanTask getTask() { From a013e34548d187ffb198c617749fff63d0e54a38 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Fri, 24 Apr 2020 17:12:01 +0100 Subject: [PATCH 28/51] remove test data --- ...c3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc | Bin ...557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet | Bin ...1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc | Bin ...-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc | Bin .../metadata/.v1.metadata.json.crc | Bin .../metadata/.v2.metadata.json.crc | Bin .../metadata/.version-hint.text.crc | Bin .../1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro | Bin ...1706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro | Bin .../metadata/v1.metadata.json | 0 .../metadata/v2.metadata.json | 0 .../metadata/version-hint.text | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename mr/src/test/resources/{test-table => test-table-to-delete}/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/.v1.metadata.json.crc (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/.v2.metadata.json.crc (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/.version-hint.text.crc (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/v1.metadata.json (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/v2.metadata.json (100%) rename mr/src/test/resources/{test-table => test-table-to-delete}/metadata/version-hint.text (100%) diff --git a/mr/src/test/resources/test-table/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc b/mr/src/test/resources/test-table-to-delete/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc similarity index 100% rename from mr/src/test/resources/test-table/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc rename to mr/src/test/resources/test-table-to-delete/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc diff --git a/mr/src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet b/mr/src/test/resources/test-table-to-delete/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet similarity index 100% rename from mr/src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet rename to mr/src/test/resources/test-table-to-delete/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet diff --git a/mr/src/test/resources/test-table/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc b/mr/src/test/resources/test-table-to-delete/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc similarity index 100% rename from mr/src/test/resources/test-table/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc rename to mr/src/test/resources/test-table-to-delete/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc diff --git a/mr/src/test/resources/test-table/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc b/mr/src/test/resources/test-table-to-delete/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc similarity index 100% rename from mr/src/test/resources/test-table/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc rename to mr/src/test/resources/test-table-to-delete/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc diff --git a/mr/src/test/resources/test-table/metadata/.v1.metadata.json.crc b/mr/src/test/resources/test-table-to-delete/metadata/.v1.metadata.json.crc similarity index 100% rename from mr/src/test/resources/test-table/metadata/.v1.metadata.json.crc rename to mr/src/test/resources/test-table-to-delete/metadata/.v1.metadata.json.crc diff --git a/mr/src/test/resources/test-table/metadata/.v2.metadata.json.crc b/mr/src/test/resources/test-table-to-delete/metadata/.v2.metadata.json.crc similarity index 100% rename from mr/src/test/resources/test-table/metadata/.v2.metadata.json.crc rename to mr/src/test/resources/test-table-to-delete/metadata/.v2.metadata.json.crc diff --git a/mr/src/test/resources/test-table/metadata/.version-hint.text.crc b/mr/src/test/resources/test-table-to-delete/metadata/.version-hint.text.crc similarity index 100% rename from mr/src/test/resources/test-table/metadata/.version-hint.text.crc rename to mr/src/test/resources/test-table-to-delete/metadata/.version-hint.text.crc diff --git a/mr/src/test/resources/test-table/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro b/mr/src/test/resources/test-table-to-delete/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro similarity index 100% rename from mr/src/test/resources/test-table/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro rename to mr/src/test/resources/test-table-to-delete/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro diff --git a/mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro b/mr/src/test/resources/test-table-to-delete/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro similarity index 100% rename from mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro rename to mr/src/test/resources/test-table-to-delete/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro diff --git a/mr/src/test/resources/test-table/metadata/v1.metadata.json b/mr/src/test/resources/test-table-to-delete/metadata/v1.metadata.json similarity index 100% rename from mr/src/test/resources/test-table/metadata/v1.metadata.json rename to mr/src/test/resources/test-table-to-delete/metadata/v1.metadata.json diff --git a/mr/src/test/resources/test-table/metadata/v2.metadata.json b/mr/src/test/resources/test-table-to-delete/metadata/v2.metadata.json similarity index 100% rename from mr/src/test/resources/test-table/metadata/v2.metadata.json rename to mr/src/test/resources/test-table-to-delete/metadata/v2.metadata.json diff --git a/mr/src/test/resources/test-table/metadata/version-hint.text b/mr/src/test/resources/test-table-to-delete/metadata/version-hint.text similarity index 100% rename from mr/src/test/resources/test-table/metadata/version-hint.text rename to mr/src/test/resources/test-table-to-delete/metadata/version-hint.text From 6d24d41bdc221f2185e67f1b2ac6d2cdf0cbadf0 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Fri, 24 Apr 2020 17:12:21 +0100 Subject: [PATCH 29/51] generate test data in tests --- .../org/apache/iceberg/mr/TestHelpers.java | 97 ++++++++++++++++++ .../mr/mapred/TestIcebergInputFormat.java | 99 +++++++++++-------- .../mr/mapreduce/TestIcebergInputFormat.java | 83 +++------------- 3 files changed, 169 insertions(+), 110 deletions(-) create mode 100644 mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java b/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java new file mode 100644 index 000000000000..b2a047ed87e6 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataWriter; +import org.apache.iceberg.data.orc.GenericOrcWriter; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; + +/** + * + */ +public class TestHelpers { + + private TestHelpers() {} + + public static DataFile writeFile(File targetFile, + Table table, StructLike partitionData, FileFormat fileFormat, List records) throws IOException { + if (targetFile.exists()) { + if (!targetFile.delete()) { + throw new IOException("Unable to delete " + targetFile.getAbsolutePath()); + } + } + FileAppender appender; + switch (fileFormat) { + case AVRO: + appender = Avro.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(DataWriter::create) + .named(fileFormat.name()) + .build(); + break; + case PARQUET: + appender = Parquet.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(GenericParquetWriter::buildWriter) + .named(fileFormat.name()) + .build(); + break; + case ORC: + appender = ORC.write(Files.localOutput(targetFile)) + .schema(table.schema()) + .createWriterFunc(GenericOrcWriter::buildWriter) + .build(); + break; + default: + throw new UnsupportedOperationException("Cannot write format: " + fileFormat); + } + + try { + appender.addAll(records); + } finally { + appender.close(); + } + + DataFiles.Builder builder = DataFiles.builder(table.spec()) + .withPath(targetFile.toString()) + .withFormat(fileFormat) + .withFileSizeInBytes(targetFile.length()) + .withMetrics(appender.metrics()); + if (partitionData != null) { + builder.withPartition(partitionData); + } + return builder.build(); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index cf1fb878a838..cacd022b5161 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -19,82 +19,98 @@ package org.apache.iceberg.mr.mapred; +import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Locale; import org.apache.commons.compress.utils.Lists; -import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.types.Types; -import org.junit.After; -import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.mr.TestHelpers.writeFile; +import static org.apache.iceberg.types.Types.NestedField.required; import static org.junit.Assert.assertEquals; +@RunWith(Parameterized.class) public class TestIcebergInputFormat { private static final Logger LOG = LoggerFactory.getLogger(TestIcebergInputFormat.class); - private File tableLocation; - private Table table; - private IcebergInputFormat format = new IcebergInputFormat(); - private JobConf conf = new JobConf(); - - @Before - public void before() throws IOException { - tableLocation = java.nio.file.Files.createTempDirectory("temp").toFile(); - Schema schema = new Schema(optional(1, "name", Types.StringType.get()), - optional(2, "salary", Types.LongType.get())); - PartitionSpec spec = PartitionSpec.unpartitioned(); - HadoopTables tables = new HadoopTables(); - table = tables.create(schema, spec, tableLocation.getAbsolutePath()); - - DataFile fileA = DataFiles - .builder(spec) - .withPath("src/test/resources/test-table/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet") - .withFileSizeInBytes(1024) - .withRecordCount(3) // needs at least one record or else metrics will filter it out - .build(); - - table.newAppend().appendFile(fileA).commit(); + static final Schema SCHEMA = new Schema(required(1, "data", Types.StringType.get()), + required(2, "id", Types.LongType.get()), required(3, "date", Types.StringType.get())); + + static final PartitionSpec SPEC = + PartitionSpec.builderFor(SCHEMA).identity("date").bucket("id", 1).build(); + + private IcebergInputFormat inputFormat = new IcebergInputFormat(); + private JobConf jobConf = new JobConf(); + private Configuration conf = new Configuration(); + private HadoopTables tables = new HadoopTables(conf); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Parameterized.Parameters + public static Object[][] parameters() { + return new Object[][] { new Object[] { "parquet" }, new Object[] { "avro" } + /* + * , TODO: put orc back, seems to be an issue with different versions of Orc in Hive and + * Iceberg new Object[]{"orc"} + */ + }; } - @Test - public void testGetSplits() throws IOException { - conf.set("location", "file:" + tableLocation); - InputSplit[] splits = format.getSplits(conf, 1); - assertEquals(splits.length, 1); + private final FileFormat fileFormat; + + public TestIcebergInputFormat(String fileFormat) { + this.fileFormat = FileFormat.valueOf(fileFormat.toUpperCase(Locale.ENGLISH)); } @Test(expected = IllegalArgumentException.class) public void testGetSplitsNoLocation() throws IOException { - format.getSplits(conf, 1); + inputFormat.getSplits(jobConf, 1); } @Test(expected = IOException.class) public void testGetSplitsInvalidLocationUri() throws IOException { - conf.set("location", "http:"); - format.getSplits(conf, 1); + jobConf.set("location", "http:"); + inputFormat.getSplits(jobConf, 1); } @Test public void testGetRecordReader() throws IOException { - conf.set("location", "file:" + tableLocation); - InputSplit[] splits = format.getSplits(conf, 1); - RecordReader reader = format.getRecordReader(splits[0], conf, null); + File tableLocation = temp.newFolder(fileFormat.name()); + Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), + tableLocation.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 3, 0L); + + DataFile dataFile = writeFile(temp.newFile(), table, null, fileFormat, expectedRecords); + table.newAppend().appendFile(dataFile).commit(); + + jobConf.set("location", "file:" + tableLocation); + InputSplit[] splits = inputFormat.getSplits(jobConf, 1); + RecordReader reader = inputFormat.getRecordReader(splits[0], jobConf, null); IcebergWritable value = (IcebergWritable) reader.createValue(); List records = Lists.newArrayList(); @@ -104,8 +120,7 @@ public void testGetRecordReader() throws IOException { assertEquals(3, records.size()); } - @After - public void after() throws IOException { - FileUtils.deleteDirectory(tableLocation); - } + // TODO: add more tests, based on the mapreduce InputFormat tests (possibly refactor shared code + // into an abstract parent test class). + } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index 1f84890f8267..170e2f207075 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -42,29 +41,20 @@ import org.apache.iceberg.AppendFiles; import org.apache.iceberg.AssertHelpers; import org.apache.iceberg.DataFile; -import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; -import org.apache.iceberg.Files; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TestHelpers.Row; -import org.apache.iceberg.avro.Avro; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.avro.DataWriter; -import org.apache.iceberg.data.orc.GenericOrcWriter; -import org.apache.iceberg.data.parquet.GenericParquetWriter; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.mr.TestHelpers; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.junit.Assert; @@ -75,6 +65,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import static org.apache.iceberg.mr.TestHelpers.writeFile; import static org.apache.iceberg.types.Types.NestedField.required; @RunWith(Parameterized.class) @@ -123,7 +114,7 @@ public void testUnpartitionedTable() throws Exception { ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(table, null, format, expectedRecords); + DataFile dataFile = TestHelpers.writeFile(temp.newFile(), table, null, format, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); @@ -142,7 +133,7 @@ public void testPartitionedTable() throws Exception { location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); @@ -163,8 +154,8 @@ public void testFilterExp() throws Exception { List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); expectedRecords.get(0).set(2, "2020-03-20"); expectedRecords.get(1).set(2, "2020-03-20"); - DataFile dataFile1 = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords); - DataFile dataFile2 = writeFile(table, Row.of("2020-03-21", 0), format, + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, RandomGenericData.generate(table.schema(), 2, 0L)); table.newAppend() .appendFile(dataFile1) @@ -187,7 +178,7 @@ public void testResiduals() throws Exception { List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); expectedRecords.get(0).set(2, "2020-03-20"); expectedRecords.get(1).set(2, "2020-03-20"); - DataFile dataFile = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); @@ -213,7 +204,7 @@ public void testProjection() throws Exception { ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), location.toString()); List inputRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(table, Row.of("2020-03-20", 0), format, inputRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, inputRecords); table.newAppend() .appendFile(dataFile) .commit(); @@ -252,7 +243,8 @@ public void testIdentityPartitionProjections() throws Exception { for (Record record : inputRecords) { record.set(1, "2020-03-2" + idx); record.set(2, idx.toString()); - append.appendFile(writeFile(table, Row.of("2020-03-2" + idx, idx.toString()), format, ImmutableList.of(record))); + append.appendFile(writeFile(temp.newFile(), table, Row.of("2020-03-2" + idx, idx.toString()), + format, ImmutableList.of(record))); idx += 1; } append.commit(); @@ -320,11 +312,12 @@ public void testSnapshotReads() throws Exception { location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); table.newAppend() - .appendFile(writeFile(table, null, format, expectedRecords)) + .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) .commit(); long snapshotId = table.currentSnapshot().snapshotId(); table.newAppend() - .appendFile(writeFile(table, null, format, RandomGenericData.generate(table.schema(), 1, 0L))) + .appendFile(writeFile(temp.newFile(), table, null, format, + RandomGenericData.generate(table.schema(), 1, 0L))) .commit(); Job job = Job.getInstance(conf); @@ -345,7 +338,7 @@ public void testLocality() throws Exception { location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); table.newAppend() - .appendFile(writeFile(table, null, format, expectedRecords)) + .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) .commit(); Job job = Job.getInstance(conf); IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); @@ -379,7 +372,7 @@ public void testCustomCatalog() throws Exception { ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name())); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); @@ -429,50 +422,4 @@ private static Iterable readRecords( return records; } - private DataFile writeFile( - Table table, StructLike partitionData, FileFormat fileFormat, List records) throws IOException { - File file = temp.newFile(); - Assert.assertTrue(file.delete()); - FileAppender appender; - switch (fileFormat) { - case AVRO: - appender = Avro.write(Files.localOutput(file)) - .schema(table.schema()) - .createWriterFunc(DataWriter::create) - .named(fileFormat.name()) - .build(); - break; - case PARQUET: - appender = Parquet.write(Files.localOutput(file)) - .schema(table.schema()) - .createWriterFunc(GenericParquetWriter::buildWriter) - .named(fileFormat.name()) - .build(); - break; - case ORC: - appender = ORC.write(Files.localOutput(file)) - .schema(table.schema()) - .createWriterFunc(GenericOrcWriter::buildWriter) - .build(); - break; - default: - throw new UnsupportedOperationException("Cannot write format: " + fileFormat); - } - - try { - appender.addAll(records); - } finally { - appender.close(); - } - - DataFiles.Builder builder = DataFiles.builder(table.spec()) - .withPath(file.toString()) - .withFormat(format) - .withFileSizeInBytes(file.length()) - .withMetrics(appender.metrics()); - if (partitionData != null) { - builder.withPartition(partitionData); - } - return builder.build(); - } } From ce76c36cb3090d4321b1bf3c8fe268bab85b9d9a Mon Sep 17 00:00:00 2001 From: awoodhead Date: Fri, 24 Apr 2020 17:19:16 +0100 Subject: [PATCH 30/51] fix missed static call --- .../org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index 170e2f207075..95963ad4ef89 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -114,7 +114,7 @@ public void testUnpartitionedTable() throws Exception { ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = TestHelpers.writeFile(temp.newFile(), table, null, format, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, null, format, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); From 0d25ea61c8a675d0b8f85ddb47ebb65b7aeda5b8 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 4 May 2020 17:55:42 +0100 Subject: [PATCH 31/51] - refactor tests common to both input formats --- .../iceberg/mr/mapred/IcebergInputFormat.java | 7 +- .../mr/mapred/IcebergReaderFactory.java | 14 +- .../iceberg/mr/BaseInputFormatTest.java | 110 ++++++++++++++ .../org/apache/iceberg/mr/TestHelpers.java | 31 ++++ .../mr/mapred/TestIcebergInputFormat.java | 88 ++++-------- .../mr/mapreduce/TestIcebergInputFormat.java | 134 +++++------------- 6 files changed, 217 insertions(+), 167 deletions(-) create mode 100644 mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 82ca2bc9bfe7..1b4428128b4c 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -59,6 +59,7 @@ public class IcebergInputFormat implements InputFormat, CombineHiveInputFormat.AvoidSplitCombination { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); + static final String TABLE_LOCATION = "location"; static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; private Table table; @@ -75,7 +76,7 @@ public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { private Table findTable(JobConf conf) throws IOException { HadoopTables tables = new HadoopTables(conf); - String tableDir = conf.get("location"); + String tableDir = conf.get(TABLE_LOCATION); if (tableDir == null) { throw new IllegalArgumentException("Table 'location' not set in JobConf"); } @@ -134,8 +135,8 @@ private void nextTask() { DataFile file = currentTask.file(); InputFile inputFile = HadoopInputFile.fromLocation(file.path(), conf); Schema tableSchema = table.schema(); - - reader = IcebergReaderFactory.createReader(file, currentTask, inputFile, tableSchema, reuseContainers); + IcebergReaderFactory readerFactory = new IcebergReaderFactory(); + reader = readerFactory.createReader(file, currentTask, inputFile, tableSchema, reuseContainers); recordIterator = reader.iterator(); } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java index 37bb40f54bba..1f3f8e6489aa 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java @@ -23,7 +23,6 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataReader; import org.apache.iceberg.data.orc.GenericOrcReader; import org.apache.iceberg.data.parquet.GenericParquetReaders; @@ -32,12 +31,9 @@ import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; -class IcebergReaderFactory { +class IcebergReaderFactory { - private IcebergReaderFactory() { - } - - public static CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, + public CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, Schema tableSchema, boolean reuseContainers) { switch (file.format()) { case AVRO: @@ -53,7 +49,7 @@ public static CloseableIterable createReader(DataFile file, FileScanTask } } - private static CloseableIterable buildAvroReader(FileScanTask task, InputFile inputFile, Schema schema, + private CloseableIterable buildAvroReader(FileScanTask task, InputFile inputFile, Schema schema, boolean reuseContainers) { Avro.ReadBuilder builder = Avro.read(inputFile) .createReaderFunc(DataReader::create) @@ -67,7 +63,7 @@ private static CloseableIterable buildAvroReader(FileScanTask task, InputFile in return builder.build(); } - private static CloseableIterable buildOrcReader(FileScanTask task, InputFile inputFile, Schema schema, + private CloseableIterable buildOrcReader(FileScanTask task, InputFile inputFile, Schema schema, boolean reuseContainers) { ORC.ReadBuilder builder = ORC.read(inputFile) .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema)) @@ -77,7 +73,7 @@ private static CloseableIterable buildOrcReader(FileScanTask task, InputFile inp return builder.build(); } - private static CloseableIterable buildParquetReader(FileScanTask task, InputFile inputFile, Schema schema, + private CloseableIterable buildParquetReader(FileScanTask task, InputFile inputFile, Schema schema, boolean reuseContainers) { Parquet.ReadBuilder builder = Parquet.read(inputFile) .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema)) diff --git a/mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java b/mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java new file mode 100644 index 000000000000..8aa24aeca522 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.data.RandomGenericData; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.TestHelpers.Row; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.iceberg.mr.TestHelpers.writeFile; +import static org.apache.iceberg.types.Types.NestedField.required; + + +@RunWith(Parameterized.class) +public abstract class BaseInputFormatTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Parameterized.Parameters + public static Object[][] parameters() { + return new Object[][]{ + new Object[]{"parquet"}, + new Object[]{"avro"}, + new Object[]{"orc"} + }; + } + + protected static final Schema SCHEMA = new Schema( + required(1, "data", Types.StringType.get()), + required(2, "id", Types.LongType.get()), + required(3, "date", Types.StringType.get())); + + protected static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) + .identity("date") + .bucket("id", 1) + .build(); + + protected Configuration conf = new Configuration(); + protected HadoopTables tables = new HadoopTables(conf); + + protected FileFormat fileFormat; + + protected abstract void runAndValidate(File tableLocation, List expectedRecords) throws IOException; + + @Test + public void testUnpartitionedTable() throws Exception { + File tableLocation = temp.newFolder(fileFormat.name()); + Table table = tables + .create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), tableLocation.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + DataFile dataFile = writeFile(temp.newFile(), table, null, fileFormat, expectedRecords); + table.newAppend().appendFile(dataFile).commit(); + runAndValidate(tableLocation, expectedRecords); + } + + @Test + public void testPartitionedTable() throws Exception { + File tableLocation = temp.newFolder(fileFormat.name()); + Assert.assertTrue(tableLocation.delete()); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), + tableLocation.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + expectedRecords.get(0).set(2, "2020-03-20"); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + + runAndValidate(tableLocation, expectedRecords); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java b/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java index b2a047ed87e6..bddddd6afefc 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java +++ b/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java @@ -44,6 +44,37 @@ public class TestHelpers { private TestHelpers() {} + /** + * Implements {@link StructLike#get} for passing data in tests. + */ + public static class Row implements StructLike { + public static Row of(Object... values) { + return new Row(values); + } + + private final Object[] values; + + private Row(Object... values) { + this.values = values; + } + + @Override + public int size() { + return values.length; + } + + @Override + @SuppressWarnings("unchecked") + public T get(int pos, Class javaClass) { + return javaClass.cast(values[pos]); + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException("Setting values is not supported"); + } + } + public static DataFile writeFile(File targetFile, Table table, StructLike partitionData, FileFormat fileFormat, List records) throws IOException { if (targetFile.exists()) { diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index cacd022b5161..58cd1afeb5b2 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -19,108 +19,82 @@ package org.apache.iceberg.mr.mapred; -import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.List; import java.util.Locale; -import org.apache.commons.compress.utils.Lists; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; -import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.types.Types; -import org.junit.Rule; +import org.apache.iceberg.mr.BaseInputFormatTest; +import org.junit.Assert; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.iceberg.mr.TestHelpers.writeFile; -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class TestIcebergInputFormat { +public class TestIcebergInputFormat extends BaseInputFormatTest { private static final Logger LOG = LoggerFactory.getLogger(TestIcebergInputFormat.class); - static final Schema SCHEMA = new Schema(required(1, "data", Types.StringType.get()), - required(2, "id", Types.LongType.get()), required(3, "date", Types.StringType.get())); - - static final PartitionSpec SPEC = - PartitionSpec.builderFor(SCHEMA).identity("date").bucket("id", 1).build(); - private IcebergInputFormat inputFormat = new IcebergInputFormat(); - private JobConf jobConf = new JobConf(); - private Configuration conf = new Configuration(); - private HadoopTables tables = new HadoopTables(conf); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); @Parameterized.Parameters public static Object[][] parameters() { return new Object[][] { new Object[] { "parquet" }, new Object[] { "avro" } /* - * , TODO: put orc back, seems to be an issue with different versions of Orc in Hive and - * Iceberg new Object[]{"orc"} + * , TODO: put orc back, seems to be an issue with different versions of Orc in Hive and Iceberg new + * Object[]{"orc"} */ }; } - private final FileFormat fileFormat; - public TestIcebergInputFormat(String fileFormat) { this.fileFormat = FileFormat.valueOf(fileFormat.toUpperCase(Locale.ENGLISH)); } @Test(expected = IllegalArgumentException.class) public void testGetSplitsNoLocation() throws IOException { + JobConf jobConf = new JobConf(); inputFormat.getSplits(jobConf, 1); } @Test(expected = IOException.class) public void testGetSplitsInvalidLocationUri() throws IOException { - jobConf.set("location", "http:"); + JobConf jobConf = new JobConf(); + jobConf.set(IcebergInputFormat.TABLE_LOCATION, "http:"); inputFormat.getSplits(jobConf, 1); } - @Test - public void testGetRecordReader() throws IOException { - File tableLocation = temp.newFolder(fileFormat.name()); - Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - tableLocation.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 3, 0L); + @Override + protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { + JobConf jobConf = new JobConf(); + jobConf.set(IcebergInputFormat.TABLE_LOCATION, "file:" + tableLocation); + validate(jobConf, expectedRecords); + } - DataFile dataFile = writeFile(temp.newFile(), table, null, fileFormat, expectedRecords); - table.newAppend().appendFile(dataFile).commit(); + private void validate(JobConf jobConf, List expectedRecords) throws IOException { + List actualRecords = readRecords(jobConf); + Assert.assertEquals(expectedRecords, actualRecords); + } - jobConf.set("location", "file:" + tableLocation); + private List readRecords(JobConf jobConf) throws IOException { InputSplit[] splits = inputFormat.getSplits(jobConf, 1); - RecordReader reader = inputFormat.getRecordReader(splits[0], jobConf, null); - IcebergWritable value = (IcebergWritable) reader.createValue(); - - List records = Lists.newArrayList(); - while (reader.next(null, value)) { - records.add(value.getRecord().copy()); + try { + RecordReader reader = inputFormat.getRecordReader(splits[0], jobConf, null); + List records = new ArrayList<>(); + IcebergWritable value = (IcebergWritable) reader.createValue(); + while (reader.next(null, value)) { + records.add(value.getRecord().copy()); + } + return records; + } catch (IOException e) { + throw new UncheckedIOException(e); } - assertEquals(3, records.size()); } - // TODO: add more tests, based on the mapreduce InputFormat tests (possibly refactor shared code - // into an abstract parent test class). - } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index 2dd49191dc09..957f165ea7f4 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -46,115 +47,52 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.TestHelpers.Row; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.BaseInputFormatTest; +import org.apache.iceberg.mr.TestHelpers.Row; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import static org.apache.iceberg.mr.TestHelpers.writeFile; -import static org.apache.iceberg.types.Types.NestedField.required; @RunWith(Parameterized.class) -public class TestIcebergInputFormat { - static final Schema SCHEMA = new Schema( - required(1, "data", Types.StringType.get()), - required(2, "id", Types.LongType.get()), - required(3, "date", Types.StringType.get())); - - static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) - .identity("date") - .bucket("id", 1) - .build(); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - private HadoopTables tables; - private Configuration conf; - - @Parameterized.Parameters - public static Object[][] parameters() { - return new Object[][]{ - new Object[]{"parquet"}, - new Object[]{"avro"}, - new Object[]{"orc"} - }; - } - - private final FileFormat format; +public class TestIcebergInputFormat extends BaseInputFormatTest { public TestIcebergInputFormat(String format) { - this.format = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); - } - - @Before - public void before() { - conf = new Configuration(); - tables = new HadoopTables(conf); - } - - @Test - public void testUnpartitionedTable() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, null, format, expectedRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); - validate(job, expectedRecords); + this.fileFormat = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); } - @Test - public void testPartitionedTable() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - + @Override + protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { Job job = Job.getInstance(conf); IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); + configBuilder.readFrom(tableLocation.toString()); validate(job, expectedRecords); } + //TODO: try move as many methods below into base class (once functionality is implemented in + // mapred InputFormat) @Test public void testFilterExp() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); expectedRecords.get(0).set(2, "2020-03-20"); expectedRecords.get(1).set(2, "2020-03-20"); - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); - DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), fileFormat, RandomGenericData.generate(table.schema(), 2, 0L)); table.newAppend() .appendFile(dataFile1) @@ -169,10 +107,10 @@ public void testFilterExp() throws Exception { @Test public void testResiduals() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List writeRecords = RandomGenericData.generate(table.schema(), 2, 0L); writeRecords.get(0).set(1, 123L); @@ -183,8 +121,8 @@ public void testResiduals() throws Exception { List expectedRecords = new ArrayList<>(); expectedRecords.add(writeRecords.get(0)); - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, writeRecords); - DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, writeRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), fileFormat, RandomGenericData.generate(table.schema(), 2, 0L)); table.newAppend() .appendFile(dataFile1) @@ -210,16 +148,16 @@ public void testResiduals() throws Exception { @Test public void testFailedResidualFiltering() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); expectedRecords.get(0).set(2, "2020-03-20"); expectedRecords.get(1).set(2, "2020-03-20"); - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); table.newAppend() .appendFile(dataFile1) .commit(); @@ -249,14 +187,14 @@ public void testFailedResidualFiltering() throws Exception { @Test public void testProjection() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Schema projectedSchema = TypeUtil.select(SCHEMA, ImmutableSet.of(1)); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List inputRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, inputRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, inputRecords); table.newAppend() .appendFile(dataFile) .commit(); @@ -283,10 +221,10 @@ public void testProjection() throws Exception { @Test public void testIdentityPartitionProjections() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Table table = tables.create(LOG_SCHEMA, IDENTITY_PARTITION_SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List inputRecords = RandomGenericData.generate(LOG_SCHEMA, 10, 0); @@ -296,7 +234,7 @@ public void testIdentityPartitionProjections() throws Exception { record.set(1, "2020-03-2" + idx); record.set(2, idx.toString()); append.appendFile(writeFile(temp.newFile(), table, Row.of("2020-03-2" + idx, idx.toString()), - format, ImmutableList.of(record))); + fileFormat, ImmutableList.of(record))); idx += 1; } append.commit(); @@ -357,18 +295,18 @@ private void validateIdentityPartitionProjections( @Test public void testSnapshotReads() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) + .appendFile(writeFile(temp.newFile(), table, null, fileFormat, expectedRecords)) .commit(); long snapshotId = table.currentSnapshot().snapshotId(); table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, format, + .appendFile(writeFile(temp.newFile(), table, null, fileFormat, RandomGenericData.generate(table.schema(), 1, 0L))) .commit(); @@ -383,14 +321,14 @@ public void testSnapshotReads() throws Exception { @Test public void testLocality() throws Exception { - File location = temp.newFolder(format.name()); + File location = temp.newFolder(fileFormat.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) + .appendFile(writeFile(temp.newFile(), table, null, fileFormat, expectedRecords)) .commit(); Job job = Job.getInstance(conf); IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); @@ -421,10 +359,10 @@ public void testCustomCatalog() throws Exception { Catalog catalog = new HadoopCatalogFunc().apply(conf); TableIdentifier tableIdentifier = TableIdentifier.of("db", "t"); Table table = catalog.createTable(tableIdentifier, SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name())); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name())); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); From 3695c084a02a29836637be55238b1270986e0756 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 4 May 2020 18:00:29 +0100 Subject: [PATCH 32/51] removed UncheckedIOException try/catch --- .../mr/mapred/TestIcebergInputFormat.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 58cd1afeb5b2..e64ff021ea38 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -21,7 +21,6 @@ import java.io.File; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -84,17 +83,13 @@ private void validate(JobConf jobConf, List expectedRecords) throws IOEx private List readRecords(JobConf jobConf) throws IOException { InputSplit[] splits = inputFormat.getSplits(jobConf, 1); - try { - RecordReader reader = inputFormat.getRecordReader(splits[0], jobConf, null); - List records = new ArrayList<>(); - IcebergWritable value = (IcebergWritable) reader.createValue(); - while (reader.next(null, value)) { - records.add(value.getRecord().copy()); - } - return records; - } catch (IOException e) { - throw new UncheckedIOException(e); + RecordReader reader = inputFormat.getRecordReader(splits[0], jobConf, null); + List records = new ArrayList<>(); + IcebergWritable value = (IcebergWritable) reader.createValue(); + while (reader.next(null, value)) { + records.add(value.getRecord().copy()); } + return records; } } From ad27b357f64f0899513228927516afa7e52c7248 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 4 May 2020 21:54:47 +0100 Subject: [PATCH 33/51] first cut at refactoring duplicate code between InputFormats --- .../iceberg/mr/IcebergRecordReader.java | 115 ++++++++ .../apache/iceberg/mr/InputFormatConfig.java | 155 ++++++++++ .../iceberg/mr/mapred/IcebergInputFormat.java | 12 +- .../mr/mapred/IcebergReaderFactory.java | 89 ------ .../mr/mapreduce/IcebergInputFormat.java | 268 ++---------------- .../mr/mapreduce/TestIcebergInputFormat.java | 57 +--- 6 files changed, 299 insertions(+), 397 deletions(-) create mode 100644 mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java new file mode 100644 index 000000000000..8ddb543f4495 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.data.avro.DataReader; +import org.apache.iceberg.data.orc.GenericOrcReader; +import org.apache.iceberg.data.parquet.GenericParquetReaders; +import org.apache.iceberg.expressions.Evaluator; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; + +public class IcebergRecordReader { + + private boolean applyResidual; + private boolean caseSensitive; + private boolean reuseContainers; + + private void initialize(Configuration conf) { + this.applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); + this.caseSensitive = conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true); + this.reuseContainers = conf.getBoolean(InputFormatConfig.REUSE_CONTAINERS, false); + } + + public CloseableIterable createReader(Configuration config, FileScanTask currentTask, Schema readSchema) { + initialize(config); + DataFile file = currentTask.file(); + // TODO we should make use of FileIO to create inputFile + InputFile inputFile = HadoopInputFile.fromLocation(file.path(), config); + switch (file.format()) { + case AVRO: + return newAvroIterable(inputFile, currentTask, readSchema); + case ORC: + return newOrcIterable(inputFile, currentTask, readSchema); + case PARQUET: + return newParquetIterable(inputFile, currentTask, readSchema); + default: + throw new UnsupportedOperationException( + String.format("Cannot read %s file: %s", file.format().name(), file.path())); + } + } + + private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile).project(readSchema).split(task.start(), task.length()); + if (reuseContainers) { + avroReadBuilder.reuseContainers(); + } + avroReadBuilder.createReaderFunc(DataReader::create); + return applyResidualFiltering(avroReadBuilder.build(), task.residual(), readSchema); + } + + private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + Parquet.ReadBuilder parquetReadBuilder = Parquet + .read(inputFile) + .project(readSchema) + .filter(task.residual()) + .caseSensitive(caseSensitive) + .split(task.start(), task.length()); + if (reuseContainers) { + parquetReadBuilder.reuseContainers(); + } + + parquetReadBuilder.createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); + + return applyResidualFiltering(parquetReadBuilder.build(), task.residual(), readSchema); + } + + private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + ORC.ReadBuilder orcReadBuilder = ORC + .read(inputFile) + .project(readSchema) + .caseSensitive(caseSensitive) + .split(task.start(), task.length()); + // ORC does not support reuse containers yet + orcReadBuilder.createReaderFunc(fileSchema -> GenericOrcReader.buildReader(readSchema, fileSchema)); + return applyResidualFiltering(orcReadBuilder.build(), task.residual(), readSchema); + } + + private CloseableIterable applyResidualFiltering(CloseableIterable iter, Expression residual, Schema readSchema) { + if (applyResidual && residual != null && residual != Expressions.alwaysTrue()) { + Evaluator filter = new Evaluator(readSchema.asStruct(), residual, caseSensitive); + return CloseableIterable.filter(iter, record -> filter.eval((StructLike) record)); + } else { + return iter; + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java new file mode 100644 index 000000000000..bbfecd0e6b25 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr; + +import com.google.common.base.Preconditions; +import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.common.DynConstructors; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.hadoop.HadoopTables; + +public class InputFormatConfig { + + private InputFormatConfig() {} + + public static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; + public static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; + public static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; + public static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; + public static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; + public static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; + public static final String READ_SCHEMA = "iceberg.mr.read.schema"; + public static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; + public static final String SPLIT_SIZE = "iceberg.mr.split.size"; + public static final String TABLE_PATH = "iceberg.mr.table.path"; + public static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; + public static final String LOCALITY = "iceberg.mr.locality"; + public static final String CATALOG = "iceberg.mr.catalog"; + + public static class ConfigBuilder { + private final Configuration conf; + + public ConfigBuilder(Configuration conf) { + this.conf = conf; + // defaults + conf.setBoolean(SKIP_RESIDUAL_FILTERING, false); + conf.setBoolean(CASE_SENSITIVE, true); + conf.setBoolean(REUSE_CONTAINERS, false); + conf.setBoolean(LOCALITY, false); + } + + public ConfigBuilder readFrom(String path) { + conf.set(TABLE_PATH, path); + Table table = findTable(conf); + conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + return this; + } + + public ConfigBuilder filter(Expression expression) { + conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); + return this; + } + + public ConfigBuilder project(Schema schema) { + conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder reuseContainers(boolean reuse) { + conf.setBoolean(InputFormatConfig.REUSE_CONTAINERS, reuse); + return this; + } + + public ConfigBuilder caseSensitive(boolean caseSensitive) { + conf.setBoolean(InputFormatConfig.CASE_SENSITIVE, caseSensitive); + return this; + } + + public ConfigBuilder snapshotId(long snapshotId) { + conf.setLong(SNAPSHOT_ID, snapshotId); + return this; + } + + public ConfigBuilder asOfTime(long asOfTime) { + conf.setLong(AS_OF_TIMESTAMP, asOfTime); + return this; + } + + public ConfigBuilder splitSize(long splitSize) { + conf.setLong(SPLIT_SIZE, splitSize); + return this; + } + + /** + * If this API is called. The input splits + * constructed will have host location information + */ + public ConfigBuilder preferLocality() { + conf.setBoolean(LOCALITY, true); + return this; + } + + public ConfigBuilder catalogFunc(Class> catalogFuncClass) { + conf.setClass(CATALOG, catalogFuncClass, Function.class); + return this; + } + + /** + * Compute platforms pass down filters to data sources. If the data source cannot apply some filters, or only + * partially applies the filter, it will return the residual filter back. If the platform can correctly apply + * the residual filters, then it should call this api. Otherwise the current api will throw an exception if the + * passed in filter is not completely satisfied. + */ + public ConfigBuilder skipResidualFiltering() { + conf.setBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, true); + return this; + } + } + + public static Table findTable(Configuration conf) { + String path = conf.get(InputFormatConfig.TABLE_PATH); + Preconditions.checkArgument(path != null, "Table path should not be null"); + if (path.contains("/")) { + HadoopTables tables = new HadoopTables(conf); + return tables.load(path); + } + + String catalogFuncClass = conf.get(InputFormatConfig.CATALOG); + if (catalogFuncClass != null) { + Function catalogFunc = (Function) + DynConstructors.builder(Function.class) + .impl(catalogFuncClass) + .build() + .newInstance(); + Catalog catalog = catalogFunc.apply(conf); + TableIdentifier tableIdentifier = TableIdentifier.parse(path); + return catalog.loadTable(tableIdentifier); + } else { + throw new IllegalArgumentException("No custom catalog specified to load table " + path); + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 1b4428128b4c..32900b3888c4 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -38,15 +38,12 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.iceberg.CombinedScanTask; -import org.apache.iceberg.DataFile; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopInputFile; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.InputFile; import org.apache.iceberg.mr.SerializationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +57,6 @@ public class IcebergInputFormat implements InputFormat, CombineHiveI private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); static final String TABLE_LOCATION = "location"; - static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; private Table table; @@ -116,12 +112,10 @@ public class IcebergRecordReader implements RecordReader private CloseableIterable reader; private Iterator recordIterator; private Record currentRecord; - private boolean reuseContainers; public IcebergRecordReader(InputSplit split, JobConf conf) throws IOException { this.split = (IcebergSplit) split; this.conf = conf; - this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); initialise(); } @@ -132,11 +126,9 @@ private void initialise() { private void nextTask() { FileScanTask currentTask = tasks.next(); - DataFile file = currentTask.file(); - InputFile inputFile = HadoopInputFile.fromLocation(file.path(), conf); Schema tableSchema = table.schema(); - IcebergReaderFactory readerFactory = new IcebergReaderFactory(); - reader = readerFactory.createReader(file, currentTask, inputFile, tableSchema, reuseContainers); + org.apache.iceberg.mr.IcebergRecordReader wrappedReader = new org.apache.iceberg.mr.IcebergRecordReader(); + reader = wrappedReader.createReader(conf, currentTask, tableSchema); recordIterator = reader.iterator(); } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java deleted file mode 100644 index 1f3f8e6489aa..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergReaderFactory.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import org.apache.iceberg.DataFile; -import org.apache.iceberg.FileScanTask; -import org.apache.iceberg.Schema; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.data.avro.DataReader; -import org.apache.iceberg.data.orc.GenericOrcReader; -import org.apache.iceberg.data.parquet.GenericParquetReaders; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; - -class IcebergReaderFactory { - - public CloseableIterable createReader(DataFile file, FileScanTask currentTask, InputFile inputFile, - Schema tableSchema, boolean reuseContainers) { - switch (file.format()) { - case AVRO: - return buildAvroReader(currentTask, inputFile, tableSchema, reuseContainers); - case ORC: - return buildOrcReader(currentTask, inputFile, tableSchema, reuseContainers); - case PARQUET: - return buildParquetReader(currentTask, inputFile, tableSchema, reuseContainers); - - default: - throw new UnsupportedOperationException(String.format("Cannot read %s file: %s", file.format().name(), - file.path())); - } - } - - private CloseableIterable buildAvroReader(FileScanTask task, InputFile inputFile, Schema schema, - boolean reuseContainers) { - Avro.ReadBuilder builder = Avro.read(inputFile) - .createReaderFunc(DataReader::create) - .project(schema) - .split(task.start(), task.length()); - - if (reuseContainers) { - builder.reuseContainers(); - } - - return builder.build(); - } - - private CloseableIterable buildOrcReader(FileScanTask task, InputFile inputFile, Schema schema, - boolean reuseContainers) { - ORC.ReadBuilder builder = ORC.read(inputFile) - .createReaderFunc(fileSchema -> GenericOrcReader.buildReader(schema, fileSchema)) - .project(schema) - .split(task.start(), task.length()); - - return builder.build(); - } - - private CloseableIterable buildParquetReader(FileScanTask task, InputFile inputFile, Schema schema, - boolean reuseContainers) { - Parquet.ReadBuilder builder = Parquet.read(inputFile) - .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema)) - .project(schema) - .split(task.start(), task.length()); - - if (reuseContainers) { - builder.reuseContainers(); - } - - return builder.build(); - } -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 0d35644ab89f..9371ec70e65b 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -19,7 +19,6 @@ package org.apache.iceberg.mr.mapreduce; -import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -32,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.InputFormat; @@ -52,27 +50,15 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.common.DynConstructors; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.avro.DataReader; -import org.apache.iceberg.data.orc.GenericOrcReader; -import org.apache.iceberg.data.parquet.GenericParquetReaders; import org.apache.iceberg.exceptions.RuntimeIOException; -import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.hadoop.HadoopInputFile; -import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.hadoop.Util; import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.SerializationUtil; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.slf4j.Logger; @@ -85,20 +71,6 @@ public class IcebergInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); - static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time"; - static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; - static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression"; - static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model"; - static final String READ_SCHEMA = "iceberg.mr.read.schema"; - static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; - static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id"; - static final String SPLIT_SIZE = "iceberg.mr.split.size"; - static final String TABLE_PATH = "iceberg.mr.table.path"; - static final String TABLE_SCHEMA = "iceberg.mr.table.schema"; - static final String LOCALITY = "iceberg.mr.locality"; - static final String CATALOG = "iceberg.mr.catalog"; - static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; - private transient List splits; private enum InMemoryDataModel { @@ -113,100 +85,9 @@ private enum InMemoryDataModel { * * @param job the {@code Job} to configure */ - public static ConfigBuilder configure(Job job) { + public static InputFormatConfig.ConfigBuilder configure(Job job) { job.setInputFormatClass(IcebergInputFormat.class); - return new ConfigBuilder(job.getConfiguration()); - } - - public static class ConfigBuilder { - private final Configuration conf; - - public ConfigBuilder(Configuration conf) { - this.conf = conf; - // defaults - conf.setEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); - conf.setBoolean(SKIP_RESIDUAL_FILTERING, false); - conf.setBoolean(CASE_SENSITIVE, true); - conf.setBoolean(REUSE_CONTAINERS, false); - conf.setBoolean(LOCALITY, false); - } - - public ConfigBuilder readFrom(String path) { - conf.set(TABLE_PATH, path); - Table table = findTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - return this; - } - - public ConfigBuilder filter(Expression expression) { - conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); - return this; - } - - public ConfigBuilder project(Schema schema) { - conf.set(READ_SCHEMA, SchemaParser.toJson(schema)); - return this; - } - - public ConfigBuilder reuseContainers(boolean reuse) { - conf.setBoolean(REUSE_CONTAINERS, reuse); - return this; - } - - public ConfigBuilder caseSensitive(boolean caseSensitive) { - conf.setBoolean(CASE_SENSITIVE, caseSensitive); - return this; - } - - public ConfigBuilder snapshotId(long snapshotId) { - conf.setLong(SNAPSHOT_ID, snapshotId); - return this; - } - - public ConfigBuilder asOfTime(long asOfTime) { - conf.setLong(AS_OF_TIMESTAMP, asOfTime); - return this; - } - - public ConfigBuilder splitSize(long splitSize) { - conf.setLong(SPLIT_SIZE, splitSize); - return this; - } - - /** - * If this API is called. The input splits - * constructed will have host location information - */ - public ConfigBuilder preferLocality() { - conf.setBoolean(LOCALITY, true); - return this; - } - - public ConfigBuilder catalogFunc(Class> catalogFuncClass) { - conf.setClass(CATALOG, catalogFuncClass, Function.class); - return this; - } - - public ConfigBuilder useHiveRows() { - conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.HIVE.name()); - return this; - } - - public ConfigBuilder usePigTuples() { - conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.PIG.name()); - return this; - } - - /** - * Compute platforms pass down filters to data sources. If the data source cannot apply some filters, or only - * partially applies the filter, it will return the residual filter back. If the platform can correctly apply - * the residual filters, then it should call this api. Otherwise the current api will throw an exception if the - * passed in filter is not completely satisfied. - */ - public ConfigBuilder skipResidualFiltering() { - conf.setBoolean(SKIP_RESIDUAL_FILTERING, true); - return this; - } + return new InputFormatConfig.ConfigBuilder(job.getConfiguration()); } @Override @@ -217,35 +98,35 @@ public List getSplits(JobContext context) { } Configuration conf = context.getConfiguration(); - Table table = findTable(conf); + Table table = InputFormatConfig.findTable(conf); TableScan scan = table.newScan() - .caseSensitive(conf.getBoolean(CASE_SENSITIVE, true)); - long snapshotId = conf.getLong(SNAPSHOT_ID, -1); + .caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); + long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); if (snapshotId != -1) { scan = scan.useSnapshot(snapshotId); } - long asOfTime = conf.getLong(AS_OF_TIMESTAMP, -1); + long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); if (asOfTime != -1) { scan = scan.asOfTime(asOfTime); } - long splitSize = conf.getLong(SPLIT_SIZE, 0); + long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); if (splitSize > 0) { scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); } - String schemaStr = conf.get(READ_SCHEMA); + String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); if (schemaStr != null) { scan.project(SchemaParser.fromJson(schemaStr)); } // TODO add a filter parser to get rid of Serialization - Expression filter = SerializationUtil.deserializeFromBase64(conf.get(FILTER_EXPRESSION)); + Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); if (filter != null) { scan = scan.filter(filter); } splits = Lists.newArrayList(); - boolean applyResidual = !conf.getBoolean(SKIP_RESIDUAL_FILTERING, false); - InMemoryDataModel model = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); + boolean applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); + InMemoryDataModel model = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); try (CloseableIterable tasksIterable = scan.planTasks()) { tasksIterable.forEach(task -> { if (applyResidual && (model == InMemoryDataModel.HIVE || model == InMemoryDataModel.PIG)) { @@ -282,8 +163,6 @@ private static final class IcebergRecordReader extends RecordReader private TaskAttemptContext context; private Schema tableSchema; private Schema expectedSchema; - private boolean reuseContainers; - private boolean caseSensitive; private InMemoryDataModel inMemoryDataModel; private Map namesToPos; private Iterator tasks; @@ -298,13 +177,11 @@ public void initialize(InputSplit split, TaskAttemptContext newContext) { CombinedScanTask task = ((IcebergSplit) split).task; this.context = newContext; this.tasks = task.files().iterator(); - this.tableSchema = SchemaParser.fromJson(conf.get(TABLE_SCHEMA)); - String readSchemaStr = conf.get(READ_SCHEMA); + this.tableSchema = SchemaParser.fromJson(conf.get(InputFormatConfig.TABLE_SCHEMA)); + String readSchemaStr = conf.get(InputFormatConfig.READ_SCHEMA); this.expectedSchema = readSchemaStr != null ? SchemaParser.fromJson(readSchemaStr) : tableSchema; this.namesToPos = buildNameToPos(expectedSchema); - this.reuseContainers = conf.getBoolean(REUSE_CONTAINERS, false); - this.caseSensitive = conf.getBoolean(CASE_SENSITIVE, true); - this.inMemoryDataModel = conf.getEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); + this.inMemoryDataModel = conf.getEnum(InputFormatConfig.IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC); this.currentIterator = open(tasks.next()); } @@ -378,24 +255,8 @@ private Iterator open(FileScanTask currentTask) { } private Iterator open(FileScanTask currentTask, Schema readSchema) { - DataFile file = currentTask.file(); - // TODO we should make use of FileIO to create inputFile - InputFile inputFile = HadoopInputFile.fromLocation(file.path(), context.getConfiguration()); - CloseableIterable iterable; - switch (file.format()) { - case AVRO: - iterable = newAvroIterable(inputFile, currentTask, readSchema); - break; - case ORC: - iterable = newOrcIterable(inputFile, currentTask, readSchema); - break; - case PARQUET: - iterable = newParquetIterable(inputFile, currentTask, readSchema); - break; - default: - throw new UnsupportedOperationException( - String.format("Cannot read %s file: %s", file.format().name(), file.path())); - } + org.apache.iceberg.mr.IcebergRecordReader wrappedReader = new org.apache.iceberg.mr.IcebergRecordReader(); + CloseableIterable iterable = wrappedReader.createReader(context.getConfiguration(), currentTask, readSchema); currentCloseable = iterable; return iterable.iterator(); } @@ -443,99 +304,6 @@ private Record withIdentityPartitionColumns( return row; } - private CloseableIterable applyResidualFiltering(CloseableIterable iter, Expression residual, - Schema readSchema) { - boolean applyResidual = !context.getConfiguration().getBoolean(SKIP_RESIDUAL_FILTERING, false); - - if (applyResidual && residual != null && residual != Expressions.alwaysTrue()) { - Evaluator filter = new Evaluator(readSchema.asStruct(), residual, caseSensitive); - return CloseableIterable.filter(iter, record -> filter.eval((StructLike) record)); - } else { - return iter; - } - } - - private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { - Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile) - .project(readSchema) - .split(task.start(), task.length()); - if (reuseContainers) { - avroReadBuilder.reuseContainers(); - } - - switch (inMemoryDataModel) { - case PIG: - case HIVE: - //TODO implement value readers for Pig and Hive - throw new UnsupportedOperationException("Avro support not yet supported for Pig and Hive"); - case GENERIC: - avroReadBuilder.createReaderFunc(DataReader::create); - } - return applyResidualFiltering(avroReadBuilder.build(), task.residual(), readSchema); - } - - private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { - Parquet.ReadBuilder parquetReadBuilder = Parquet.read(inputFile) - .project(readSchema) - .filter(task.residual()) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); - if (reuseContainers) { - parquetReadBuilder.reuseContainers(); - } - - switch (inMemoryDataModel) { - case PIG: - case HIVE: - //TODO implement value readers for Pig and Hive - throw new UnsupportedOperationException("Parquet support not yet supported for Pig and Hive"); - case GENERIC: - parquetReadBuilder.createReaderFunc( - fileSchema -> GenericParquetReaders.buildReader(readSchema, fileSchema)); - } - return applyResidualFiltering(parquetReadBuilder.build(), task.residual(), readSchema); - } - - private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { - ORC.ReadBuilder orcReadBuilder = ORC.read(inputFile) - .project(readSchema) - .caseSensitive(caseSensitive) - .split(task.start(), task.length()); - // ORC does not support reuse containers yet - switch (inMemoryDataModel) { - case PIG: - case HIVE: - //TODO: implement value readers for Pig and Hive - throw new UnsupportedOperationException("ORC support not yet supported for Pig and Hive"); - case GENERIC: - orcReadBuilder.createReaderFunc(fileSchema -> GenericOrcReader.buildReader(readSchema, fileSchema)); - } - - return applyResidualFiltering(orcReadBuilder.build(), task.residual(), readSchema); - } - } - - private static Table findTable(Configuration conf) { - String path = conf.get(TABLE_PATH); - Preconditions.checkArgument(path != null, "Table path should not be null"); - if (path.contains("/")) { - HadoopTables tables = new HadoopTables(conf); - return tables.load(path); - } - - String catalogFuncClass = conf.get(CATALOG); - if (catalogFuncClass != null) { - Function catalogFunc = (Function) - DynConstructors.builder(Function.class) - .impl(catalogFuncClass) - .build() - .newInstance(); - Catalog catalog = catalogFunc.apply(conf); - TableIdentifier tableIdentifier = TableIdentifier.parse(path); - return catalog.loadTable(tableIdentifier); - } else { - throw new IllegalArgumentException("No custom catalog specified to load table " + path); - } } static class IcebergSplit extends InputSplit implements Writable { @@ -556,7 +324,7 @@ public long getLength() { @Override public String[] getLocations() { - boolean localityPreferred = conf.getBoolean(LOCALITY, false); + boolean localityPreferred = conf.getBoolean(InputFormatConfig.LOCALITY, false); if (!localityPreferred) { return ANYWHERE; } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index 957f165ea7f4..7b8258db8ee4 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -40,7 +40,6 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.iceberg.AppendFiles; -import org.apache.iceberg.AssertHelpers; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; @@ -54,6 +53,7 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.mr.BaseInputFormatTest; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.TestHelpers.Row; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -74,7 +74,7 @@ public TestIcebergInputFormat(String format) { @Override protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(tableLocation.toString()); validate(job, expectedRecords); } @@ -99,7 +99,7 @@ public void testFilterExp() throws Exception { .appendFile(dataFile2) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()) .filter(Expressions.equal("date", "2020-03-20")); validate(job, expectedRecords); @@ -129,7 +129,7 @@ public void testResiduals() throws Exception { .appendFile(dataFile2) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()) .filter(Expressions.and( Expressions.equal("date", "2020-03-20"), @@ -146,45 +146,6 @@ public void testResiduals() throws Exception { validate(job, writeRecords); } - @Test - public void testFailedResidualFiltering() throws Exception { - File location = temp.newFolder(fileFormat.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - expectedRecords.get(1).set(2, "2020-03-20"); - - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); - table.newAppend() - .appendFile(dataFile1) - .commit(); - - Job jobShouldFail1 = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); - configBuilder.useHiveRows().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 0))); - AssertHelpers.assertThrows( - "Residuals are not evaluated today for Iceberg Generics In memory model of HIVE", - UnsupportedOperationException.class, "Filter expression ref(name=\"id\") == 0 is not completely satisfied.", - () -> validate(jobShouldFail1, expectedRecords)); - - Job jobShouldFail2 = Job.getInstance(conf); - configBuilder = IcebergInputFormat.configure(jobShouldFail2); - configBuilder.usePigTuples().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 0))); - AssertHelpers.assertThrows( - "Residuals are not evaluated today for Iceberg Generics In memory model of PIG", - UnsupportedOperationException.class, "Filter expression ref(name=\"id\") == 0 is not completely satisfied.", - () -> validate(jobShouldFail2, expectedRecords)); - } - @Test public void testProjection() throws Exception { File location = temp.newFolder(fileFormat.name()); @@ -200,7 +161,7 @@ public void testProjection() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder .readFrom(location.toString()) .project(projectedSchema); @@ -275,7 +236,7 @@ private static Schema withColumns(String... names) { private void validateIdentityPartitionProjections( String tablePath, Schema projectedSchema, List inputRecords) throws Exception { Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder .readFrom(tablePath) .project(projectedSchema); @@ -311,7 +272,7 @@ public void testSnapshotReads() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder .readFrom(location.toString()) .snapshotId(snapshotId); @@ -331,7 +292,7 @@ public void testLocality() throws Exception { .appendFile(writeFile(temp.newFile(), table, null, fileFormat, expectedRecords)) .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder.readFrom(location.toString()); for (InputSplit split : splits(job.getConfiguration())) { @@ -368,7 +329,7 @@ public void testCustomCatalog() throws Exception { .commit(); Job job = Job.getInstance(conf); - IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); configBuilder .catalogFunc(HadoopCatalogFunc.class) .readFrom(tableIdentifier.toString()); From 92502530009f7fef4e0b7f53f07eb6ddb3c1a162 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Tue, 5 May 2020 14:21:36 +0100 Subject: [PATCH 34/51] remove test data (tests now create their own) --- ...d-46fb-804e-e9806abf81c7-00000.parquet.crc | Bin 16 -> 0 bytes ...-ae0d-46fb-804e-e9806abf81c7-00000.parquet | Bin 686 -> 0 bytes ...32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc | Bin 44 -> 0 bytes ...ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc | Bin 28 -> 0 bytes .../metadata/.v1.metadata.json.crc | Bin 16 -> 0 bytes .../metadata/.v2.metadata.json.crc | Bin 20 -> 0 bytes .../metadata/.version-hint.text.crc | Bin 12 -> 0 bytes ...3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro | Bin 4544 -> 0 bytes ...-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro | Bin 2100 -> 0 bytes .../metadata/v1.metadata.json | 31 ------------ .../metadata/v2.metadata.json | 47 ------------------ .../metadata/version-hint.text | 1 - 12 files changed, 79 deletions(-) delete mode 100644 mr/src/test/resources/test-table-to-delete/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc delete mode 100644 mr/src/test/resources/test-table-to-delete/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/.1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro.crc delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/.v1.metadata.json.crc delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/.v2.metadata.json.crc delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/.version-hint.text.crc delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/v1.metadata.json delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/v2.metadata.json delete mode 100644 mr/src/test/resources/test-table-to-delete/metadata/version-hint.text diff --git a/mr/src/test/resources/test-table-to-delete/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc b/mr/src/test/resources/test-table-to-delete/data/.00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet.crc deleted file mode 100644 index caa4e3e7edee6e4a126e7a28b57bfdc329056058..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}8A*GUY<|OV3&WD>(*K diff --git a/mr/src/test/resources/test-table-to-delete/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet b/mr/src/test/resources/test-table-to-delete/data/00000-1-c7557bc3-ae0d-46fb-804e-e9806abf81c7-00000.parquet deleted file mode 100644 index 772144ca6e7be33d4f390f9f7f5de20c5fb967be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 686 zcmb7C&ui2`6n>habTC#PM4(XrkiC!3m$Z_SOia^Y<^6(f$X+TvW2De zR1m}-#DmbACqd}Jg9i^HcoBN=;?;k|qc6K6h3dg!nD=Jh`@ZkJ$84-UbSTg}G!j%r zN2QwSLVa#M2{(P28f0x0O#tBP3k}b5g+agHj|X$FJ|CU1=kGjm`)&BNS6zC!+grYI zkHCX-gCmZ4lu(Pd@1r57;Xl5719jo&wVUvI{nPu)Wg^fkv{TBHsl1`Rxl{@P7~02$ znWsbjFRvIoQ&$u#aC~s#);tDjg>~{tSOi_HRtz%ohq4N-3f6~LHTP3Ln>-?* ztITAgrkQk+wKP!KER_;n)ejZ@SgC2lygH~Brgqs{=K5Bz)a&}63RekgGL^1%As!@Dto+`Y)jtZd6A97f(sreGL3TH zdA!Gyh<6;p%eFCNY6Q>Z&N#^=hGIX>r8q^9j0SvP%y27zaI-NX^S_YkR*8GE4@Pg> z&vwSLSRQ89uFUdeYh*bM$8I=QJs2lxWch6`irca0HCF;x#;)6JMz(9ao^08n7p!`# Vb|8buxz2}JPt CUlWl4 diff --git a/mr/src/test/resources/test-table-to-delete/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc b/mr/src/test/resources/test-table-to-delete/metadata/.snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro.crc deleted file mode 100644 index 68cbb3719c4c209a5f89af23ed3fc55f220881d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28 kcmYc;N@ieSU}6wRd@k2gs=ec9!HHX5i=4mi4L>mj0EyTPEC2ui diff --git a/mr/src/test/resources/test-table-to-delete/metadata/.v1.metadata.json.crc b/mr/src/test/resources/test-table-to-delete/metadata/.v1.metadata.json.crc deleted file mode 100644 index 87238dff5fd450a287db59ad77696ebd8c7ddc99..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}8A)IQin^wij0cEh`5^ diff --git a/mr/src/test/resources/test-table-to-delete/metadata/.v2.metadata.json.crc b/mr/src/test/resources/test-table-to-delete/metadata/.v2.metadata.json.crc deleted file mode 100644 index 500d5ca3c03cf017664a4e9772ccda954abac742..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20 bcmYc;N@ieSU}CTjo~$-ulh)Lt1hazxHc19h diff --git a/mr/src/test/resources/test-table-to-delete/metadata/.version-hint.text.crc b/mr/src/test/resources/test-table-to-delete/metadata/.version-hint.text.crc deleted file mode 100644 index 20031206a3b58c7bd0e0b0cf48215fa64e60ea8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}BKEx{ntC5%2=_ diff --git a/mr/src/test/resources/test-table-to-delete/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro b/mr/src/test/resources/test-table-to-delete/metadata/1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c-m0.avro deleted file mode 100644 index cfd5b85f8fe17c93a8a19ff50be99fed1447ce52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4544 zcmb_fTWB0r7*5ilwu!Y$EtG~Tr$I{Hv~$UI35bG-Mw%9{kSvq4yJwSQW@k1tvk7Tg zMO5(8KE#Jm6jG=J1zS-cqV-m-Vk$yPC|afQ#air(XrbUk6+LGz=bV|{*=&|PExR+{ zf4=|!zyCkq4yK=JY;CnOV<-=gH5~@dbOC`RupO(IaX_$) z)5;RO&m#{FoI|#wA;WRw3XJ_%YzI0;n?U^Ju)#9xph-!TW$>;63-*}lXygF{xRm@_ zxCmK5WT6AK9M%zgKbEVdkax=YQLt8k&KQe8>rcxUI#UvPx|p14hvp>Mk)caCy3x&`rPgQyH7+{EW+Tk2WdGy@X-8 zG{+HT9y2!tDbSjPdXdinC~u+MOQt$O9*p7))F1a2O&%7=uve{kZ}bpPXzP=EAIV*j z>dCz?N{bh&Ja3+&6mZUr<+sA z($ZwhkQ*sbU*Hyb}%W&xig8mB4&BG z%OZ}B+cSF4(ft+nbau%6#6aEU7GSxr6KzmsqRQa{j}%amC2mf`?x}mioU6~DBUbd#6fX*_#Z$> zVp0!-M@&-;oYhyyHg&k6sgp^k{2$8v6pu#*FSMeUF^Cu_jtC{d7*LCn0AvOEUX7$e zZ^ppLxf{@bM9%OQVlvnk-cx==gj63R<-W)-c8;ix6bTeVN+CnFCQ>q=!b+B7jm#7> zm2u2S8`s24;Zlxxfwht%YK|jC+8dBclRpXWe@gd<4t_|uiwc)k(mOG{sFYyZZ6A6#oloa}FGxkKG>@4kkIe=twqc=nUacm4h6eP_(u z&L98WoV(OCIC5pC|JbQbJ6_v4@Xoo^TN`%{zcfG9wf)7F<)b$b4~!q!bqckN3_Lb| zXWJj!7nRX{W|uGCf1&g3BNJa~hkl*;XrTSXq_jNo+`-&b`g(HY b(|<1fwerT%vkx4Z+x+v~^znZiqv`(-(X8@) diff --git a/mr/src/test/resources/test-table-to-delete/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro b/mr/src/test/resources/test-table-to-delete/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro deleted file mode 100644 index 81e4ba7929460647c0ffe287d5325c71d69306e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2100 zcmb`HPfXKL9LMLP#1IAvQ9yzTG{V75T?bpYkr;R2fC2y9h=}#IecPVt+S$H=3oH@B zg%~47Q4`^2yg<|o=utT+k;H=sjRrj$&P4Q}F}{xWwd>dk4lKJWK_zwf7Kg)4ii z8^Exp9@oVbOoQdxQ3mHSkclxG6je=P{EPz9#_u#xPy*_hPaqkR#|Z>wNoQh%qa|-- z08WuNOpuN>q$I71V@P6Rp+F$-FRhcIl5|$eYDpdIsLKEiBaBpq*E3M$37DRV6zn=M z=qLn|1SMWqKt@lg*jYHpC|Ox1rG=x1{EP$>AS+`gW;{19ZC;_XgeUGGivFstkc*m6%J4I=Yk95vcJ3*^HeOt35`@VW?N$ z4eJbTHk0{37@Hr;`x&x`G*qzJZ6seg92}`RP|#IWB`DUCbjkq31}n-0)4@f8Q7#${ zM#3ByWP>ah2)7^+3Q$X$lCSva(21K_k*cyaW>e5o@lCBYcr}YONHiwJkSb?tZp?d& zz?KrI;AI#E>uf@h5@@g-s+VCHirI)xir7wi8<7CAZr{b#N0$q-jW*>-#odG`sFK<{ zV}&WJg3UWaaTG=8YzSPYn1)9tUnlaKE5h`F!|n1hi_)XrTHB`=#mv&2u9yu_M6uvD zpm@0gQh=7TGs4@@^&9U5> Mo9vz2b?$cm0y@>v9smFU diff --git a/mr/src/test/resources/test-table-to-delete/metadata/v1.metadata.json b/mr/src/test/resources/test-table-to-delete/metadata/v1.metadata.json deleted file mode 100644 index 0c08d1732619..000000000000 --- a/mr/src/test/resources/test-table-to-delete/metadata/v1.metadata.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "format-version" : 1, - "table-uuid" : "5ec03633-03bc-4c4b-8ef9-f799c143e3e7", - "location" : "mr/src/test/resources/test-table", - "last-updated-ms" : 1582645440292, - "last-column-id" : 2, - "schema" : { - "type" : "struct", - "fields" : [ { - "id" : 1, - "name" : "name", - "required" : false, - "type" : "string" - }, { - "id" : 2, - "name" : "salary", - "required" : false, - "type" : "long" - } ] - }, - "partition-spec" : [ ], - "default-spec-id" : 0, - "partition-specs" : [ { - "spec-id" : 0, - "fields" : [ ] - } ], - "properties" : { }, - "current-snapshot-id" : -1, - "snapshots" : [ ], - "snapshot-log" : [ ] -} \ No newline at end of file diff --git a/mr/src/test/resources/test-table-to-delete/metadata/v2.metadata.json b/mr/src/test/resources/test-table-to-delete/metadata/v2.metadata.json deleted file mode 100644 index 80a84f69913f..000000000000 --- a/mr/src/test/resources/test-table-to-delete/metadata/v2.metadata.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "format-version" : 1, - "table-uuid" : "5ec03633-03bc-4c4b-8ef9-f799c143e3e7", - "location" : "mr/src/test/resources/test-table", - "last-updated-ms" : 1582645443979, - "last-column-id" : 2, - "schema" : { - "type" : "struct", - "fields" : [ { - "id" : 1, - "name" : "name", - "required" : false, - "type" : "string" - }, { - "id" : 2, - "name" : "salary", - "required" : false, - "type" : "long" - } ] - }, - "partition-spec" : [ ], - "default-spec-id" : 0, - "partition-specs" : [ { - "spec-id" : 0, - "fields" : [ ] - } ], - "properties" : { }, - "current-snapshot-id" : 7829799286772121706, - "snapshots" : [ { - "snapshot-id" : 7829799286772121706, - "timestamp-ms" : 1582645443979, - "summary" : { - "operation" : "append", - "spark.app.id" : "local-1582645439954", - "added-data-files" : "1", - "added-records" : "3", - "changed-partition-count" : "1", - "total-records" : "3", - "total-data-files" : "1" - }, - "manifest-list" : "mr/src/test/resources/test-table/metadata/snap-7829799286772121706-1-1a3ffe32-d8da-47cf-9a8c-0e4c889a3a4c.avro" - } ], - "snapshot-log" : [ { - "timestamp-ms" : 1582645443979, - "snapshot-id" : 7829799286772121706 - } ] -} \ No newline at end of file diff --git a/mr/src/test/resources/test-table-to-delete/metadata/version-hint.text b/mr/src/test/resources/test-table-to-delete/metadata/version-hint.text deleted file mode 100644 index d8263ee98605..000000000000 --- a/mr/src/test/resources/test-table-to-delete/metadata/version-hint.text +++ /dev/null @@ -1 +0,0 @@ -2 \ No newline at end of file From f94401adb281a597747db91811d29ec7d0a8fe52 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Tue, 5 May 2020 17:31:33 +0100 Subject: [PATCH 35/51] refactor (mostly common) findTable and tableScan code --- .../apache/iceberg/mr/InputFormatConfig.java | 72 +++++++++++++++---- .../iceberg/mr/mapred/IcebergInputFormat.java | 28 ++------ .../mr/mapreduce/IcebergInputFormat.java | 26 +------ .../mr/mapred/TestIcebergInputFormat.java | 7 +- 4 files changed, 70 insertions(+), 63 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java index bbfecd0e6b25..af3b4afa183a 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -20,11 +20,15 @@ package org.apache.iceberg.mr; import com.google.common.base.Preconditions; +import java.net.URI; +import java.net.URISyntaxException; import java.util.function.Function; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TableScan; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.common.DynConstructors; @@ -35,6 +39,7 @@ public class InputFormatConfig { private InputFormatConfig() {} + // configuration values for Iceberg input formats public static final String REUSE_CONTAINERS = "iceberg.mr.reuse.containers"; public static final String CASE_SENSITIVE = "iceberg.mr.case.sensitive"; public static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering"; @@ -49,6 +54,9 @@ private InputFormatConfig() {} public static final String LOCALITY = "iceberg.mr.locality"; public static final String CATALOG = "iceberg.mr.catalog"; + // configuration value set by Hive to contain Table location + public static final String TABLE_LOCATION = "location"; + public static class ConfigBuilder { private final Configuration conf; @@ -104,8 +112,7 @@ public ConfigBuilder splitSize(long splitSize) { } /** - * If this API is called. The input splits - * constructed will have host location information + * If this API is called. The input splits constructed will have host location information */ public ConfigBuilder preferLocality() { conf.setBoolean(LOCALITY, true); @@ -119,9 +126,9 @@ public ConfigBuilder catalogFunc(Class catalogFunc = (Function) - DynConstructors.builder(Function.class) - .impl(catalogFuncClass) - .build() - .newInstance(); + Function catalogFunc = (Function) DynConstructors + .builder(Function.class) + .impl(catalogFuncClass) + .build() + .newInstance(); Catalog catalog = catalogFunc.apply(conf); TableIdentifier tableIdentifier = TableIdentifier.parse(path); return catalog.loadTable(tableIdentifier); @@ -152,4 +173,31 @@ public static Table findTable(Configuration conf) { } } + public static TableScan createTableScan(Configuration conf, Table table) { + TableScan scan = table.newScan().caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); + long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); + if (snapshotId != -1) { + scan = scan.useSnapshot(snapshotId); + } + long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); + if (asOfTime != -1) { + scan = scan.asOfTime(asOfTime); + } + long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); + if (splitSize > 0) { + scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); + } + String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); + if (schemaStr != null) { + scan.project(SchemaParser.fromJson(schemaStr)); + } + + // TODO add a filter parser to get rid of Serialization + Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); + if (filter != null) { + scan = scan.filter(filter); + } + return scan; + } + } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 32900b3888c4..821e67cdff06 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -22,8 +22,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; @@ -41,9 +39,10 @@ import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.SerializationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,36 +55,19 @@ public class IcebergInputFormat implements InputFormat, CombineHiveInputFormat.AvoidSplitCombination { private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); - static final String TABLE_LOCATION = "location"; - private Table table; @Override public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { - table = findTable(conf); - CloseableIterable taskIterable = table.newScan().planTasks(); + table = InputFormatConfig.findTable(conf); + TableScan scan = InputFormatConfig.createTableScan(conf, table); + CloseableIterable taskIterable = scan.planTasks(); List tasks = (List) StreamSupport .stream(taskIterable.spliterator(), false) .collect(Collectors.toList()); return createSplits(tasks, table.location()); } - private Table findTable(JobConf conf) throws IOException { - HadoopTables tables = new HadoopTables(conf); - String tableDir = conf.get(TABLE_LOCATION); - if (tableDir == null) { - throw new IllegalArgumentException("Table 'location' not set in JobConf"); - } - URI location = null; - try { - location = new URI(tableDir); - } catch (URISyntaxException e) { - throw new IOException("Unable to create URI for table location: '" + tableDir + "'", e); - } - table = tables.load(location.getPath()); - return table; - } - private InputSplit[] createSplits(List tasks, String location) { InputSplit[] splits = new InputSplit[tasks.size()]; for (int i = 0; i < tasks.size(); i++) { diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 9371ec70e65b..52c7e70077d6 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -48,7 +48,6 @@ import org.apache.iceberg.SchemaParser; import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; @@ -99,30 +98,7 @@ public List getSplits(JobContext context) { Configuration conf = context.getConfiguration(); Table table = InputFormatConfig.findTable(conf); - TableScan scan = table.newScan() - .caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); - long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); - if (snapshotId != -1) { - scan = scan.useSnapshot(snapshotId); - } - long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); - if (asOfTime != -1) { - scan = scan.asOfTime(asOfTime); - } - long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); - if (splitSize > 0) { - scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); - } - String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); - if (schemaStr != null) { - scan.project(SchemaParser.fromJson(schemaStr)); - } - - // TODO add a filter parser to get rid of Serialization - Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); - if (filter != null) { - scan = scan.filter(filter); - } + TableScan scan = InputFormatConfig.createTableScan(conf, table); splits = Lists.newArrayList(); boolean applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index e64ff021ea38..5b7844b4ee02 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -30,6 +30,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.data.Record; import org.apache.iceberg.mr.BaseInputFormatTest; +import org.apache.iceberg.mr.InputFormatConfig; import org.junit.Assert; import org.junit.Test; import org.junit.runners.Parameterized; @@ -62,17 +63,17 @@ public void testGetSplitsNoLocation() throws IOException { inputFormat.getSplits(jobConf, 1); } - @Test(expected = IOException.class) + @Test(expected = IllegalArgumentException.class) public void testGetSplitsInvalidLocationUri() throws IOException { JobConf jobConf = new JobConf(); - jobConf.set(IcebergInputFormat.TABLE_LOCATION, "http:"); + jobConf.set(InputFormatConfig.TABLE_LOCATION, "http:"); inputFormat.getSplits(jobConf, 1); } @Override protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { JobConf jobConf = new JobConf(); - jobConf.set(IcebergInputFormat.TABLE_LOCATION, "file:" + tableLocation); + jobConf.set(InputFormatConfig.TABLE_LOCATION, "file:" + tableLocation); validate(jobConf, expectedRecords); } From 13386760b828fa3f2b4e1cbc601bb0041d508bd6 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Tue, 5 May 2020 17:44:14 +0100 Subject: [PATCH 36/51] put some of the generic templates back --- .../org/apache/iceberg/mr/IcebergRecordReader.java | 12 ++++++------ .../apache/iceberg/mr/mapred/IcebergInputFormat.java | 7 ++++--- .../iceberg/mr/mapreduce/IcebergInputFormat.java | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java b/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java index 8ddb543f4495..db37243f80d6 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java +++ b/mr/src/main/java/org/apache/iceberg/mr/IcebergRecordReader.java @@ -37,7 +37,7 @@ import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; -public class IcebergRecordReader { +public class IcebergRecordReader { private boolean applyResidual; private boolean caseSensitive; @@ -49,7 +49,7 @@ private void initialize(Configuration conf) { this.reuseContainers = conf.getBoolean(InputFormatConfig.REUSE_CONTAINERS, false); } - public CloseableIterable createReader(Configuration config, FileScanTask currentTask, Schema readSchema) { + public CloseableIterable createReader(Configuration config, FileScanTask currentTask, Schema readSchema) { initialize(config); DataFile file = currentTask.file(); // TODO we should make use of FileIO to create inputFile @@ -67,7 +67,7 @@ public CloseableIterable createReader(Configuration config, FileScanTask current } } - private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Avro.ReadBuilder avroReadBuilder = Avro.read(inputFile).project(readSchema).split(task.start(), task.length()); if (reuseContainers) { avroReadBuilder.reuseContainers(); @@ -76,7 +76,7 @@ private CloseableIterable newAvroIterable(InputFile inputFile, FileScanTask task return applyResidualFiltering(avroReadBuilder.build(), task.residual(), readSchema); } - private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { Parquet.ReadBuilder parquetReadBuilder = Parquet .read(inputFile) .project(readSchema) @@ -92,7 +92,7 @@ private CloseableIterable newParquetIterable(InputFile inputFile, FileScanTask t return applyResidualFiltering(parquetReadBuilder.build(), task.residual(), readSchema); } - private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { + private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, Schema readSchema) { ORC.ReadBuilder orcReadBuilder = ORC .read(inputFile) .project(readSchema) @@ -103,7 +103,7 @@ private CloseableIterable newOrcIterable(InputFile inputFile, FileScanTask task, return applyResidualFiltering(orcReadBuilder.build(), task.residual(), readSchema); } - private CloseableIterable applyResidualFiltering(CloseableIterable iter, Expression residual, Schema readSchema) { + private CloseableIterable applyResidualFiltering(CloseableIterable iter, Expression residual, Schema readSchema) { if (applyResidual && residual != null && residual != Expressions.alwaysTrue()) { Evaluator filter = new Evaluator(readSchema.asStruct(), residual, caseSensitive); return CloseableIterable.filter(iter, record -> filter.eval((StructLike) record)); diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 821e67cdff06..7b073051f523 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -77,7 +77,7 @@ private InputSplit[] createSplits(List tasks, String location) } @Override - public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { + public RecordReader getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException { return new IcebergRecordReader(split, job); } @@ -86,7 +86,7 @@ public boolean shouldSkipCombine(Path path, Configuration conf) throws IOExcepti return true; } - public class IcebergRecordReader implements RecordReader { + public class IcebergRecordReader implements RecordReader { private JobConf conf; private IcebergSplit split; @@ -109,7 +109,8 @@ private void initialise() { private void nextTask() { FileScanTask currentTask = tasks.next(); Schema tableSchema = table.schema(); - org.apache.iceberg.mr.IcebergRecordReader wrappedReader = new org.apache.iceberg.mr.IcebergRecordReader(); + org.apache.iceberg.mr.IcebergRecordReader wrappedReader = + new org.apache.iceberg.mr.IcebergRecordReader(); reader = wrappedReader.createReader(conf, currentTask, tableSchema); recordIterator = reader.iterator(); } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 52c7e70077d6..a65b84653255 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -231,7 +231,7 @@ private Iterator open(FileScanTask currentTask) { } private Iterator open(FileScanTask currentTask, Schema readSchema) { - org.apache.iceberg.mr.IcebergRecordReader wrappedReader = new org.apache.iceberg.mr.IcebergRecordReader(); + org.apache.iceberg.mr.IcebergRecordReader wrappedReader = new org.apache.iceberg.mr.IcebergRecordReader(); CloseableIterable iterable = wrappedReader.createReader(context.getConfiguration(), currentTask, readSchema); currentCloseable = iterable; return iterable.iterator(); From b5a389c42535581724dc88221391a44d7dfe3306 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Wed, 6 May 2020 14:07:22 +0100 Subject: [PATCH 37/51] orc tests appear to be working --- .../iceberg/mr/mapred/TestIcebergInputFormat.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 5b7844b4ee02..02c7e568a3a2 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -33,7 +33,6 @@ import org.apache.iceberg.mr.InputFormatConfig; import org.junit.Assert; import org.junit.Test; -import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,16 +42,6 @@ public class TestIcebergInputFormat extends BaseInputFormatTest { private IcebergInputFormat inputFormat = new IcebergInputFormat(); - @Parameterized.Parameters - public static Object[][] parameters() { - return new Object[][] { new Object[] { "parquet" }, new Object[] { "avro" } - /* - * , TODO: put orc back, seems to be an issue with different versions of Orc in Hive and Iceberg new - * Object[]{"orc"} - */ - }; - } - public TestIcebergInputFormat(String fileFormat) { this.fileFormat = FileFormat.valueOf(fileFormat.toUpperCase(Locale.ENGLISH)); } From e1b81dc9fbcb2f3a20fc29d0bfc9750d816554c9 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 4 Jun 2020 13:53:22 +0100 Subject: [PATCH 38/51] added a HiveRunner test for the mapred InputFormat --- build.gradle | 26 + mr/dependencies.lock | 9166 ++++++++++++++--- .../mr/mapred/TestHiveIcebergInputFormat.java | 85 + 3 files changed, 7978 insertions(+), 1299 deletions(-) create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java diff --git a/build.gradle b/build.gradle index 66e2274aa04c..19184f5b1431 100644 --- a/build.gradle +++ b/build.gradle @@ -323,6 +323,32 @@ project(':iceberg-mr') { exclude group: 'com.google.guava' } + testCompile("com.klarna:hiverunner:5.2.1") { + exclude group: 'javax.jms', module: 'jms' + exclude group: 'org.apache.hive', module: 'hive-exec' + exclude group: 'org.codehaus.jettison', module: 'jettison' + exclude group: 'org.apache.calcite.avatica' + //exclude group: 'com.fasterxml.jackson.core' + } + + testCompile("org.apache.hive:hive-exec::core") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.pentaho' // missing dependency + exclude group: 'org.apache.hive', module: 'hive-llap-tez' + exclude group: 'org.apache.logging.log4j' + exclude group: 'com.google.protobuf', module: 'protobuf-java' + exclude group: 'org.apache.calcite.avatica' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + //exclude group: 'com.fasterxml.jackson.core' + } + + testCompile("org.apache.calcite:calcite-core") + testCompile("com.esotericsoftware.kryo:kryo:2.24.0") + testCompile("com.fasterxml.jackson.core:jackson-annotations:2.6.5") + //testCompile("com.fasterxml.jackson.core:jackson-databind:2.6.0") + + testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') testCompile project(path: ':iceberg-core', configuration: 'testArtifacts') diff --git a/mr/dependencies.lock b/mr/dependencies.lock index 555afd070dcc..da03056f5794 100644 --- a/mr/dependencies.lock +++ b/mr/dependencies.lock @@ -1,4 +1,365 @@ { + "allProcessors": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "annotationProcessor": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "checkstyle": { + "antlr:antlr": { + "locked": "2.7.7", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.1.3", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "26.0-jre", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.puppycrawl.tools:checkstyle": { + "locked": "8.13" + }, + "commons-beanutils:commons-beanutils": { + "locked": "1.9.3", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "commons-cli:commons-cli": { + "locked": "1.4", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "commons-collections:commons-collections": { + "locked": "3.2.2", + "transitive": [ + "commons-beanutils:commons-beanutils" + ] + }, + "net.sf.saxon:Saxon-HE": { + "locked": "9.8.0-14", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "org.antlr:antlr4-runtime": { + "locked": "4.7.1", + "transitive": [ + "com.puppycrawl.tools:checkstyle" + ] + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.2", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.14", + "transitive": [ + "com.google.guava:guava" + ] + } + }, "compile": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", @@ -285,7 +646,8 @@ "com.google.code.gson:gson": { "locked": "2.2.4", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec" ] }, "com.google.errorprone:error_prone_annotations": { @@ -294,21 +656,6 @@ "com.github.ben-manes.caffeine:caffeine" ] }, - "com.google.guava:guava": { - "locked": "16.0.1", - "transitive": [ - "org.apache.curator:curator-client", - "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" - ] - }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ @@ -399,7 +746,7 @@ ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", "org.apache.hadoop:hadoop-auth", @@ -407,6 +754,7 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-exec", "org.apache.httpcomponents:httpclient" ] }, @@ -432,7 +780,8 @@ "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec" ] }, "commons-io:commons-io": { @@ -440,7 +789,8 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-exec" ] }, "commons-lang:commons-lang": { @@ -452,11 +802,14 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-vector-code-gen", + "org.apache.velocity:velocity" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", @@ -539,6 +892,12 @@ "org.apache.zookeeper:zookeeper" ] }, + "junit:junit": { + "locked": "3.8.1", + "transitive": [ + "jline:jline" + ] + }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ @@ -549,6 +908,32 @@ "org.apache.zookeeper:zookeeper" ] }, + "org.antlr:ST4": { + "locked": "4.0.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.antlr:antlr-runtime": { + "locked": "3.5.2", + "transitive": [ + "org.antlr:ST4", + "org.apache.hive:hive-exec" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ @@ -560,7 +945,8 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-exec" ] }, "org.apache.commons:commons-math3": { @@ -569,6 +955,12 @@ "org.apache.hadoop:hadoop-common" ] }, + "org.apache.curator:apache-curator": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.apache.curator:curator-client": { "locked": "2.7.1", "transitive": [ @@ -580,7 +972,9 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec" ] }, "org.apache.curator:curator-recipes": { @@ -715,23 +1109,46 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, - "org.apache.htrace:htrace-core": { - "locked": "3.1.0-incubating", + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-exec": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.htrace:htrace-core": { + "locked": "3.1.0-incubating", + "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs" ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -773,6 +1190,12 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ @@ -831,6 +1254,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common" + ] + }, + "org.apache.velocity:velocity": { + "locked": "1.5", + "transitive": [ + "org.apache.hive:hive-vector-code-gen" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -840,12 +1275,15 @@ "org.apache.zookeeper:zookeeper": { "locked": "3.4.6", "transitive": [ + "org.apache.curator:apache-curator", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec" ] }, "org.checkerframework:checker-qual": { @@ -854,6 +1292,12 @@ "com.github.ben-manes.caffeine:caffeine" ] }, + "org.codehaus.groovy:groovy-all": { + "locked": "2.4.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.codehaus.jackson:jackson-core-asl": { "locked": "1.9.13", "transitive": [ @@ -898,6 +1342,12 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.fusesource.leveldbjni:leveldbjni-all": { "locked": "1.8", "transitive": [ @@ -931,6 +1381,10 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-vector-code-gen", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", @@ -941,6 +1395,7 @@ "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper" ] }, @@ -962,6 +1417,18 @@ "org.apache.parquet:parquet-hadoop" ] }, + "oro:oro": { + "locked": "2.0.8", + "transitive": [ + "org.apache.velocity:velocity" + ] + }, + "stax:stax-api": { + "locked": "1.0.1", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -1005,7 +1472,8 @@ "com.google.code.gson:gson": { "locked": "2.2.4", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec" ] }, "com.google.guava:guava": { @@ -1023,12 +1491,24 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { @@ -1051,14 +1531,18 @@ "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-core": { @@ -1068,14 +1552,18 @@ "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-server": { @@ -1113,7 +1601,7 @@ ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", "org.apache.hadoop:hadoop-auth", @@ -1121,6 +1609,7 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive:hive-exec", "org.apache.httpcomponents:httpclient" ] }, @@ -1128,7 +1617,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" ] }, "commons-configuration:commons-configuration": { @@ -1146,7 +1636,8 @@ "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec" ] }, "commons-io:commons-io": { @@ -1154,7 +1645,9 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive:hive-exec" ] }, "commons-lang:commons-lang": { @@ -1166,11 +1659,16 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-vector-code-gen", + "org.apache.velocity:velocity" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", @@ -1182,8 +1680,11 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", "org.apache.httpcomponents:httpclient" ] }, @@ -1237,7 +1738,9 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "javax.xml.stream:stax-api": { @@ -1252,6 +1755,12 @@ "org.apache.zookeeper:zookeeper" ] }, + "junit:junit": { + "locked": "3.8.1", + "transitive": [ + "jline:jline" + ] + }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ @@ -1260,14 +1769,42 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.zookeeper:zookeeper" ] }, + "org.antlr:ST4": { + "locked": "4.0.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.antlr:antlr-runtime": { + "locked": "3.5.2", + "transitive": [ + "org.antlr:ST4", + "org.apache.hive:hive-exec" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.commons:commons-compress": { - "locked": "1.4.1", + "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-exec" ] }, "org.apache.commons:commons-math3": { @@ -1276,6 +1813,12 @@ "org.apache.hadoop:hadoop-common" ] }, + "org.apache.curator:apache-curator": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.apache.curator:curator-client": { "locked": "2.7.1", "transitive": [ @@ -1287,7 +1830,9 @@ "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec" ] }, "org.apache.curator:curator-recipes": { @@ -1324,7 +1869,12 @@ "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-auth": { @@ -1388,8 +1938,11 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-client": { @@ -1404,8 +1957,17 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -1413,7 +1975,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -1422,6 +1987,53 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-exec": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ @@ -1430,26 +2042,56 @@ ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.4.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive.shims:hive-shims-common" + ] + }, + "org.apache.velocity:velocity": { + "locked": "1.5", + "transitive": [ + "org.apache.hive:hive-vector-code-gen" ] }, "org.apache.zookeeper:zookeeper": { "locked": "3.4.6", "transitive": [ + "org.apache.curator:apache-curator", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec" + ] + }, + "org.codehaus.groovy:groovy-all": { + "locked": "2.4.4", + "transitive": [ + "org.apache.hive:hive-exec" ] }, "org.codehaus.jackson:jackson-core-asl": { @@ -1493,7 +2135,15 @@ "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", + "transitive": [ + "org.apache.hive:hive-exec" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -1501,12 +2151,14 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.slf4j:slf4j-api": { - "locked": "1.7.10", + "locked": "1.7.12", "transitive": [ "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -1522,6 +2174,14 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-vector-code-gen", + "org.apache.thrift:libthrift", "org.apache.zookeeper:zookeeper" ] }, @@ -1531,10 +2191,16 @@ "com.google.inject:guice" ] }, - "org.tukaani:xz": { - "locked": "1.0", + "oro:oro": { + "locked": "2.0.8", + "transitive": [ + "org.apache.velocity:velocity" + ] + }, + "stax:stax-api": { + "locked": "1.0.1", "transitive": [ - "org.apache.commons:commons-compress" + "org.apache.hive:hive-exec" ] }, "xerces:xercesImpl": { @@ -1782,6 +2448,154 @@ ] } }, + "errorprone": { + "com.github.kevinstern:software-and-algorithms": { + "locked": "1.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.auto:auto-common": { + "locked": "0.10", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "errorproneJavac": { + "com.google.errorprone:javac": { + "locked": "9+181-r4173-1", + "requested": "9+181-r4173-1" + } + }, "runtime": { "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", @@ -2232,38 +3046,259 @@ ] } }, - "testCompile": { - "aopalliance:aopalliance": { + "testAnnotationProcessor": { + "com.github.kevinstern:software-and-algorithms": { "locked": "1.0", "transitive": [ - "com.google.inject:guice" + "com.google.errorprone:error_prone_check_api" ] }, - "asm:asm": { - "locked": "3.1", + "com.github.stephenc.jcip:jcip-annotations": { + "locked": "1.0-1", "transitive": [ - "com.sun.jersey:jersey-server", - "org.sonatype.sisu.inject:cglib" + "com.google.errorprone:error_prone_core" ] }, - "com.fasterxml.jackson.core:jackson-annotations": { - "locked": "2.10.2", + "com.google.auto:auto-common": { + "locked": "0.10", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind" + "com.google.errorprone:error_prone_core" ] }, - "com.fasterxml.jackson.core:jackson-core": { - "locked": "2.10.2", + "com.google.code.findbugs:jFormatString": { + "locked": "3.0.0", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind", - "org.apache.avro:avro", - "org.apache.iceberg:iceberg-core" + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.2", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_annotation": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core", + "com.google.guava:guava" + ] + }, + "com.google.errorprone:error_prone_check_api": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.errorprone:error_prone_core": { + "locked": "2.3.3", + "transitive": [ + "com.palantir.baseline:baseline-error-prone" + ] + }, + "com.google.errorprone:error_prone_type_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:failureaccess": { + "locked": "1.0.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.guava:guava": { + "locked": "27.0.1-jre", + "transitive": [ + "com.google.auto:auto-common", + "com.google.errorprone:error_prone_annotation", + "com.google.errorprone:error_prone_core" + ] + }, + "com.google.guava:listenablefuture": { + "locked": "9999.0-empty-to-avoid-conflict-with-guava", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.j2objc:j2objc-annotations": { + "locked": "1.1", + "transitive": [ + "com.google.guava:guava" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.4.0", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + }, + "com.googlecode.java-diff-utils:diffutils": { + "locked": "1.3.0", + "transitive": [ + "com.google.errorprone:error_prone_check_api" + ] + }, + "com.palantir.baseline:baseline-error-prone": { + "locked": "0.55.0", + "requested": "0.55.0" + }, + "org.checkerframework:checker-qual": { + "locked": "2.5.3", + "transitive": [ + "com.google.guava:guava", + "org.checkerframework:dataflow", + "org.checkerframework:javacutil" + ] + }, + "org.checkerframework:dataflow": { + "locked": "2.5.3", + "transitive": [ + "com.google.errorprone:error_prone_check_api", + "com.google.errorprone:error_prone_core" + ] + }, + "org.checkerframework:javacutil": { + "locked": "2.5.3", + "transitive": [ + "org.checkerframework:dataflow" + ] + }, + "org.codehaus.mojo:animal-sniffer-annotations": { + "locked": "1.17", + "transitive": [ + "com.google.guava:guava" + ] + }, + "org.pcollections:pcollections": { + "locked": "2.1.2", + "transitive": [ + "com.google.errorprone:error_prone_core" + ] + } + }, + "testCompile": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, + "aopalliance:aopalliance": { + "locked": "1.0", + "transitive": [ + "com.google.inject:guice" + ] + }, + "asm:asm": { + "locked": "3.1", + "transitive": [ + "asm:asm-tree", + "com.sun.jersey:jersey-server", + "org.sonatype.sisu.inject:cglib" + ] + }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "ch.qos.logback:logback-classic": { + "locked": "1.0.9", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "ch.qos.logback:logback-core": { + "locked": "1.0.9", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "co.cask.tephra:tephra-api": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-core": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-hbase-compat-1.0": { + "locked": "0.6.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.beust:jcommander": { + "locked": "1.30", + "transitive": [ + "org.apache.slider:slider-core" + ] + }, + "com.esotericsoftware.kryo:kryo": { + "locked": "2.24.0", + "requested": "2.24.0" + }, + "com.esotericsoftware.minlog:minlog": { + "locked": "1.2", + "transitive": [ + "com.esotericsoftware.kryo:kryo" + ] + }, + "com.fasterxml.jackson.core:jackson-annotations": { + "locked": "2.10.2", + "requested": "2.6.5", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.calcite.avatica:avatica" + ] + }, + "com.fasterxml.jackson.core:jackson-core": { + "locked": "2.10.2", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.iceberg:iceberg-core" ] }, "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -2273,9 +3308,24 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", @@ -2287,13 +3337,26 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.calcite:calcite-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals", + "org.apache.twill:twill-api", + "org.apache.twill:twill-common", + "org.apache.twill:twill-zookeeper" ] }, "com.google.code.gson:gson": { "locked": "2.2.4", "transitive": [ - "org.apache.hadoop:hadoop-common" + "co.cask.tephra:tephra-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, "com.google.errorprone:error_prone_annotations": { @@ -2303,8 +3366,14 @@ ] }, "com.google.guava:guava": { - "locked": "16.0.1", - "transitive": [ + "locked": "18.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.jolbox:bonecp", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", + "org.apache.calcite:calcite-linq4j", + "org.apache.curator:apache-curator", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", @@ -2313,21 +3382,68 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-vector-code-gen", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", + "org.reflections:reflections" + ] + }, + "com.google.inject.extensions:guice-assistedinject": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "co.cask.tephra:tephra-core", + "com.google.inject.extensions:guice-assistedinject", + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.google.protobuf:protobuf-java": { - "locked": "2.5.0", + "locked": "3.0.0-beta-1", "transitive": [ + "org.apache.calcite.avatica:avatica", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-app", @@ -2337,22 +3453,77 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-metastore", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "com.jolbox:bonecp": { + "locked": "0.8.0.RELEASE", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.klarna:hiverunner": { + "locked": "5.2.1", + "requested": "5.2.1" + }, + "com.lmax:disruptor": { + "locked": "3.3.0", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "com.ning:async-http-client": { + "locked": "1.8.16", + "transitive": [ + "org.apache.tez:tez-runtime-library" ] }, "com.sun.jersey.contribs:jersey-guice": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "com.sun.jersey:jersey-client": { "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "com.sun.jersey:jersey-core": { @@ -2361,22 +3532,36 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-server" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" ] }, "com.sun.xml.bind:jaxb-impl": { @@ -2385,6 +3570,26 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server" + ] + }, + "com.yammer.metrics:metrics-core": { + "locked": "2.2.0", + "transitive": [ + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" + ] + }, + "com.zaxxer:HikariCP": { + "locked": "2.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -2403,26 +3608,48 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.tez:tez-dag" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.httpcomponents:httpclient", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-library" ] }, "commons-collections:commons-collections": { "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-mapreduce" ] }, "commons-configuration:commons-configuration": { @@ -2431,16 +3658,38 @@ "org.apache.hadoop:hadoop-common" ] }, + "commons-daemon:commons-daemon": { + "locked": "1.0.13", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "commons-dbcp:commons-dbcp": { + "locked": "1.4", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-metastore" + ] + }, "commons-digester:commons-digester": { "locked": "1.8", "transitive": [ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" ] }, "commons-io:commons-io": { @@ -2448,7 +3697,14 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "commons-lang:commons-lang": { @@ -2460,25 +3716,64 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.velocity:velocity", + "org.pentaho:pentaho-aggdesigner-algorithm" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", + "net.sf.jpam:jpam", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.pentaho:pentaho-aggdesigner-algorithm" ] }, "commons-net:commons-net": { @@ -2490,26 +3785,88 @@ "commons-pool:commons-pool": { "locked": "1.6", "transitive": [ + "commons-dbcp:commons-dbcp", + "org.apache.hive:hive-metastore", "org.apache.parquet:parquet-hadoop" ] }, + "dom4j:dom4j": { + "locked": "1.6.1", + "transitive": [ + "org.reflections:reflections" + ] + }, "io.airlift:aircompressor": { "locked": "0.15", "transitive": [ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { - "locked": "3.7.0.Final", + "locked": "3.9.2.Final", "transitive": [ + "com.ning:async-http-client", "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", "org.apache.zookeeper:zookeeper" ] }, "io.netty:netty-all": { "locked": "4.0.23.Final", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server" + ] + }, + "it.unimi.dsi:fastutil": { + "locked": "6.5.6", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "jakarta.jms:jakarta.jms-api": { + "locked": "2.0.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions" + ] + }, + "javassist:javassist": { + "locked": "3.12.1.GA", + "transitive": [ + "org.reflections:reflections" + ] + }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" ] }, "javax.annotation:javax.annotation-api": { @@ -2525,17 +3882,56 @@ "com.sun.jersey.contribs:jersey-guice" ] }, - "javax.servlet.jsp:jsp-api": { - "locked": "2.1", + "javax.jdo:jdo-api": { + "locked": "3.0.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hive:hive-metastore" + ] + }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet.jsp:jsp-api": { + "locked": "2.1", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.slider:slider-core" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" ] }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.slider:slider-core", + "org.apache.tez:tez-dag", + "org.eclipse.jetty.aggregate:jetty-all", + "tomcat:jasper-runtime" + ] + }, + "javax.transaction:jta": { + "locked": "1.1", + "transitive": [ + "javax.jdo:jdo-api" + ] + }, + "javax.transaction:transaction-api": { + "locked": "1.1", + "transitive": [ + "org.datanucleus:javax.jdo" ] }, "javax.xml.bind:jaxb-api": { @@ -2543,18 +3939,46 @@ "transitive": [ "com.sun.xml.bind:jaxb-impl", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.orc:orc-core" ] }, + "javolution:javolution": { + "locked": "5.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-common" + ] + }, "junit:junit": { - "locked": "4.12" + "locked": "4.12", + "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server" + ] }, "log4j:log4j": { "locked": "1.2.17", @@ -2564,13 +3988,120 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", "org.apache.zookeeper:zookeeper" ] }, + "net.hydromatic:eigenbase-properties": { + "locked": "1.1.5", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.jpam:jpam": { + "locked": "1.1", + "transitive": [ + "org.apache.hive:hive-service" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.antlr:ST4": { + "locked": "4.0.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.antlr:antlr-runtime": { + "locked": "3.5.2", + "transitive": [ + "org.antlr:ST4", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ - "org.apache.iceberg:iceberg-core" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.iceberg:iceberg-core", + "org.apache.slider:slider-core" + ] + }, + "org.apache.calcite.avatica:avatica": { + "locked": "1.8.0", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.apache.calcite.avatica:avatica-metrics": { + "locked": "1.8.0", + "transitive": [ + "org.apache.calcite.avatica:avatica" + ] + }, + "org.apache.calcite:calcite-core": { + "locked": "1.10.0", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-exec" + ] + }, + "org.apache.calcite:calcite-druid": { + "locked": "1.10.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.calcite:calcite-linq4j": { + "locked": "1.10.0", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid" + ] + }, + "org.apache.commons:commons-collections4": { + "locked": "4.1", + "transitive": [ + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag" ] }, "org.apache.commons:commons-compress": { @@ -2578,33 +4109,78 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.2", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-tez" + ] + }, + "org.apache.commons:commons-math": { + "locked": "2.2", + "transitive": [ + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" ] }, "org.apache.commons:commons-math3": { "locked": "3.1.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.tez:tez-dag" + ] + }, + "org.apache.curator:apache-curator": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-client" ] }, "org.apache.curator:curator-client": { "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-framework", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.slider:slider-core" ] }, "org.apache.curator:curator-framework": { "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" ] }, "org.apache.curator:curator-recipes": { "locked": "2.7.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" + ] + }, + "org.apache.derby:derby": { + "locked": "10.10.2.0", + "transitive": [ + "org.apache.hive:hive-metastore" ] }, "org.apache.directory.api:api-asn1-api": { @@ -2631,32 +4207,100 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-archives": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core" ] }, "org.apache.hadoop:hadoop-auth": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-api", + "org.apache.tez:tez-runtime-library" ] }, "org.apache.hadoop:hadoop-client": { - "locked": "2.7.3" + "locked": "2.7.3", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] }, "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, "org.apache.hadoop:hadoop-hdfs": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "org.apache.hadoop:hadoop-mapreduce-client-app": { @@ -2670,14 +4314,22 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-app", "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.tez:tez-mapreduce" ] }, "org.apache.hadoop:hadoop-mapreduce-client-core": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:tez-mapreduce" ] }, "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { @@ -2699,14 +4351,28 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, "org.apache.hadoop:hadoop-yarn-client": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" ] }, "org.apache.hadoop:hadoop-yarn-common": { @@ -2715,8 +4381,30 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-yarn-registry": { + "locked": "2.7.1", + "transitive": [ + "org.apache.slider:slider-core" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -2724,7 +4412,10 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -2733,998 +4424,854 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, - "org.apache.htrace:htrace-core": { - "locked": "3.1.0-incubating", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive.shims:hive-shims-0.23" ] }, - "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.tez:tez-dag" ] }, - "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "org.apache.hbase:hbase-annotations": { + "locked": "1.1.1", "transitive": [ - "org.apache.httpcomponents:httpclient" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-protocol" ] }, - "org.apache.iceberg:iceberg-api": { - "project": true, + "org.apache.hbase:hbase-client": { + "locked": "1.1.1", "transitive": [ - "org.apache.iceberg:iceberg-core", - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet" + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" ] }, - "org.apache.iceberg:iceberg-bundled-guava": { - "project": true, + "org.apache.hbase:hbase-common": { + "locked": "1.1.1", "transitive": [ - "org.apache.iceberg:iceberg-api", - "org.apache.iceberg:iceberg-common" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" ] }, - "org.apache.iceberg:iceberg-common": { - "project": true, + "org.apache.hbase:hbase-hadoop-compat": { + "locked": "1.1.1", "transitive": [ - "org.apache.iceberg:iceberg-core" + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" ] }, - "org.apache.iceberg:iceberg-core": { - "project": true, + "org.apache.hbase:hbase-hadoop2-compat": { + "locked": "1.1.1", "transitive": [ - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet" + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" ] }, - "org.apache.iceberg:iceberg-data": { - "project": true - }, - "org.apache.iceberg:iceberg-orc": { - "project": true + "org.apache.hbase:hbase-prefix-tree": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server" + ] }, - "org.apache.iceberg:iceberg-parquet": { - "project": true + "org.apache.hbase:hbase-procedure": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server" + ] }, - "org.apache.orc:orc-core": { - "locked": "1.6.3", + "org.apache.hbase:hbase-protocol": { + "locked": "1.1.1", "transitive": [ - "org.apache.iceberg:iceberg-orc" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server" ] }, - "org.apache.orc:orc-shims": { - "locked": "1.6.3", + "org.apache.hbase:hbase-server": { + "locked": "1.1.1", "transitive": [ - "org.apache.orc:orc-core" + "org.apache.hive:hive-llap-server" ] }, - "org.apache.parquet:parquet-avro": { - "locked": "1.11.0", + "org.apache.hive.hcatalog:hive-hcatalog-core": { + "locked": "2.3.7", "transitive": [ - "org.apache.iceberg:iceberg-parquet" + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client" ] }, - "org.apache.parquet:parquet-column": { - "locked": "1.11.0", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-avro", - "org.apache.parquet:parquet-hadoop" + "org.apache.hive.hcatalog:hive-webhcat-java-client" ] }, - "org.apache.parquet:parquet-common": { - "locked": "1.11.0", + "org.apache.hive.hcatalog:hive-webhcat-java-client": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-column", - "org.apache.parquet:parquet-encoding" + "com.klarna:hiverunner" ] }, - "org.apache.parquet:parquet-encoding": { - "locked": "1.11.0", + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-column" + "org.apache.hive:hive-shims" ] }, - "org.apache.parquet:parquet-format-structures": { - "locked": "1.11.0", + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-avro", - "org.apache.parquet:parquet-common", - "org.apache.parquet:parquet-hadoop" + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" ] }, - "org.apache.parquet:parquet-hadoop": { - "locked": "1.11.0", + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-avro" + "org.apache.hive:hive-shims" ] }, - "org.apache.parquet:parquet-jackson": { - "locked": "1.11.0", + "org.apache.hive:hive-cli": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-hadoop" + "org.apache.hive.hcatalog:hive-hcatalog-core" ] }, - "org.apache.yetus:audience-annotations": { - "locked": "0.11.0", + "org.apache.hive:hive-common": { + "locked": "2.3.7", "transitive": [ - "org.apache.parquet:parquet-common" + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-serde" ] }, - "org.apache.zookeeper:zookeeper": { - "locked": "3.4.6", + "org.apache.hive:hive-exec": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-jdbc": { + "locked": "2.3.7", "transitive": [ - "org.apache.curator:curator-client", - "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "com.klarna:hiverunner" ] }, - "org.checkerframework:checker-qual": { - "locked": "2.6.0", + "org.apache.hive:hive-llap-client": { + "locked": "2.3.7", "transitive": [ - "com.github.ben-manes.caffeine:caffeine" + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez" ] }, - "org.codehaus.jackson:jackson-core-asl": { - "locked": "1.9.13", + "org.apache.hive:hive-llap-common": { + "locked": "2.3.7", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.codehaus.jackson:jackson-jaxrs", - "org.codehaus.jackson:jackson-mapper-asl", - "org.codehaus.jackson:jackson-xc" + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-server" ] }, - "org.codehaus.jackson:jackson-jaxrs": { - "locked": "1.9.13", + "org.apache.hive:hive-llap-server": { + "locked": "2.3.7", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hive:hive-service" ] }, - "org.codehaus.jackson:jackson-mapper-asl": { - "locked": "1.9.13", + "org.apache.hive:hive-llap-tez": { + "locked": "2.3.7", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.codehaus.jackson:jackson-jaxrs", - "org.codehaus.jackson:jackson-xc" + "org.apache.hive:hive-llap-server" ] }, - "org.codehaus.jackson:jackson-xc": { - "locked": "1.9.13", + "org.apache.hive:hive-metastore": { + "locked": "2.3.7", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-service" ] }, - "org.codehaus.jettison:jettison": { - "locked": "1.1", + "org.apache.hive:hive-serde": { + "locked": "2.3.7", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" ] }, - "org.fusesource.leveldbjni:leveldbjni-all": { - "locked": "1.8", + "org.apache.hive:hive-service": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc" ] }, - "org.hamcrest:hamcrest-core": { - "locked": "1.3", + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", "transitive": [ - "junit:junit", - "org.mockito:mockito-core" + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service" ] }, - "org.jetbrains:annotations": { - "locked": "17.0.0", + "org.apache.hive:hive-shims": { + "locked": "2.3.7", "transitive": [ - "org.apache.orc:orc-core" + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde" ] }, - "org.mockito:mockito-core": { - "locked": "1.10.19" + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] }, - "org.objenesis:objenesis": { - "locked": "2.1", + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", "transitive": [ - "org.mockito:mockito-core" + "org.apache.hive:hive-exec" ] }, - "org.slf4j:slf4j-api": { - "locked": "1.7.25", + "org.apache.htrace:htrace-core": { + "locked": "3.1.0-incubating", "transitive": [ - "org.apache.avro:avro", - "org.apache.curator:curator-client", - "org.apache.directory.api:api-asn1-api", - "org.apache.directory.api:api-util", - "org.apache.directory.server:apacheds-i18n", - "org.apache.directory.server:apacheds-kerberos-codec", - "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.iceberg:iceberg-api", - "org.apache.iceberg:iceberg-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.2", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hadoop:hadoop-auth", + "org.apache.hive:hive-jdbc", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.httpcomponents:httpcore": { + "locked": "4.4.4", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-jdbc", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.iceberg:iceberg-api": { + "project": true, + "transitive": [ "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet", - "org.apache.orc:orc-core", - "org.apache.orc:orc-shims", - "org.apache.parquet:parquet-common", - "org.apache.parquet:parquet-format-structures", - "org.apache.zookeeper:zookeeper", - "org.slf4j:slf4j-simple" + "org.apache.iceberg:iceberg-parquet" ] }, - "org.slf4j:slf4j-simple": { - "locked": "1.7.25" - }, - "org.sonatype.sisu.inject:cglib": { - "locked": "2.2.1-v20090111", + "org.apache.iceberg:iceberg-bundled-guava": { + "project": true, "transitive": [ - "com.google.inject:guice" + "org.apache.iceberg:iceberg-api", + "org.apache.iceberg:iceberg-common" ] }, - "org.threeten:threeten-extra": { - "locked": "1.5.0", + "org.apache.iceberg:iceberg-common": { + "project": true, "transitive": [ - "org.apache.orc:orc-core" + "org.apache.iceberg:iceberg-core" ] }, - "org.xerial.snappy:snappy-java": { - "locked": "1.1.7.3", + "org.apache.iceberg:iceberg-core": { + "project": true, "transitive": [ - "org.apache.parquet:parquet-hadoop" + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet" ] }, - "xerces:xercesImpl": { - "locked": "2.9.1", + "org.apache.iceberg:iceberg-data": { + "project": true + }, + "org.apache.iceberg:iceberg-orc": { + "project": true + }, + "org.apache.iceberg:iceberg-parquet": { + "project": true + }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive:hive-exec" ] }, - "xml-apis:xml-apis": { - "locked": "1.3.04", + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", "transitive": [ - "xerces:xercesImpl" + "org.apache.hive:hive-common" ] }, - "xmlenc:xmlenc": { - "locked": "0.52", + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" ] - } - }, - "testCompileClasspath": { - "aopalliance:aopalliance": { - "locked": "1.0", + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", "transitive": [ - "com.google.inject:guice" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" ] }, - "asm:asm": { - "locked": "3.1", + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", "transitive": [ - "com.sun.jersey:jersey-server", - "org.sonatype.sisu.inject:cglib" + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" ] }, - "com.fasterxml.jackson.core:jackson-annotations": { - "locked": "2.10.2", + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind" + "org.apache.hive:hive-common" ] }, - "com.fasterxml.jackson.core:jackson-core": { - "locked": "2.10.2", + "org.apache.orc:orc-core": { + "locked": "1.6.3", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind", - "org.apache.avro:avro", - "org.apache.iceberg:iceberg-core" + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.iceberg:iceberg-orc" ] }, - "com.fasterxml.jackson.core:jackson-databind": { - "locked": "2.10.2", + "org.apache.orc:orc-shims": { + "locked": "1.6.3", "transitive": [ - "org.apache.avro:avro", - "org.apache.iceberg:iceberg-core" + "org.apache.orc:orc-core" ] }, - "com.github.ben-manes.caffeine:caffeine": { - "locked": "2.7.0", + "org.apache.parquet:parquet-avro": { + "locked": "1.11.0", "transitive": [ - "org.apache.iceberg:iceberg-core" + "org.apache.iceberg:iceberg-parquet" ] }, - "com.github.stephenc.findbugs:findbugs-annotations": { - "locked": "1.3.9-1", + "org.apache.parquet:parquet-column": { + "locked": "1.11.0", "transitive": [ - "org.apache.iceberg:iceberg-api", - "org.apache.iceberg:iceberg-common", - "org.apache.iceberg:iceberg-core", - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet" + "org.apache.parquet:parquet-avro", + "org.apache.parquet:parquet-hadoop" ] }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.0", + "org.apache.parquet:parquet-common": { + "locked": "1.11.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.parquet:parquet-column", + "org.apache.parquet:parquet-encoding" ] }, - "com.google.code.gson:gson": { - "locked": "2.2.4", + "org.apache.parquet:parquet-encoding": { + "locked": "1.11.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.parquet:parquet-column" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.3", + "org.apache.parquet:parquet-format-structures": { + "locked": "1.11.0", "transitive": [ - "com.github.ben-manes.caffeine:caffeine" + "org.apache.parquet:parquet-avro", + "org.apache.parquet:parquet-common", + "org.apache.parquet:parquet-hadoop" ] }, - "com.google.guava:guava": { - "locked": "16.0.1", + "org.apache.parquet:parquet-hadoop": { + "locked": "1.11.0", "transitive": [ - "org.apache.curator:curator-client", - "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.parquet:parquet-avro" ] }, - "com.google.inject:guice": { - "locked": "3.0", + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", "transitive": [ - "com.sun.jersey.contribs:jersey-guice", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-serde" ] }, - "com.google.protobuf:protobuf-java": { - "locked": "2.5.0", + "org.apache.parquet:parquet-jackson": { + "locked": "1.11.0", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.parquet:parquet-hadoop" ] }, - "com.sun.jersey.contribs:jersey-guice": { - "locked": "1.9", + "org.apache.slider:slider-core": { + "locked": "0.90.2-incubating", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-llap-server" ] }, - "com.sun.jersey:jersey-client": { - "locked": "1.9", + "org.apache.tez:hadoop-shim": { + "locked": "0.9.1", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals" ] }, - "com.sun.jersey:jersey-core": { - "locked": "1.9", + "org.apache.tez:tez-api": { + "locked": "0.9.1", "transitive": [ - "com.sun.jersey:jersey-client", - "com.sun.jersey:jersey-json", - "com.sun.jersey:jersey-server", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, - "com.sun.jersey:jersey-json": { - "locked": "1.9", + "org.apache.tez:tez-common": { + "locked": "0.9.1", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "com.klarna:hiverunner", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, - "com.sun.jersey:jersey-server": { - "locked": "1.9", + "org.apache.tez:tez-dag": { + "locked": "0.9.1", "transitive": [ - "com.sun.jersey.contribs:jersey-guice", - "org.apache.hadoop:hadoop-yarn-common" + "com.klarna:hiverunner" ] }, - "com.sun.xml.bind:jaxb-impl": { - "locked": "2.2.3-1", + "org.apache.tez:tez-mapreduce": { + "locked": "0.9.1", "transitive": [ - "com.sun.jersey:jersey-json" + "com.klarna:hiverunner" ] }, - "commons-beanutils:commons-beanutils": { - "locked": "1.7.0", + "org.apache.tez:tez-runtime-internals": { + "locked": "0.9.1", "transitive": [ - "commons-digester:commons-digester" + "org.apache.tez:tez-dag" ] }, - "commons-beanutils:commons-beanutils-core": { - "locked": "1.8.0", + "org.apache.tez:tez-runtime-library": { + "locked": "0.9.1", "transitive": [ - "commons-configuration:commons-configuration" + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" ] }, - "commons-cli:commons-cli": { - "locked": "1.2", + "org.apache.thrift:libfb303": { + "locked": "0.9.3", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" ] }, - "commons-codec:commons-codec": { - "locked": "1.6", + "org.apache.thrift:libthrift": { + "locked": "0.9.3", "transitive": [ - "commons-httpclient:commons-httpclient", - "org.apache.hadoop:hadoop-auth", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "co.cask.tephra:tephra-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" ] }, - "commons-collections:commons-collections": { - "locked": "3.2.2", + "org.apache.twill:twill-api": { + "locked": "0.6.0-incubating", "transitive": [ - "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" ] }, - "commons-configuration:commons-configuration": { - "locked": "1.6", + "org.apache.twill:twill-common": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hadoop:hadoop-common" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-api", + "org.apache.twill:twill-zookeeper" ] }, - "commons-digester:commons-digester": { - "locked": "1.8", + "org.apache.twill:twill-core": { + "locked": "0.6.0-incubating", "transitive": [ - "commons-configuration:commons-configuration" + "co.cask.tephra:tephra-core" ] }, - "commons-httpclient:commons-httpclient": { - "locked": "3.1", + "org.apache.twill:twill-discovery-api": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hadoop:hadoop-common" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-core" ] }, - "commons-io:commons-io": { - "locked": "2.4", + "org.apache.twill:twill-discovery-core": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core" ] }, - "commons-lang:commons-lang": { - "locked": "2.6", + "org.apache.twill:twill-zookeeper": { + "locked": "0.6.0-incubating", "transitive": [ - "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, - "commons-logging:commons-logging": { - "locked": "1.1.3", + "org.apache.velocity:velocity": { + "locked": "1.5", "transitive": [ - "commons-beanutils:commons-beanutils", - "commons-beanutils:commons-beanutils-core", - "commons-configuration:commons-configuration", - "commons-digester:commons-digester", - "commons-httpclient:commons-httpclient", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hive:hive-vector-code-gen" ] }, - "commons-net:commons-net": { - "locked": "3.1", + "org.apache.yetus:audience-annotations": { + "locked": "0.11.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.parquet:parquet-common" ] }, - "commons-pool:commons-pool": { - "locked": "1.6", + "org.apache.zookeeper:zookeeper": { + "locked": "3.4.6", "transitive": [ - "org.apache.parquet:parquet-hadoop" + "org.apache.curator:apache-curator", + "org.apache.curator:curator-client", + "org.apache.curator:curator-framework", + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.slider:slider-core", + "org.apache.twill:twill-zookeeper" ] }, - "io.airlift:aircompressor": { - "locked": "0.15", + "org.apiguardian:apiguardian-api": { + "locked": "1.1.0", "transitive": [ - "org.apache.orc:orc-core" + "org.junit.jupiter:junit-jupiter-api", + "org.junit.jupiter:junit-jupiter-engine", + "org.junit.jupiter:junit-jupiter-params", + "org.junit.platform:junit-platform-commons", + "org.junit.platform:junit-platform-engine" ] }, - "io.netty:netty": { - "locked": "3.7.0.Final", + "org.checkerframework:checker-qual": { + "locked": "2.6.0", "transitive": [ - "org.apache.hadoop:hadoop-hdfs", - "org.apache.zookeeper:zookeeper" + "com.github.ben-manes.caffeine:caffeine" ] }, - "io.netty:netty-all": { - "locked": "4.0.23.Final", + "org.codehaus.groovy:groovy-all": { + "locked": "2.4.4", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive:hive-exec" ] }, - "javax.annotation:javax.annotation-api": { - "locked": "1.3.2", + "org.codehaus.jackson:jackson-core-asl": { + "locked": "1.9.13", "transitive": [ - "org.apache.parquet:parquet-format-structures" + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", + "org.codehaus.jackson:jackson-jaxrs", + "org.codehaus.jackson:jackson-mapper-asl", + "org.codehaus.jackson:jackson-xc" ] }, - "javax.inject:javax.inject": { - "locked": "1", + "org.codehaus.jackson:jackson-jaxrs": { + "locked": "1.9.13", "transitive": [ - "com.google.inject:guice", - "com.sun.jersey.contribs:jersey-guice" + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" ] }, - "javax.servlet:servlet-api": { - "locked": "2.5", + "org.codehaus.jackson:jackson-mapper-asl": { + "locked": "1.9.13", "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.slider:slider-core", + "org.codehaus.jackson:jackson-jaxrs", + "org.codehaus.jackson:jackson-xc" ] }, - "javax.xml.bind:jaxb-api": { - "locked": "2.2.11", + "org.codehaus.jackson:jackson-xc": { + "locked": "1.9.13", "transitive": [ - "com.sun.xml.bind:jaxb-impl", + "com.sun.jersey:jersey-json", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.orc:orc-core" + "org.apache.slider:slider-core" ] }, - "jline:jline": { - "locked": "0.9.94", + "org.codehaus.janino:commons-compiler": { + "locked": "2.7.6", "transitive": [ - "org.apache.zookeeper:zookeeper" + "org.apache.calcite:calcite-core", + "org.codehaus.janino:janino" ] }, - "junit:junit": { - "locked": "4.12" - }, - "log4j:log4j": { - "locked": "1.2.17", + "org.codehaus.janino:janino": { + "locked": "2.7.6", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.zookeeper:zookeeper" + "org.apache.calcite:calcite-core" ] }, - "org.apache.avro:avro": { - "locked": "1.9.2", + "org.codehaus.jettison:jettison": { + "locked": "1.1", "transitive": [ - "org.apache.iceberg:iceberg-core" + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, - "org.apache.commons:commons-compress": { - "locked": "1.19", + "org.datanucleus:datanucleus-api-jdo": { + "locked": "4.2.4", "transitive": [ - "org.apache.avro:avro", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hive:hive-metastore" ] }, - "org.apache.commons:commons-math3": { - "locked": "3.1.1", + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" ] }, - "org.apache.curator:curator-client": { - "locked": "2.7.1", + "org.datanucleus:datanucleus-rdbms": { + "locked": "4.1.19", "transitive": [ - "org.apache.curator:curator-framework", - "org.apache.hadoop:hadoop-common" + "org.apache.hive:hive-metastore" ] }, - "org.apache.curator:curator-framework": { - "locked": "2.7.1", + "org.datanucleus:javax.jdo": { + "locked": "3.2.0-m3", "transitive": [ - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hive:hive-metastore" ] }, - "org.apache.curator:curator-recipes": { - "locked": "2.7.1", + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hive:hive-common", + "org.apache.hive:hive-service" ] }, - "org.apache.directory.api:api-asn1-api": { - "locked": "1.0.0-M20", + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", "transitive": [ - "org.apache.directory.server:apacheds-kerberos-codec" + "org.apache.hive:hive-common" ] }, - "org.apache.directory.api:api-util": { - "locked": "1.0.0-M20", + "org.fusesource.leveldbjni:leveldbjni-all": { + "locked": "1.8", "transitive": [ - "org.apache.directory.server:apacheds-kerberos-codec" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, - "org.apache.directory.server:apacheds-i18n": { - "locked": "2.0.0-M15", + "org.hamcrest:hamcrest-core": { + "locked": "1.3", "transitive": [ - "org.apache.directory.server:apacheds-kerberos-codec" + "junit:junit", + "org.mockito:mockito-core" ] }, - "org.apache.directory.server:apacheds-kerberos-codec": { - "locked": "2.0.0-M15", + "org.jamon:jamon-runtime": { + "locked": "2.3.1", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service" ] }, - "org.apache.hadoop:hadoop-annotations": { - "locked": "2.7.3", + "org.jetbrains:annotations": { + "locked": "17.0.0", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.orc:orc-core" ] }, - "org.apache.hadoop:hadoop-auth": { - "locked": "2.7.3", + "org.jruby.jcodings:jcodings": { + "locked": "1.0.8", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hbase:hbase-client", + "org.jruby.joni:joni" ] }, - "org.apache.hadoop:hadoop-client": { - "locked": "2.7.3" - }, - "org.apache.hadoop:hadoop-common": { - "locked": "2.7.3", + "org.jruby.joni:joni": { + "locked": "2.1.2", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hbase:hbase-client" ] }, - "org.apache.hadoop:hadoop-hdfs": { - "locked": "2.7.3", + "org.junit.jupiter:junit-jupiter": { + "locked": "5.6.0", "transitive": [ - "org.apache.hadoop:hadoop-client" + "com.klarna:hiverunner" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-app": { - "locked": "2.7.3", + "org.junit.jupiter:junit-jupiter-api": { + "locked": "5.6.0", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.junit.jupiter:junit-jupiter", + "org.junit.jupiter:junit-jupiter-engine", + "org.junit.jupiter:junit-jupiter-params" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-common": { - "locked": "2.7.3", + "org.junit.jupiter:junit-jupiter-engine": { + "locked": "5.6.0", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.junit.jupiter:junit-jupiter" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-core": { - "locked": "2.7.3", + "org.junit.jupiter:junit-jupiter-params": { + "locked": "5.6.0", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.junit.jupiter:junit-jupiter" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { - "locked": "2.7.3", + "org.junit.platform:junit-platform-commons": { + "locked": "1.6.0", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.junit.jupiter:junit-jupiter-api", + "org.junit.platform:junit-platform-engine" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { - "locked": "2.7.3", + "org.junit.platform:junit-platform-engine": { + "locked": "1.6.0", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + "org.junit.jupiter:junit-jupiter-engine" ] }, - "org.apache.hadoop:hadoop-yarn-api": { - "locked": "2.7.3", + "org.mockito:mockito-core": { + "locked": "1.10.19" + }, + "org.objenesis:objenesis": { + "locked": "2.1", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "com.esotericsoftware.kryo:kryo", + "org.mockito:mockito-core" ] }, - "org.apache.hadoop:hadoop-yarn-client": { - "locked": "2.7.3", + "org.opentest4j:opentest4j": { + "locked": "1.2.0", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.junit.jupiter:junit-jupiter-api", + "org.junit.platform:junit-platform-engine" ] }, - "org.apache.hadoop:hadoop-yarn-common": { - "locked": "2.7.3", + "org.ow2.asm:asm-all": { + "locked": "5.0.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.twill:twill-core" ] }, - "org.apache.hadoop:hadoop-yarn-server-common": { - "locked": "2.7.3", + "org.pentaho:pentaho-aggdesigner-algorithm": { + "locked": "5.1.5-jhyde", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.calcite:calcite-core" ] }, - "org.apache.hadoop:hadoop-yarn-server-nodemanager": { - "locked": "2.7.3", + "org.reflections:reflections": { + "locked": "0.9.8", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "com.klarna:hiverunner" ] }, - "org.apache.htrace:htrace-core": { - "locked": "3.1.0-incubating", + "org.roaringbitmap:RoaringBitmap": { + "locked": "0.4.9", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" - ] - }, - "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", - "transitive": [ - "org.apache.hadoop:hadoop-auth" - ] - }, - "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", - "transitive": [ - "org.apache.httpcomponents:httpclient" - ] - }, - "org.apache.iceberg:iceberg-api": { - "project": true, - "transitive": [ - "org.apache.iceberg:iceberg-core", - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet" - ] - }, - "org.apache.iceberg:iceberg-bundled-guava": { - "project": true, - "transitive": [ - "org.apache.iceberg:iceberg-api", - "org.apache.iceberg:iceberg-common" - ] - }, - "org.apache.iceberg:iceberg-common": { - "project": true, - "transitive": [ - "org.apache.iceberg:iceberg-core" - ] - }, - "org.apache.iceberg:iceberg-core": { - "project": true, - "transitive": [ - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet" - ] - }, - "org.apache.iceberg:iceberg-data": { - "project": true - }, - "org.apache.iceberg:iceberg-orc": { - "project": true - }, - "org.apache.iceberg:iceberg-parquet": { - "project": true - }, - "org.apache.orc:orc-core": { - "locked": "1.6.3", - "transitive": [ - "org.apache.iceberg:iceberg-orc" - ] - }, - "org.apache.orc:orc-shims": { - "locked": "1.6.3", - "transitive": [ - "org.apache.orc:orc-core" - ] - }, - "org.apache.parquet:parquet-avro": { - "locked": "1.11.0", - "transitive": [ - "org.apache.iceberg:iceberg-parquet" - ] - }, - "org.apache.parquet:parquet-column": { - "locked": "1.11.0", - "transitive": [ - "org.apache.parquet:parquet-avro", - "org.apache.parquet:parquet-hadoop" - ] - }, - "org.apache.parquet:parquet-common": { - "locked": "1.11.0", - "transitive": [ - "org.apache.parquet:parquet-column", - "org.apache.parquet:parquet-encoding" - ] - }, - "org.apache.parquet:parquet-encoding": { - "locked": "1.11.0", - "transitive": [ - "org.apache.parquet:parquet-column" - ] - }, - "org.apache.parquet:parquet-format-structures": { - "locked": "1.11.0", - "transitive": [ - "org.apache.parquet:parquet-avro", - "org.apache.parquet:parquet-common", - "org.apache.parquet:parquet-hadoop" - ] - }, - "org.apache.parquet:parquet-hadoop": { - "locked": "1.11.0", - "transitive": [ - "org.apache.parquet:parquet-avro" - ] - }, - "org.apache.parquet:parquet-jackson": { - "locked": "1.11.0", - "transitive": [ - "org.apache.parquet:parquet-hadoop" - ] - }, - "org.apache.yetus:audience-annotations": { - "locked": "0.11.0", - "transitive": [ - "org.apache.parquet:parquet-common" - ] - }, - "org.apache.zookeeper:zookeeper": { - "locked": "3.4.6", - "transitive": [ - "org.apache.curator:curator-client", - "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" - ] - }, - "org.checkerframework:checker-qual": { - "locked": "2.6.0", - "transitive": [ - "com.github.ben-manes.caffeine:caffeine" - ] - }, - "org.codehaus.jackson:jackson-core-asl": { - "locked": "1.9.13", - "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.codehaus.jackson:jackson-jaxrs", - "org.codehaus.jackson:jackson-mapper-asl", - "org.codehaus.jackson:jackson-xc" - ] - }, - "org.codehaus.jackson:jackson-jaxrs": { - "locked": "1.9.13", - "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" - ] - }, - "org.codehaus.jackson:jackson-mapper-asl": { - "locked": "1.9.13", - "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.codehaus.jackson:jackson-jaxrs", - "org.codehaus.jackson:jackson-xc" - ] - }, - "org.codehaus.jackson:jackson-xc": { - "locked": "1.9.13", - "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" - ] - }, - "org.codehaus.jettison:jettison": { - "locked": "1.1", - "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" - ] - }, - "org.fusesource.leveldbjni:leveldbjni-all": { - "locked": "1.8", - "transitive": [ - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" - ] - }, - "org.hamcrest:hamcrest-core": { - "locked": "1.3", - "transitive": [ - "junit:junit" - ] - }, - "org.jetbrains:annotations": { - "locked": "17.0.0", - "transitive": [ - "org.apache.orc:orc-core" + "org.apache.tez:tez-runtime-library" ] }, - "org.mockito:mockito-core": { - "locked": "1.10.19" - }, "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "com.jolbox:bonecp", + "com.ning:async-http-client", + "com.yammer.metrics:metrics-core", + "com.zaxxer:HikariCP", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.calcite.avatica:avatica-metrics", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", "org.apache.directory.api:api-util", @@ -3739,16 +5286,49 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.slider:slider-core", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.thrift:libthrift", + "org.apache.twill:twill-common", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -3774,6 +5354,35 @@ "org.apache.parquet:parquet-hadoop" ] }, + "oro:oro": { + "locked": "2.0.8", + "transitive": [ + "org.apache.velocity:velocity" + ] + }, + "stax:stax-api": { + "locked": "1.0.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.codehaus.jettison:jettison" + ] + }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -3783,6 +5392,7 @@ "xml-apis:xml-apis": { "locked": "1.3.04", "transitive": [ + "dom4j:dom4j", "xerces:xercesImpl" ] }, @@ -3794,7 +5404,13 @@ ] } }, - "testRuntime": { + "testCompileClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -3804,14 +5420,83 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "ch.qos.logback:logback-classic": { + "locked": "1.0.9", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "ch.qos.logback:logback-core": { + "locked": "1.0.9", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "co.cask.tephra:tephra-api": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-core": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-hbase-compat-1.0": { + "locked": "0.6.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.beust:jcommander": { + "locked": "1.30", + "transitive": [ + "org.apache.slider:slider-core" + ] + }, + "com.esotericsoftware.kryo:kryo": { + "locked": "2.24.0", + "requested": "2.24.0" + }, + "com.esotericsoftware.minlog:minlog": { + "locked": "1.2", + "transitive": [ + "com.esotericsoftware.kryo:kryo" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", + "requested": "2.6.5", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind" + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.calcite.avatica:avatica" ] }, "com.fasterxml.jackson.core:jackson-core": { @@ -3819,13 +5504,17 @@ "transitive": [ "com.fasterxml.jackson.core:jackson-databind", "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", "org.apache.iceberg:iceberg-core" ] }, "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -3835,9 +5524,23 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", @@ -3849,13 +5552,26 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.calcite:calcite-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals", + "org.apache.twill:twill-api", + "org.apache.twill:twill-common", + "org.apache.twill:twill-zookeeper" ] }, "com.google.code.gson:gson": { "locked": "2.2.4", "transitive": [ - "org.apache.hadoop:hadoop-common" + "co.cask.tephra:tephra-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, "com.google.errorprone:error_prone_annotations": { @@ -3865,8 +5581,14 @@ ] }, "com.google.guava:guava": { - "locked": "16.0.1", - "transitive": [ + "locked": "18.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.jolbox:bonecp", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", + "org.apache.calcite:calcite-linq4j", + "org.apache.curator:apache-curator", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", @@ -3876,20 +5598,60 @@ "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-vector-code-gen", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", + "org.reflections:reflections" + ] + }, + "com.google.inject.extensions:guice-assistedinject": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.slider:slider-core" ] }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "co.cask.tephra:tephra-core", + "com.google.inject.extensions:guice-assistedinject", + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, "com.google.protobuf:protobuf-java": { - "locked": "2.5.0", + "locked": "3.0.0-beta-1", "transitive": [ + "org.apache.calcite.avatica:avatica", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-app", @@ -3900,7 +5662,54 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-metastore", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "com.jolbox:bonecp": { + "locked": "0.8.0.RELEASE", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.klarna:hiverunner": { + "locked": "5.2.1", + "requested": "5.2.1" + }, + "com.lmax:disruptor": { + "locked": "3.3.0", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "com.ning:async-http-client": { + "locked": "1.8.16", + "transitive": [ + "org.apache.tez:tez-runtime-library" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -3914,7 +5723,9 @@ "locked": "1.9", "transitive": [ "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "com.sun.jersey:jersey-core": { @@ -3923,22 +5734,32 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-server" ] }, "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "com.sun.jersey:jersey-server": { "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" ] }, "com.sun.xml.bind:jaxb-impl": { @@ -3947,6 +5768,26 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server" + ] + }, + "com.yammer.metrics:metrics-core": { + "locked": "2.2.0", + "transitive": [ + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" + ] + }, + "com.zaxxer:HikariCP": { + "locked": "2.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -3965,26 +5806,47 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.tez:tez-dag" ] }, "commons-codec:commons-codec": { - "locked": "1.6", + "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.httpcomponents:httpclient", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-library" ] }, "commons-collections:commons-collections": { "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-mapreduce" ] }, "commons-configuration:commons-configuration": { @@ -3993,16 +5855,38 @@ "org.apache.hadoop:hadoop-common" ] }, + "commons-daemon:commons-daemon": { + "locked": "1.0.13", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "commons-dbcp:commons-dbcp": { + "locked": "1.4", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-metastore" + ] + }, "commons-digester:commons-digester": { "locked": "1.8", "transitive": [ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" ] }, "commons-io:commons-io": { @@ -4010,7 +5894,13 @@ "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, "commons-lang:commons-lang": { @@ -4022,17 +5912,41 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.velocity:velocity", + "org.pentaho:pentaho-aggdesigner-algorithm" ] }, "commons-logging:commons-logging": { - "locked": "1.1.3", + "locked": "1.2", "transitive": [ "commons-beanutils:commons-beanutils", "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", + "net.sf.jpam:jpam", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -4040,7 +5954,17 @@ "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.pentaho:pentaho-aggdesigner-algorithm" ] }, "commons-net:commons-net": { @@ -4052,26 +5976,87 @@ "commons-pool:commons-pool": { "locked": "1.6", "transitive": [ + "commons-dbcp:commons-dbcp", + "org.apache.hive:hive-metastore", "org.apache.parquet:parquet-hadoop" ] }, + "dom4j:dom4j": { + "locked": "1.6.1", + "transitive": [ + "org.reflections:reflections" + ] + }, "io.airlift:aircompressor": { "locked": "0.15", "transitive": [ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { - "locked": "3.7.0.Final", + "locked": "3.9.2.Final", "transitive": [ + "com.ning:async-http-client", "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", "org.apache.zookeeper:zookeeper" ] }, "io.netty:netty-all": { "locked": "4.0.23.Final", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server" + ] + }, + "it.unimi.dsi:fastutil": { + "locked": "6.5.6", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "jakarta.jms:jakarta.jms-api": { + "locked": "2.0.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions" + ] + }, + "javassist:javassist": { + "locked": "3.12.1.GA", + "transitive": [ + "org.reflections:reflections" + ] + }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" ] }, "javax.annotation:javax.annotation-api": { @@ -4087,17 +6072,49 @@ "com.sun.jersey.contribs:jersey-guice" ] }, - "javax.servlet.jsp:jsp-api": { - "locked": "2.1", + "javax.jdo:jdo-api": { + "locked": "3.0.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hive:hive-metastore" + ] + }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" ] }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.slider:slider-core", + "org.apache.tez:tez-dag", + "org.eclipse.jetty.aggregate:jetty-all", + "tomcat:jasper-runtime" + ] + }, + "javax.transaction:jta": { + "locked": "1.1", + "transitive": [ + "javax.jdo:jdo-api" + ] + }, + "javax.transaction:transaction-api": { + "locked": "1.1", + "transitive": [ + "org.datanucleus:javax.jdo" ] }, "javax.xml.bind:jaxb-api": { @@ -4109,30 +6126,158 @@ "org.apache.orc:orc-core" ] }, + "javolution:javolution": { + "locked": "5.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-common" + ] + }, "junit:junit": { - "locked": "4.12" + "locked": "4.12", + "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server" + ] }, "log4j:log4j": { "locked": "1.2.17", "transitive": [ - "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", "org.apache.zookeeper:zookeeper" ] }, + "net.hydromatic:eigenbase-properties": { + "locked": "1.1.5", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.jpam:jpam": { + "locked": "1.1", + "transitive": [ + "org.apache.hive:hive-service" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.antlr:ST4": { + "locked": "4.0.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.antlr:antlr-runtime": { + "locked": "3.5.2", + "transitive": [ + "org.antlr:ST4", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ - "org.apache.iceberg:iceberg-core" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.iceberg:iceberg-core", + "org.apache.slider:slider-core" + ] + }, + "org.apache.calcite.avatica:avatica": { + "locked": "1.8.0", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.apache.calcite.avatica:avatica-metrics": { + "locked": "1.8.0", + "transitive": [ + "org.apache.calcite.avatica:avatica" + ] + }, + "org.apache.calcite:calcite-core": { + "locked": "1.10.0", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-exec" + ] + }, + "org.apache.calcite:calcite-druid": { + "locked": "1.10.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.calcite:calcite-linq4j": { + "locked": "1.10.0", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid" + ] + }, + "org.apache.commons:commons-collections4": { + "locked": "4.1", + "transitive": [ + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag" ] }, "org.apache.commons:commons-compress": { @@ -4140,33 +6285,78 @@ "transitive": [ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.2", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-tez" + ] + }, + "org.apache.commons:commons-math": { + "locked": "2.2", + "transitive": [ + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" ] }, "org.apache.commons:commons-math3": { "locked": "3.1.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.tez:tez-dag" + ] + }, + "org.apache.curator:apache-curator": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-client" ] }, "org.apache.curator:curator-client": { "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-framework", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.slider:slider-core" ] }, "org.apache.curator:curator-framework": { "locked": "2.7.1", "transitive": [ "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" ] }, "org.apache.curator:curator-recipes": { "locked": "2.7.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" + ] + }, + "org.apache.derby:derby": { + "locked": "10.10.2.0", + "transitive": [ + "org.apache.hive:hive-metastore" ] }, "org.apache.directory.api:api-asn1-api": { @@ -4193,32 +6383,96 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-archives": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core" ] }, "org.apache.hadoop:hadoop-auth": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-api", + "org.apache.tez:tez-runtime-library" ] }, "org.apache.hadoop:hadoop-client": { - "locked": "2.7.3" + "locked": "2.7.3", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] }, "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, "org.apache.hadoop:hadoop-hdfs": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.slider:slider-core" ] }, "org.apache.hadoop:hadoop-mapreduce-client-app": { @@ -4232,14 +6486,22 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-app", "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.tez:tez-mapreduce" ] }, "org.apache.hadoop:hadoop-mapreduce-client-core": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:tez-mapreduce" ] }, "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { @@ -4261,14 +6523,26 @@ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, "org.apache.hadoop:hadoop-yarn-client": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" ] }, "org.apache.hadoop:hadoop-yarn-common": { @@ -4277,8 +6551,22 @@ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-registry", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-yarn-registry": { + "locked": "2.7.1", + "transitive": [ + "org.apache.slider:slider-core" ] }, "org.apache.hadoop:hadoop-yarn-server-common": { @@ -4286,7 +6574,8 @@ "transitive": [ "org.apache.hadoop:hadoop-mapreduce-client-common", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, "org.apache.hadoop:hadoop-yarn-server-nodemanager": { @@ -4295,23 +6584,243 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.0", + "transitive": [ + "org.apache.tez:tez-dag" + ] + }, + "org.apache.hbase:hbase-annotations": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-protocol" + ] + }, + "org.apache.hbase:hbase-client": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hbase:hbase-common": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hbase:hbase-hadoop-compat": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hbase:hbase-hadoop2-compat": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hbase:hbase-procedure": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.hbase:hbase-protocol": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.hbase:hbase-server": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hive.hcatalog:hive-hcatalog-core": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client" + ] + }, + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-webhcat-java-client" + ] + }, + "org.apache.hive.hcatalog:hive-webhcat-java-client": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-cli": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-exec": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-jdbc": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.hive:hive-llap-client": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez" + ] + }, + "org.apache.hive:hive-llap-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hive:hive-llap-server": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-service" + ] + }, + "org.apache.hive:hive-llap-tez": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hive:hive-metastore": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-service" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hive:hive-service": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc" + ] + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server" ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.5.2", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hadoop:hadoop-auth", + "org.apache.hive:hive-jdbc", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.4", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-jdbc", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -4344,762 +6853,4378 @@ "org.apache.iceberg:iceberg-parquet" ] }, - "org.apache.iceberg:iceberg-data": { - "project": true - }, - "org.apache.iceberg:iceberg-orc": { - "project": true - }, - "org.apache.iceberg:iceberg-parquet": { - "project": true + "org.apache.iceberg:iceberg-data": { + "project": true + }, + "org.apache.iceberg:iceberg-orc": { + "project": true + }, + "org.apache.iceberg:iceberg-parquet": { + "project": true + }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.orc:orc-core": { + "locked": "1.6.3", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.iceberg:iceberg-orc" + ] + }, + "org.apache.orc:orc-shims": { + "locked": "1.6.3", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "org.apache.parquet:parquet-avro": { + "locked": "1.11.0", + "transitive": [ + "org.apache.iceberg:iceberg-parquet" + ] + }, + "org.apache.parquet:parquet-column": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-avro", + "org.apache.parquet:parquet-hadoop" + ] + }, + "org.apache.parquet:parquet-common": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-column", + "org.apache.parquet:parquet-encoding" + ] + }, + "org.apache.parquet:parquet-encoding": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-column" + ] + }, + "org.apache.parquet:parquet-format-structures": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-avro", + "org.apache.parquet:parquet-common", + "org.apache.parquet:parquet-hadoop" + ] + }, + "org.apache.parquet:parquet-hadoop": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-avro" + ] + }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.parquet:parquet-jackson": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-hadoop" + ] + }, + "org.apache.slider:slider-core": { + "locked": "0.90.2-incubating", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.tez:hadoop-shim": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals" + ] + }, + "org.apache.tez:tez-api": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.tez:tez-common": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.tez:tez-dag": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.tez:tez-mapreduce": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.tez:tez-runtime-internals": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-dag" + ] + }, + "org.apache.tez:tez-runtime-library": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" + ] + }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, + "org.apache.twill:twill-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-common": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-api", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "org.apache.twill:twill-discovery-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.twill:twill-discovery-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core" + ] + }, + "org.apache.twill:twill-zookeeper": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.velocity:velocity": { + "locked": "1.5", + "transitive": [ + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.yetus:audience-annotations": { + "locked": "0.11.0", + "transitive": [ + "org.apache.parquet:parquet-common" + ] + }, + "org.apache.zookeeper:zookeeper": { + "locked": "3.4.6", + "transitive": [ + "org.apache.curator:apache-curator", + "org.apache.curator:curator-client", + "org.apache.curator:curator-framework", + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.slider:slider-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apiguardian:apiguardian-api": { + "locked": "1.1.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.jupiter:junit-jupiter-params", + "org.junit.platform:junit-platform-commons" + ] + }, + "org.checkerframework:checker-qual": { + "locked": "2.6.0", + "transitive": [ + "com.github.ben-manes.caffeine:caffeine" + ] + }, + "org.codehaus.groovy:groovy-all": { + "locked": "2.4.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.codehaus.jackson:jackson-core-asl": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", + "org.codehaus.jackson:jackson-jaxrs", + "org.codehaus.jackson:jackson-mapper-asl", + "org.codehaus.jackson:jackson-xc" + ] + }, + "org.codehaus.jackson:jackson-jaxrs": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] + }, + "org.codehaus.jackson:jackson-mapper-asl": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.slider:slider-core", + "org.codehaus.jackson:jackson-jaxrs", + "org.codehaus.jackson:jackson-xc" + ] + }, + "org.codehaus.jackson:jackson-xc": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.slider:slider-core" + ] + }, + "org.codehaus.janino:commons-compiler": { + "locked": "2.7.6", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.codehaus.janino:janino" + ] + }, + "org.codehaus.janino:janino": { + "locked": "2.7.6", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.codehaus.jettison:jettison": { + "locked": "1.1", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-server-nodemanager" + ] + }, + "org.datanucleus:datanucleus-api-jdo": { + "locked": "4.2.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-rdbms": { + "locked": "4.1.19", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:javax.jdo": { + "locked": "3.2.0-m3", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-service" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.fusesource.leveldbjni:leveldbjni-all": { + "locked": "1.8", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager" + ] + }, + "org.hamcrest:hamcrest-core": { + "locked": "1.3", + "transitive": [ + "junit:junit" + ] + }, + "org.jamon:jamon-runtime": { + "locked": "2.3.1", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service" + ] + }, + "org.jetbrains:annotations": { + "locked": "17.0.0", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "org.jruby.jcodings:jcodings": { + "locked": "1.0.8", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.jruby.joni:joni" + ] + }, + "org.jruby.joni:joni": { + "locked": "2.1.2", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.junit.jupiter:junit-jupiter": { + "locked": "5.6.0", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.junit.jupiter:junit-jupiter-api": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter", + "org.junit.jupiter:junit-jupiter-params" + ] + }, + "org.junit.jupiter:junit-jupiter-params": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter" + ] + }, + "org.junit.platform:junit-platform-commons": { + "locked": "1.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api" + ] + }, + "org.mockito:mockito-core": { + "locked": "1.10.19" + }, + "org.objenesis:objenesis": { + "locked": "2.1", + "transitive": [ + "com.esotericsoftware.kryo:kryo" + ] + }, + "org.opentest4j:opentest4j": { + "locked": "1.2.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api" + ] + }, + "org.ow2.asm:asm-all": { + "locked": "5.0.2", + "transitive": [ + "org.apache.twill:twill-core" + ] + }, + "org.pentaho:pentaho-aggdesigner-algorithm": { + "locked": "5.1.5-jhyde", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.reflections:reflections": { + "locked": "0.9.8", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.roaringbitmap:RoaringBitmap": { + "locked": "0.4.9", + "transitive": [ + "org.apache.tez:tez-runtime-library" + ] + }, + "org.slf4j:slf4j-api": { + "locked": "1.7.25", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "com.jolbox:bonecp", + "com.ning:async-http-client", + "com.yammer.metrics:metrics-core", + "com.zaxxer:HikariCP", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.calcite.avatica:avatica-metrics", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", + "org.apache.curator:curator-client", + "org.apache.directory.api:api-asn1-api", + "org.apache.directory.api:api-util", + "org.apache.directory.server:apacheds-i18n", + "org.apache.directory.server:apacheds-kerberos-codec", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", + "org.apache.iceberg:iceberg-api", + "org.apache.iceberg:iceberg-common", + "org.apache.iceberg:iceberg-core", + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.orc:orc-core", + "org.apache.orc:orc-shims", + "org.apache.parquet:parquet-common", + "org.apache.parquet:parquet-format-structures", + "org.apache.slider:slider-core", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.thrift:libthrift", + "org.apache.twill:twill-common", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", + "org.apache.zookeeper:zookeeper", + "org.slf4j:slf4j-simple" + ] + }, + "org.slf4j:slf4j-simple": { + "locked": "1.7.25" + }, + "org.sonatype.sisu.inject:cglib": { + "locked": "2.2.1-v20090111", + "transitive": [ + "com.google.inject:guice" + ] + }, + "org.threeten:threeten-extra": { + "locked": "1.5.0", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "org.xerial.snappy:snappy-java": { + "locked": "1.1.7.3", + "transitive": [ + "org.apache.parquet:parquet-hadoop" + ] + }, + "oro:oro": { + "locked": "2.0.8", + "transitive": [ + "org.apache.velocity:velocity" + ] + }, + "stax:stax-api": { + "locked": "1.0.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.codehaus.jettison:jettison" + ] + }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "xerces:xercesImpl": { + "locked": "2.9.1", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "xml-apis:xml-apis": { + "locked": "1.3.04", + "transitive": [ + "dom4j:dom4j", + "xerces:xercesImpl" + ] + }, + "xmlenc:xmlenc": { + "locked": "0.52", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs" + ] + } + }, + "testRuntime": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, + "aopalliance:aopalliance": { + "locked": "1.0", + "transitive": [ + "com.google.inject:guice" + ] + }, + "asm:asm": { + "locked": "3.1", + "transitive": [ + "asm:asm-tree", + "com.sun.jersey:jersey-server", + "org.sonatype.sisu.inject:cglib" + ] + }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "ch.qos.logback:logback-classic": { + "locked": "1.0.9", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "ch.qos.logback:logback-core": { + "locked": "1.0.9", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "co.cask.tephra:tephra-api": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-core": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-hbase-compat-1.0": { + "locked": "0.6.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.beust:jcommander": { + "locked": "1.30", + "transitive": [ + "org.apache.slider:slider-core" + ] + }, + "com.esotericsoftware.kryo:kryo": { + "locked": "2.24.0", + "requested": "2.24.0" + }, + "com.esotericsoftware.minlog:minlog": { + "locked": "1.2", + "transitive": [ + "com.esotericsoftware.kryo:kryo" + ] + }, + "com.fasterxml.jackson.core:jackson-annotations": { + "locked": "2.10.2", + "requested": "2.6.5", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.calcite.avatica:avatica" + ] + }, + "com.fasterxml.jackson.core:jackson-core": { + "locked": "2.10.2", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.iceberg:iceberg-core" + ] + }, + "com.fasterxml.jackson.core:jackson-databind": { + "locked": "2.10.2", + "transitive": [ + "io.dropwizard.metrics:metrics-json", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-common", + "org.apache.iceberg:iceberg-core" + ] + }, + "com.github.ben-manes.caffeine:caffeine": { + "locked": "2.7.0", + "transitive": [ + "org.apache.iceberg:iceberg-core" + ] + }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "com.github.stephenc.findbugs:findbugs-annotations": { + "locked": "1.3.9-1", + "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.iceberg:iceberg-api", + "org.apache.iceberg:iceberg-common", + "org.apache.iceberg:iceberg-core", + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.0", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals", + "org.apache.twill:twill-api", + "org.apache.twill:twill-common", + "org.apache.twill:twill-zookeeper" + ] + }, + "com.google.code.gson:gson": { + "locked": "2.2.4", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.github.ben-manes.caffeine:caffeine" + ] + }, + "com.google.guava:guava": { + "locked": "18.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.jolbox:bonecp", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", + "org.apache.calcite:calcite-linq4j", + "org.apache.curator:apache-curator", + "org.apache.curator:curator-client", + "org.apache.curator:curator-framework", + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-vector-code-gen", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", + "org.reflections:reflections" + ] + }, + "com.google.inject.extensions:guice-assistedinject": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core" + ] + }, + "com.google.inject:guice": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.google.inject.extensions:guice-assistedinject", + "com.google.inject.extensions:guice-servlet", + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.0.0-beta-1", + "transitive": [ + "org.apache.calcite.avatica:avatica", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-metastore", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "com.jolbox:bonecp": { + "locked": "0.8.0.RELEASE", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.klarna:hiverunner": { + "locked": "5.2.1", + "requested": "5.2.1" + }, + "com.lmax:disruptor": { + "locked": "3.3.0", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "com.ning:async-http-client": { + "locked": "1.8.16", + "transitive": [ + "org.apache.tez:tez-runtime-library" + ] + }, + "com.sun.jersey.contribs:jersey-guice": { + "locked": "1.9", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "com.sun.jersey:jersey-client": { + "locked": "1.9", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "com.sun.jersey:jersey-core": { + "locked": "1.9", + "transitive": [ + "com.sun.jersey:jersey-client", + "com.sun.jersey:jersey-json", + "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-server" + ] + }, + "com.sun.jersey:jersey-json": { + "locked": "1.9", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "com.sun.jersey:jersey-server": { + "locked": "1.9", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] + }, + "com.sun.xml.bind:jaxb-impl": { + "locked": "2.2.3-1", + "transitive": [ + "com.sun.jersey:jersey-json" + ] + }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server" + ] + }, + "com.yammer.metrics:metrics-core": { + "locked": "2.2.0", + "transitive": [ + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" + ] + }, + "com.zaxxer:HikariCP": { + "locked": "2.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "commons-beanutils:commons-beanutils": { + "locked": "1.7.0", + "transitive": [ + "commons-digester:commons-digester" + ] + }, + "commons-beanutils:commons-beanutils-core": { + "locked": "1.8.0", + "transitive": [ + "commons-configuration:commons-configuration" + ] + }, + "commons-cli:commons-cli": { + "locked": "1.2", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.tez:tez-dag" + ] + }, + "commons-codec:commons-codec": { + "locked": "1.9", + "transitive": [ + "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.httpcomponents:httpclient", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-library" + ] + }, + "commons-collections:commons-collections": { + "locked": "3.2.2", + "transitive": [ + "commons-configuration:commons-configuration", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-mapreduce" + ] + }, + "commons-configuration:commons-configuration": { + "locked": "1.6", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "commons-daemon:commons-daemon": { + "locked": "1.0.13", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "commons-dbcp:commons-dbcp": { + "locked": "1.4", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-metastore" + ] + }, + "commons-digester:commons-digester": { + "locked": "1.8", + "transitive": [ + "commons-configuration:commons-configuration" + ] + }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, + "commons-httpclient:commons-httpclient": { + "locked": "3.1", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" + ] + }, + "commons-io:commons-io": { + "locked": "2.4", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "commons-lang:commons-lang": { + "locked": "2.6", + "transitive": [ + "commons-configuration:commons-configuration", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.velocity:velocity", + "org.pentaho:pentaho-aggdesigner-algorithm" + ] + }, + "commons-logging:commons-logging": { + "locked": "1.2", + "transitive": [ + "commons-beanutils:commons-beanutils", + "commons-beanutils:commons-beanutils-core", + "commons-configuration:commons-configuration", + "commons-digester:commons-digester", + "commons-el:commons-el", + "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", + "net.sf.jpam:jpam", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.pentaho:pentaho-aggdesigner-algorithm" + ] + }, + "commons-net:commons-net": { + "locked": "3.1", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "commons-pool:commons-pool": { + "locked": "1.6", + "transitive": [ + "commons-dbcp:commons-dbcp", + "org.apache.hive:hive-metastore", + "org.apache.parquet:parquet-hadoop" + ] + }, + "dom4j:dom4j": { + "locked": "1.6.1", + "transitive": [ + "org.reflections:reflections" + ] + }, + "io.airlift:aircompressor": { + "locked": "0.15", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.netty:netty": { + "locked": "3.9.2.Final", + "transitive": [ + "com.ning:async-http-client", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", + "org.apache.zookeeper:zookeeper" + ] + }, + "io.netty:netty-all": { + "locked": "4.0.23.Final", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server" + ] + }, + "it.unimi.dsi:fastutil": { + "locked": "6.5.6", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "jakarta.jms:jakarta.jms-api": { + "locked": "2.0.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions" + ] + }, + "javassist:javassist": { + "locked": "3.12.1.GA", + "transitive": [ + "org.reflections:reflections" + ] + }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.annotation:javax.annotation-api": { + "locked": "1.3.2", + "transitive": [ + "org.apache.parquet:parquet-format-structures" + ] + }, + "javax.inject:javax.inject": { + "locked": "1", + "transitive": [ + "com.google.inject:guice", + "com.sun.jersey.contribs:jersey-guice" + ] + }, + "javax.jdo:jdo-api": { + "locked": "3.0.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet.jsp:jsp-api": { + "locked": "2.1", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.slider:slider-core" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, + "javax.servlet:servlet-api": { + "locked": "2.5", + "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.slider:slider-core", + "org.apache.tez:tez-dag", + "org.eclipse.jetty.aggregate:jetty-all", + "tomcat:jasper-runtime" + ] + }, + "javax.transaction:jta": { + "locked": "1.1", + "transitive": [ + "javax.jdo:jdo-api" + ] + }, + "javax.transaction:transaction-api": { + "locked": "1.1", + "transitive": [ + "org.datanucleus:javax.jdo" + ] + }, + "javax.xml.bind:jaxb-api": { + "locked": "2.2.11", + "transitive": [ + "com.sun.xml.bind:jaxb-impl", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" + ] + }, + "javolution:javolution": { + "locked": "5.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "jline:jline": { + "locked": "2.12", + "transitive": [ + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.zookeeper:zookeeper" + ] + }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-common" + ] + }, + "junit:junit": { + "locked": "4.12", + "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server" + ] + }, + "log4j:log4j": { + "locked": "1.2.17", + "transitive": [ + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", + "org.apache.zookeeper:zookeeper" + ] + }, + "net.hydromatic:eigenbase-properties": { + "locked": "1.1.5", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.jpam:jpam": { + "locked": "1.1", + "transitive": [ + "org.apache.hive:hive-service" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.antlr:ST4": { + "locked": "4.0.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.antlr:antlr-runtime": { + "locked": "3.5.2", + "transitive": [ + "org.antlr:ST4", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, + "org.apache.avro:avro": { + "locked": "1.9.2", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.iceberg:iceberg-core", + "org.apache.slider:slider-core" + ] + }, + "org.apache.calcite.avatica:avatica": { + "locked": "1.8.0", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.apache.calcite.avatica:avatica-metrics": { + "locked": "1.8.0", + "transitive": [ + "org.apache.calcite.avatica:avatica" + ] + }, + "org.apache.calcite:calcite-core": { + "locked": "1.10.0", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-exec" + ] + }, + "org.apache.calcite:calcite-druid": { + "locked": "1.10.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.calcite:calcite-linq4j": { + "locked": "1.10.0", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid" + ] + }, + "org.apache.commons:commons-collections4": { + "locked": "4.1", + "transitive": [ + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag" + ] + }, + "org.apache.commons:commons-compress": { + "locked": "1.19", + "transitive": [ + "org.apache.avro:avro", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" + ] + }, + "org.apache.commons:commons-lang3": { + "locked": "3.2", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-tez" + ] + }, + "org.apache.commons:commons-math": { + "locked": "2.2", + "transitive": [ + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.commons:commons-math3": { + "locked": "3.1.1", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.tez:tez-dag" + ] + }, + "org.apache.curator:apache-curator": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-client" + ] + }, + "org.apache.curator:curator-client": { + "locked": "2.7.1", + "transitive": [ + "org.apache.curator:curator-framework", + "org.apache.hadoop:hadoop-common", + "org.apache.slider:slider-core" + ] + }, + "org.apache.curator:curator-framework": { + "locked": "2.7.1", + "transitive": [ + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" + ] + }, + "org.apache.curator:curator-recipes": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" + ] + }, + "org.apache.derby:derby": { + "locked": "10.10.2.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.directory.api:api-asn1-api": { + "locked": "1.0.0-M20", + "transitive": [ + "org.apache.directory.server:apacheds-kerberos-codec" + ] + }, + "org.apache.directory.api:api-util": { + "locked": "1.0.0-M20", + "transitive": [ + "org.apache.directory.server:apacheds-kerberos-codec" + ] + }, + "org.apache.directory.server:apacheds-i18n": { + "locked": "2.0.0-M15", + "transitive": [ + "org.apache.directory.server:apacheds-kerberos-codec" + ] + }, + "org.apache.directory.server:apacheds-kerberos-codec": { + "locked": "2.0.0-M15", + "transitive": [ + "org.apache.hadoop:hadoop-auth" + ] + }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.hadoop:hadoop-annotations": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-archives": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core" + ] + }, + "org.apache.hadoop:hadoop-auth": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-api", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-client": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] + }, + "org.apache.hadoop:hadoop-common": { + "locked": "2.7.3", + "transitive": [ + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-hdfs": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-app": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.tez:tez-mapreduce" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-core": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:tez-mapreduce" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + ] + }, + "org.apache.hadoop:hadoop-yarn-api": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-yarn-client": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" + ] + }, + "org.apache.hadoop:hadoop-yarn-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.hadoop:hadoop-yarn-registry": { + "locked": "2.7.1", + "transitive": [ + "org.apache.slider:slider-core" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-nodemanager": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.tez:tez-dag" + ] + }, + "org.apache.hbase:hbase-annotations": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-protocol" + ] + }, + "org.apache.hbase:hbase-client": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hbase:hbase-common": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hbase:hbase-hadoop-compat": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hbase:hbase-hadoop2-compat": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hbase:hbase-prefix-tree": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.hbase:hbase-procedure": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.hbase:hbase-protocol": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.hbase:hbase-server": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hive.hcatalog:hive-hcatalog-core": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client" + ] + }, + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-webhcat-java-client" + ] + }, + "org.apache.hive.hcatalog:hive-webhcat-java-client": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-cli": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-exec": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-jdbc": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.hive:hive-llap-client": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez" + ] + }, + "org.apache.hive:hive-llap-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hive:hive-llap-server": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-service" + ] + }, + "org.apache.hive:hive-llap-tez": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.hive:hive-metastore": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-service" + ] + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hive:hive-service": { + "locked": "2.3.7", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc" + ] + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.htrace:htrace-core": { + "locked": "3.1.0-incubating", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.5.2", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hadoop:hadoop-auth", + "org.apache.hive:hive-jdbc", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.httpcomponents:httpcore": { + "locked": "4.4.4", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-jdbc", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.iceberg:iceberg-api": { + "project": true, + "transitive": [ + "org.apache.iceberg:iceberg-core", + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet" + ] + }, + "org.apache.iceberg:iceberg-bundled-guava": { + "project": true, + "transitive": [ + "org.apache.iceberg:iceberg-api", + "org.apache.iceberg:iceberg-common" + ] + }, + "org.apache.iceberg:iceberg-common": { + "project": true, + "transitive": [ + "org.apache.iceberg:iceberg-core" + ] + }, + "org.apache.iceberg:iceberg-core": { + "project": true, + "transitive": [ + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet" + ] + }, + "org.apache.iceberg:iceberg-data": { + "project": true + }, + "org.apache.iceberg:iceberg-orc": { + "project": true + }, + "org.apache.iceberg:iceberg-parquet": { + "project": true + }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.orc:orc-core": { + "locked": "1.6.3", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.iceberg:iceberg-orc" + ] + }, + "org.apache.orc:orc-shims": { + "locked": "1.6.3", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "org.apache.parquet:parquet-avro": { + "locked": "1.11.0", + "transitive": [ + "org.apache.iceberg:iceberg-parquet" + ] + }, + "org.apache.parquet:parquet-column": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-avro", + "org.apache.parquet:parquet-hadoop" + ] + }, + "org.apache.parquet:parquet-common": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-column", + "org.apache.parquet:parquet-encoding" + ] + }, + "org.apache.parquet:parquet-encoding": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-column" + ] + }, + "org.apache.parquet:parquet-format-structures": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-avro", + "org.apache.parquet:parquet-common", + "org.apache.parquet:parquet-hadoop" + ] + }, + "org.apache.parquet:parquet-hadoop": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-avro" + ] + }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.parquet:parquet-jackson": { + "locked": "1.11.0", + "transitive": [ + "org.apache.parquet:parquet-hadoop" + ] + }, + "org.apache.slider:slider-core": { + "locked": "0.90.2-incubating", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.tez:hadoop-shim": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals" + ] + }, + "org.apache.tez:tez-api": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.tez:tez-common": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.tez:tez-dag": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.tez:tez-mapreduce": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.tez:tez-runtime-internals": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-dag" + ] + }, + "org.apache.tez:tez-runtime-library": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" + ] + }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, + "org.apache.twill:twill-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-common": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-api", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "org.apache.twill:twill-discovery-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.twill:twill-discovery-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core" + ] + }, + "org.apache.twill:twill-zookeeper": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.velocity:velocity": { + "locked": "1.5", + "transitive": [ + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.yetus:audience-annotations": { + "locked": "0.11.0", + "transitive": [ + "org.apache.parquet:parquet-common" + ] + }, + "org.apache.zookeeper:zookeeper": { + "locked": "3.4.6", + "transitive": [ + "org.apache.curator:apache-curator", + "org.apache.curator:curator-client", + "org.apache.curator:curator-framework", + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.slider:slider-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apiguardian:apiguardian-api": { + "locked": "1.1.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.jupiter:junit-jupiter-engine", + "org.junit.jupiter:junit-jupiter-params", + "org.junit.platform:junit-platform-commons", + "org.junit.platform:junit-platform-engine" + ] + }, + "org.checkerframework:checker-qual": { + "locked": "2.6.0", + "transitive": [ + "com.github.ben-manes.caffeine:caffeine" + ] + }, + "org.codehaus.groovy:groovy-all": { + "locked": "2.4.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.codehaus.jackson:jackson-core-asl": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", + "org.codehaus.jackson:jackson-jaxrs", + "org.codehaus.jackson:jackson-mapper-asl", + "org.codehaus.jackson:jackson-xc" + ] + }, + "org.codehaus.jackson:jackson-jaxrs": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] + }, + "org.codehaus.jackson:jackson-mapper-asl": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.slider:slider-core", + "org.codehaus.jackson:jackson-jaxrs", + "org.codehaus.jackson:jackson-xc" + ] + }, + "org.codehaus.jackson:jackson-xc": { + "locked": "1.9.13", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.slider:slider-core" + ] + }, + "org.codehaus.janino:commons-compiler": { + "locked": "2.7.6", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.codehaus.janino:janino" + ] + }, + "org.codehaus.janino:janino": { + "locked": "2.7.6", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.codehaus.jettison:jettison": { + "locked": "1.1", + "transitive": [ + "com.sun.jersey:jersey-json", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.datanucleus:datanucleus-api-jdo": { + "locked": "4.2.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-rdbms": { + "locked": "4.1.19", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:javax.jdo": { + "locked": "3.2.0-m3", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-service" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.fusesource.leveldbjni:leveldbjni-all": { + "locked": "1.8", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.hamcrest:hamcrest-core": { + "locked": "1.3", + "transitive": [ + "junit:junit", + "org.mockito:mockito-core" + ] + }, + "org.jamon:jamon-runtime": { + "locked": "2.3.1", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service" + ] + }, + "org.jetbrains:annotations": { + "locked": "17.0.0", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "org.jruby.jcodings:jcodings": { + "locked": "1.0.8", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.jruby.joni:joni" + ] + }, + "org.jruby.joni:joni": { + "locked": "2.1.2", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.junit.jupiter:junit-jupiter": { + "locked": "5.6.0", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.junit.jupiter:junit-jupiter-api": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter", + "org.junit.jupiter:junit-jupiter-engine", + "org.junit.jupiter:junit-jupiter-params" + ] + }, + "org.junit.jupiter:junit-jupiter-engine": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter" + ] + }, + "org.junit.jupiter:junit-jupiter-params": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter" + ] + }, + "org.junit.platform:junit-platform-commons": { + "locked": "1.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.platform:junit-platform-engine" + ] + }, + "org.junit.platform:junit-platform-engine": { + "locked": "1.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-engine" + ] + }, + "org.mockito:mockito-core": { + "locked": "1.10.19" + }, + "org.objenesis:objenesis": { + "locked": "2.1", + "transitive": [ + "com.esotericsoftware.kryo:kryo", + "org.mockito:mockito-core" + ] + }, + "org.opentest4j:opentest4j": { + "locked": "1.2.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.platform:junit-platform-engine" + ] + }, + "org.ow2.asm:asm-all": { + "locked": "5.0.2", + "transitive": [ + "org.apache.twill:twill-core" + ] + }, + "org.pentaho:pentaho-aggdesigner-algorithm": { + "locked": "5.1.5-jhyde", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.reflections:reflections": { + "locked": "0.9.8", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.roaringbitmap:RoaringBitmap": { + "locked": "0.4.9", + "transitive": [ + "org.apache.tez:tez-runtime-library" + ] + }, + "org.slf4j:slf4j-api": { + "locked": "1.7.25", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "com.jolbox:bonecp", + "com.ning:async-http-client", + "com.yammer.metrics:metrics-core", + "com.zaxxer:HikariCP", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.calcite.avatica:avatica-metrics", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", + "org.apache.curator:curator-client", + "org.apache.directory.api:api-asn1-api", + "org.apache.directory.api:api-util", + "org.apache.directory.server:apacheds-i18n", + "org.apache.directory.server:apacheds-kerberos-codec", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", + "org.apache.iceberg:iceberg-api", + "org.apache.iceberg:iceberg-common", + "org.apache.iceberg:iceberg-core", + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.orc:orc-core", + "org.apache.orc:orc-shims", + "org.apache.parquet:parquet-common", + "org.apache.parquet:parquet-format-structures", + "org.apache.slider:slider-core", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.thrift:libthrift", + "org.apache.twill:twill-common", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", + "org.apache.zookeeper:zookeeper", + "org.slf4j:slf4j-simple" + ] + }, + "org.slf4j:slf4j-simple": { + "locked": "1.7.25" + }, + "org.sonatype.sisu.inject:cglib": { + "locked": "2.2.1-v20090111", + "transitive": [ + "com.google.inject:guice" + ] + }, + "org.threeten:threeten-extra": { + "locked": "1.5.0", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "org.xerial.snappy:snappy-java": { + "locked": "1.1.7.3", + "transitive": [ + "org.apache.parquet:parquet-hadoop" + ] + }, + "oro:oro": { + "locked": "2.0.8", + "transitive": [ + "org.apache.velocity:velocity" + ] + }, + "stax:stax-api": { + "locked": "1.0.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.codehaus.jettison:jettison" + ] + }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "xerces:xercesImpl": { + "locked": "2.9.1", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "xml-apis:xml-apis": { + "locked": "1.3.04", + "transitive": [ + "dom4j:dom4j", + "xerces:xercesImpl" + ] + }, + "xmlenc:xmlenc": { + "locked": "0.52", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs" + ] + } + }, + "testRuntimeClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, + "aopalliance:aopalliance": { + "locked": "1.0", + "transitive": [ + "com.google.inject:guice" + ] + }, + "asm:asm": { + "locked": "3.1", + "transitive": [ + "asm:asm-tree", + "com.sun.jersey:jersey-server", + "org.sonatype.sisu.inject:cglib" + ] + }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "ch.qos.logback:logback-classic": { + "locked": "1.0.9", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "ch.qos.logback:logback-core": { + "locked": "1.0.9", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "co.cask.tephra:tephra-api": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-core": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-hbase-compat-1.0": { + "locked": "0.6.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.beust:jcommander": { + "locked": "1.30", + "transitive": [ + "org.apache.slider:slider-core" + ] + }, + "com.esotericsoftware.kryo:kryo": { + "locked": "2.24.0", + "requested": "2.24.0" + }, + "com.esotericsoftware.minlog:minlog": { + "locked": "1.2", + "transitive": [ + "com.esotericsoftware.kryo:kryo" + ] + }, + "com.fasterxml.jackson.core:jackson-annotations": { + "locked": "2.10.2", + "requested": "2.6.5", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.calcite.avatica:avatica" + ] + }, + "com.fasterxml.jackson.core:jackson-core": { + "locked": "2.10.2", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.iceberg:iceberg-core" + ] + }, + "com.fasterxml.jackson.core:jackson-databind": { + "locked": "2.10.2", + "transitive": [ + "io.dropwizard.metrics:metrics-json", + "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-common", + "org.apache.iceberg:iceberg-core" + ] + }, + "com.github.ben-manes.caffeine:caffeine": { + "locked": "2.7.0", + "transitive": [ + "org.apache.iceberg:iceberg-core" + ] + }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "com.github.stephenc.findbugs:findbugs-annotations": { + "locked": "1.3.9-1", + "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.iceberg:iceberg-api", + "org.apache.iceberg:iceberg-common", + "org.apache.iceberg:iceberg-core", + "org.apache.iceberg:iceberg-data", + "org.apache.iceberg:iceberg-orc", + "org.apache.iceberg:iceberg-parquet" + ] + }, + "com.google.code.findbugs:jsr305": { + "locked": "3.0.0", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals", + "org.apache.twill:twill-api", + "org.apache.twill:twill-common", + "org.apache.twill:twill-zookeeper" + ] + }, + "com.google.code.gson:gson": { + "locked": "2.2.4", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" + ] + }, + "com.google.errorprone:error_prone_annotations": { + "locked": "2.3.3", + "transitive": [ + "com.github.ben-manes.caffeine:caffeine" + ] + }, + "com.google.guava:guava": { + "locked": "18.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.jolbox:bonecp", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", + "org.apache.calcite:calcite-linq4j", + "org.apache.curator:apache-curator", + "org.apache.curator:curator-client", + "org.apache.curator:curator-framework", + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-vector-code-gen", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", + "org.reflections:reflections" + ] + }, + "com.google.inject.extensions:guice-assistedinject": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core" + ] + }, + "com.google.inject:guice": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.google.inject.extensions:guice-assistedinject", + "com.google.inject.extensions:guice-servlet", + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "com.google.protobuf:protobuf-java": { + "locked": "3.0.0-beta-1", + "transitive": [ + "org.apache.calcite.avatica:avatica", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-metastore", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "com.jolbox:bonecp": { + "locked": "0.8.0.RELEASE", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.klarna:hiverunner": { + "locked": "5.2.1", + "requested": "5.2.1" + }, + "com.lmax:disruptor": { + "locked": "3.3.0", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "com.ning:async-http-client": { + "locked": "1.8.16", + "transitive": [ + "org.apache.tez:tez-runtime-library" + ] + }, + "com.sun.jersey.contribs:jersey-guice": { + "locked": "1.9", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "com.sun.jersey:jersey-client": { + "locked": "1.9", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "com.sun.jersey:jersey-core": { + "locked": "1.9", + "transitive": [ + "com.sun.jersey:jersey-client", + "com.sun.jersey:jersey-json", + "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-server" + ] + }, + "com.sun.jersey:jersey-json": { + "locked": "1.9", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "com.sun.jersey:jersey-server": { + "locked": "1.9", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" + ] + }, + "com.sun.xml.bind:jaxb-impl": { + "locked": "2.2.3-1", + "transitive": [ + "com.sun.jersey:jersey-json" + ] + }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server" + ] + }, + "com.yammer.metrics:metrics-core": { + "locked": "2.2.0", + "transitive": [ + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" + ] + }, + "com.zaxxer:HikariCP": { + "locked": "2.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "commons-beanutils:commons-beanutils": { + "locked": "1.7.0", + "transitive": [ + "commons-digester:commons-digester" + ] + }, + "commons-beanutils:commons-beanutils-core": { + "locked": "1.8.0", + "transitive": [ + "commons-configuration:commons-configuration" + ] + }, + "commons-cli:commons-cli": { + "locked": "1.2", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.tez:tez-dag" + ] + }, + "commons-codec:commons-codec": { + "locked": "1.9", + "transitive": [ + "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.httpcomponents:httpclient", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-library" + ] + }, + "commons-collections:commons-collections": { + "locked": "3.2.2", + "transitive": [ + "commons-configuration:commons-configuration", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-mapreduce" + ] + }, + "commons-configuration:commons-configuration": { + "locked": "1.6", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "commons-daemon:commons-daemon": { + "locked": "1.0.13", + "transitive": [ + "org.apache.hadoop:hadoop-hdfs" + ] + }, + "commons-dbcp:commons-dbcp": { + "locked": "1.4", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-metastore" + ] + }, + "commons-digester:commons-digester": { + "locked": "1.8", + "transitive": [ + "commons-configuration:commons-configuration" + ] + }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, + "commons-httpclient:commons-httpclient": { + "locked": "3.1", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" + ] + }, + "commons-io:commons-io": { + "locked": "2.4", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" + ] + }, + "commons-lang:commons-lang": { + "locked": "2.6", + "transitive": [ + "commons-configuration:commons-configuration", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", + "org.apache.orc:orc-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.velocity:velocity", + "org.pentaho:pentaho-aggdesigner-algorithm" + ] + }, + "commons-logging:commons-logging": { + "locked": "1.2", + "transitive": [ + "commons-beanutils:commons-beanutils", + "commons-beanutils:commons-beanutils-core", + "commons-configuration:commons-configuration", + "commons-digester:commons-digester", + "commons-el:commons-el", + "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", + "net.sf.jpam:jpam", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.pentaho:pentaho-aggdesigner-algorithm" + ] + }, + "commons-net:commons-net": { + "locked": "3.1", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "commons-pool:commons-pool": { + "locked": "1.6", + "transitive": [ + "commons-dbcp:commons-dbcp", + "org.apache.hive:hive-metastore", + "org.apache.parquet:parquet-hadoop" + ] + }, + "dom4j:dom4j": { + "locked": "1.6.1", + "transitive": [ + "org.reflections:reflections" + ] + }, + "io.airlift:aircompressor": { + "locked": "0.15", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] }, - "org.apache.orc:orc-core": { - "locked": "1.6.3", + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", "transitive": [ - "org.apache.iceberg:iceberg-orc" + "org.apache.hive:hive-common" ] }, - "org.apache.orc:orc-shims": { - "locked": "1.6.3", + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", "transitive": [ - "org.apache.orc:orc-core" + "org.apache.hive:hive-common" ] }, - "org.apache.parquet:parquet-avro": { - "locked": "1.11.0", + "io.netty:netty": { + "locked": "3.9.2.Final", "transitive": [ - "org.apache.iceberg:iceberg-parquet" + "com.ning:async-http-client", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", + "org.apache.zookeeper:zookeeper" ] }, - "org.apache.parquet:parquet-column": { - "locked": "1.11.0", + "io.netty:netty-all": { + "locked": "4.0.23.Final", "transitive": [ - "org.apache.parquet:parquet-avro", - "org.apache.parquet:parquet-hadoop" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server" ] }, - "org.apache.parquet:parquet-common": { - "locked": "1.11.0", + "it.unimi.dsi:fastutil": { + "locked": "6.5.6", "transitive": [ - "org.apache.parquet:parquet-column", - "org.apache.parquet:parquet-encoding" + "co.cask.tephra:tephra-core" ] }, - "org.apache.parquet:parquet-encoding": { - "locked": "1.11.0", + "jakarta.jms:jakarta.jms-api": { + "locked": "2.0.2", "transitive": [ - "org.apache.parquet:parquet-column" + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions" ] }, - "org.apache.parquet:parquet-format-structures": { - "locked": "1.11.0", + "javassist:javassist": { + "locked": "3.12.1.GA", "transitive": [ - "org.apache.parquet:parquet-avro", - "org.apache.parquet:parquet-common", - "org.apache.parquet:parquet-hadoop" + "org.reflections:reflections" ] }, - "org.apache.parquet:parquet-hadoop": { - "locked": "1.11.0", + "javax.activation:activation": { + "locked": "1.1", "transitive": [ - "org.apache.parquet:parquet-avro" + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" ] }, - "org.apache.parquet:parquet-jackson": { - "locked": "1.11.0", + "javax.annotation:javax.annotation-api": { + "locked": "1.3.2", "transitive": [ - "org.apache.parquet:parquet-hadoop" + "org.apache.parquet:parquet-format-structures" ] }, - "org.apache.yetus:audience-annotations": { - "locked": "0.11.0", + "javax.inject:javax.inject": { + "locked": "1", "transitive": [ - "org.apache.parquet:parquet-common" + "com.google.inject:guice", + "com.sun.jersey.contribs:jersey-guice" ] }, - "org.apache.zookeeper:zookeeper": { - "locked": "3.4.6", + "javax.jdo:jdo-api": { + "locked": "3.0.1", "transitive": [ - "org.apache.curator:curator-client", - "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hive:hive-metastore" ] }, - "org.checkerframework:checker-qual": { - "locked": "2.6.0", + "javax.mail:mail": { + "locked": "1.4.1", "transitive": [ - "com.github.ben-manes.caffeine:caffeine" + "org.eclipse.jetty.aggregate:jetty-all" ] }, - "org.codehaus.jackson:jackson-core-asl": { - "locked": "1.9.13", + "javax.servlet.jsp:jsp-api": { + "locked": "2.1", "transitive": [ - "com.sun.jersey:jersey-json", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.codehaus.jackson:jackson-jaxrs", - "org.codehaus.jackson:jackson-mapper-asl", - "org.codehaus.jackson:jackson-xc" + "org.apache.slider:slider-core" ] }, - "org.codehaus.jackson:jackson-jaxrs": { - "locked": "1.9.13", + "javax.servlet:jsp-api": { + "locked": "2.0", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" + "tomcat:jasper-compiler" ] }, - "org.codehaus.jackson:jackson-mapper-asl": { - "locked": "1.9.13", + "javax.servlet:servlet-api": { + "locked": "2.5", "transitive": [ - "com.sun.jersey:jersey-json", + "javax.servlet:jsp-api", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", - "org.codehaus.jackson:jackson-jaxrs", - "org.codehaus.jackson:jackson-xc" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.slider:slider-core", + "org.apache.tez:tez-dag", + "org.eclipse.jetty.aggregate:jetty-all", + "tomcat:jasper-runtime" ] }, - "org.codehaus.jackson:jackson-xc": { - "locked": "1.9.13", + "javax.transaction:jta": { + "locked": "1.1", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" + "javax.jdo:jdo-api" ] }, - "org.codehaus.jettison:jettison": { + "javax.transaction:transaction-api": { "locked": "1.1", "transitive": [ - "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.datanucleus:javax.jdo" ] }, - "org.fusesource.leveldbjni:leveldbjni-all": { - "locked": "1.8", + "javax.xml.bind:jaxb-api": { + "locked": "2.2.11", "transitive": [ - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "com.sun.xml.bind:jaxb-impl", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.orc:orc-core" ] }, - "org.hamcrest:hamcrest-core": { - "locked": "1.3", + "javolution:javolution": { + "locked": "5.5.1", "transitive": [ - "junit:junit", - "org.mockito:mockito-core" + "org.apache.hive:hive-metastore" ] }, - "org.jetbrains:annotations": { - "locked": "17.0.0", + "jline:jline": { + "locked": "2.12", "transitive": [ - "org.apache.orc:orc-core" + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.zookeeper:zookeeper" ] }, - "org.mockito:mockito-core": { - "locked": "1.10.19" + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-common" + ] }, - "org.objenesis:objenesis": { - "locked": "2.1", + "junit:junit": { + "locked": "4.12", "transitive": [ - "org.mockito:mockito-core" + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server" ] }, - "org.slf4j:slf4j-api": { - "locked": "1.7.25", + "log4j:log4j": { + "locked": "1.2.17", "transitive": [ - "org.apache.avro:avro", - "org.apache.curator:curator-client", - "org.apache.directory.api:api-asn1-api", - "org.apache.directory.api:api-util", - "org.apache.directory.server:apacheds-i18n", - "org.apache.directory.server:apacheds-kerberos-codec", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.iceberg:iceberg-api", - "org.apache.iceberg:iceberg-common", - "org.apache.iceberg:iceberg-core", - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet", - "org.apache.orc:orc-core", - "org.apache.orc:orc-shims", - "org.apache.parquet:parquet-common", - "org.apache.parquet:parquet-format-structures", - "org.apache.zookeeper:zookeeper", - "org.slf4j:slf4j-simple" + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-protocol", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", + "org.apache.zookeeper:zookeeper" ] }, - "org.slf4j:slf4j-simple": { - "locked": "1.7.25" + "net.hydromatic:eigenbase-properties": { + "locked": "1.1.5", + "transitive": [ + "org.apache.calcite:calcite-core" + ] }, - "org.sonatype.sisu.inject:cglib": { - "locked": "2.2.1-v20090111", + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", "transitive": [ - "com.google.inject:guice" + "org.apache.hadoop:hadoop-common" ] }, - "org.threeten:threeten-extra": { - "locked": "1.5.0", + "net.sf.jpam:jpam": { + "locked": "1.1", "transitive": [ - "org.apache.orc:orc-core" + "org.apache.hive:hive-service" ] }, - "org.xerial.snappy:snappy-java": { - "locked": "1.1.7.3", + "net.sf.opencsv:opencsv": { + "locked": "2.3", "transitive": [ - "org.apache.parquet:parquet-hadoop" + "org.apache.hive:hive-serde" ] }, - "xerces:xercesImpl": { - "locked": "2.9.1", + "org.antlr:ST4": { + "locked": "4.0.4", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hive:hive-exec" ] }, - "xml-apis:xml-apis": { - "locked": "1.3.04", + "org.antlr:antlr-runtime": { + "locked": "3.5.2", "transitive": [ - "xerces:xercesImpl" + "org.antlr:ST4", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" ] }, - "xmlenc:xmlenc": { - "locked": "0.52", + "org.apache.ant:ant": { + "locked": "1.9.1", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-vector-code-gen" + ] + }, + "org.apache.ant:ant-launcher": { + "locked": "1.9.1", + "transitive": [ + "org.apache.ant:ant" + ] + }, + "org.apache.avro:avro": { + "locked": "1.9.2", "transitive": [ "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-serde", + "org.apache.iceberg:iceberg-core", + "org.apache.slider:slider-core" ] - } - }, - "testRuntimeClasspath": { - "aopalliance:aopalliance": { - "locked": "1.0", + }, + "org.apache.calcite.avatica:avatica": { + "locked": "1.8.0", "transitive": [ - "com.google.inject:guice" + "org.apache.calcite:calcite-core" ] }, - "asm:asm": { - "locked": "3.1", + "org.apache.calcite.avatica:avatica-metrics": { + "locked": "1.8.0", "transitive": [ - "com.sun.jersey:jersey-server", - "org.sonatype.sisu.inject:cglib" + "org.apache.calcite.avatica:avatica" ] }, - "com.fasterxml.jackson.core:jackson-annotations": { - "locked": "2.10.2", + "org.apache.calcite:calcite-core": { + "locked": "1.10.0", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind" + "org.apache.calcite:calcite-druid", + "org.apache.hive:hive-exec" ] }, - "com.fasterxml.jackson.core:jackson-core": { - "locked": "2.10.2", + "org.apache.calcite:calcite-druid": { + "locked": "1.10.0", "transitive": [ - "com.fasterxml.jackson.core:jackson-databind", - "org.apache.avro:avro", - "org.apache.iceberg:iceberg-core" + "org.apache.hive:hive-exec" ] }, - "com.fasterxml.jackson.core:jackson-databind": { - "locked": "2.10.2", + "org.apache.calcite:calcite-linq4j": { + "locked": "1.10.0", "transitive": [ - "org.apache.avro:avro", - "org.apache.iceberg:iceberg-core" + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid" ] }, - "com.github.ben-manes.caffeine:caffeine": { - "locked": "2.7.0", + "org.apache.commons:commons-collections4": { + "locked": "4.1", "transitive": [ - "org.apache.iceberg:iceberg-core" + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag" ] }, - "com.github.stephenc.findbugs:findbugs-annotations": { - "locked": "1.3.9-1", + "org.apache.commons:commons-compress": { + "locked": "1.19", "transitive": [ - "org.apache.iceberg:iceberg-api", - "org.apache.iceberg:iceberg-common", - "org.apache.iceberg:iceberg-core", - "org.apache.iceberg:iceberg-data", - "org.apache.iceberg:iceberg-orc", - "org.apache.iceberg:iceberg-parquet" + "org.apache.avro:avro", + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.slider:slider-core" ] }, - "com.google.code.findbugs:jsr305": { - "locked": "3.0.0", + "org.apache.commons:commons-lang3": { + "locked": "3.2", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.calcite:calcite-core", + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-tez" ] }, - "com.google.code.gson:gson": { - "locked": "2.2.4", + "org.apache.commons:commons-math": { + "locked": "2.2", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hbase:hbase-hadoop-compat", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server" ] }, - "com.google.errorprone:error_prone_annotations": { - "locked": "2.3.3", + "org.apache.commons:commons-math3": { + "locked": "3.1.1", "transitive": [ - "com.github.ben-manes.caffeine:caffeine" + "org.apache.hadoop:hadoop-common", + "org.apache.tez:tez-dag" ] }, - "com.google.guava:guava": { - "locked": "16.0.1", + "org.apache.curator:apache-curator": { + "locked": "2.7.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-llap-client" + ] + }, + "org.apache.curator:curator-client": { + "locked": "2.7.1", "transitive": [ - "org.apache.curator:curator-client", "org.apache.curator:curator-framework", - "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.slider:slider-core" ] }, - "com.google.inject:guice": { - "locked": "3.0", + "org.apache.curator:curator-framework": { + "locked": "2.7.1", "transitive": [ - "com.sun.jersey.contribs:jersey-guice", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.curator:curator-recipes", + "org.apache.hadoop:hadoop-auth", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" ] }, - "com.google.protobuf:protobuf-java": { - "locked": "2.5.0", + "org.apache.curator:curator-recipes": { + "locked": "2.7.1", "transitive": [ "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-service", + "org.apache.slider:slider-core" ] }, - "com.sun.jersey.contribs:jersey-guice": { - "locked": "1.9", + "org.apache.derby:derby": { + "locked": "10.10.2.0", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-metastore" ] }, - "com.sun.jersey:jersey-client": { - "locked": "1.9", + "org.apache.directory.api:api-asn1-api": { + "locked": "1.0.0-M20", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.directory.server:apacheds-kerberos-codec" ] }, - "com.sun.jersey:jersey-core": { - "locked": "1.9", + "org.apache.directory.api:api-util": { + "locked": "1.0.0-M20", "transitive": [ - "com.sun.jersey:jersey-client", - "com.sun.jersey:jersey-json", - "com.sun.jersey:jersey-server", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.directory.server:apacheds-kerberos-codec" ] }, - "com.sun.jersey:jersey-json": { - "locked": "1.9", + "org.apache.directory.server:apacheds-i18n": { + "locked": "2.0.0-M15", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.directory.server:apacheds-kerberos-codec" ] }, - "com.sun.jersey:jersey-server": { - "locked": "1.9", + "org.apache.directory.server:apacheds-kerberos-codec": { + "locked": "2.0.0-M15", "transitive": [ - "com.sun.jersey.contribs:jersey-guice", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-auth" ] }, - "com.sun.xml.bind:jaxb-impl": { - "locked": "2.2.3-1", + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", "transitive": [ - "com.sun.jersey:jersey-json" + "org.eclipse.jetty.aggregate:jetty-all" ] }, - "commons-beanutils:commons-beanutils": { - "locked": "1.7.0", + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", "transitive": [ - "commons-digester:commons-digester" + "org.eclipse.jetty.aggregate:jetty-all" ] }, - "commons-beanutils:commons-beanutils-core": { - "locked": "1.8.0", + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", "transitive": [ - "commons-configuration:commons-configuration" + "org.eclipse.jetty.aggregate:jetty-all" ] }, - "commons-cli:commons-cli": { - "locked": "1.2", + "org.apache.hadoop:hadoop-annotations": { + "locked": "2.7.3", "transitive": [ + "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, - "commons-codec:commons-codec": { - "locked": "1.6", + "org.apache.hadoop:hadoop-archives": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.hcatalog:hive-hcatalog-core" + ] + }, + "org.apache.hadoop:hadoop-auth": { + "locked": "2.7.3", "transitive": [ - "commons-httpclient:commons-httpclient", - "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.tez:tez-api", + "org.apache.tez:tez-runtime-library" ] }, - "commons-collections:commons-collections": { - "locked": "3.2.2", + "org.apache.hadoop:hadoop-client": { + "locked": "2.7.3", "transitive": [ - "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" ] }, - "commons-configuration:commons-configuration": { - "locked": "1.6", + "org.apache.hadoop:hadoop-common": { + "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, - "commons-digester:commons-digester": { - "locked": "1.8", + "org.apache.hadoop:hadoop-hdfs": { + "locked": "2.7.3", "transitive": [ - "commons-configuration:commons-configuration" + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.slider:slider-core", + "org.apache.tez:tez-api" ] }, - "commons-httpclient:commons-httpclient": { - "locked": "3.1", + "org.apache.hadoop:hadoop-mapreduce-client-app": { + "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-client" ] }, - "commons-io:commons-io": { - "locked": "2.4", + "org.apache.hadoop:hadoop-mapreduce-client-common": { + "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.tez:tez-mapreduce" ] }, - "commons-lang:commons-lang": { - "locked": "2.6", + "org.apache.hadoop:hadoop-mapreduce-client-core": { + "locked": "2.7.3", "transitive": [ - "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-client", + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.tez:tez-mapreduce" ] }, - "commons-logging:commons-logging": { - "locked": "1.1.3", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { + "locked": "2.7.3", "transitive": [ - "commons-beanutils:commons-beanutils", - "commons-beanutils:commons-beanutils-core", - "commons-configuration:commons-configuration", - "commons-digester:commons-digester", - "commons-httpclient:commons-httpclient", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-client" + ] + }, + "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-app", + "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + ] + }, + "org.apache.hadoop:hadoop-yarn-api": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.httpcomponents:httpclient" + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, - "commons-net:commons-net": { - "locked": "3.1", + "org.apache.hadoop:hadoop-yarn-client": { + "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" ] }, - "commons-pool:commons-pool": { - "locked": "1.6", + "org.apache.hadoop:hadoop-yarn-common": { + "locked": "2.7.3", "transitive": [ - "org.apache.parquet:parquet-hadoop" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-client", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.tez:tez-api", + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" ] }, - "io.airlift:aircompressor": { - "locked": "0.15", + "org.apache.hadoop:hadoop-yarn-registry": { + "locked": "2.7.1", "transitive": [ - "org.apache.orc:orc-core" + "org.apache.slider:slider-core" ] }, - "io.netty:netty": { - "locked": "3.7.0.Final", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", "transitive": [ - "org.apache.hadoop:hadoop-hdfs", - "org.apache.zookeeper:zookeeper" + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, - "io.netty:netty-all": { - "locked": "4.0.23.Final", + "org.apache.hadoop:hadoop-yarn-server-common": { + "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, - "javax.annotation:javax.annotation-api": { - "locked": "1.3.2", + "org.apache.hadoop:hadoop-yarn-server-nodemanager": { + "locked": "2.7.3", "transitive": [ - "org.apache.parquet:parquet-format-structures" + "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, - "javax.inject:javax.inject": { - "locked": "1", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", "transitive": [ - "com.google.inject:guice", - "com.sun.jersey.contribs:jersey-guice" + "org.apache.hive.shims:hive-shims-0.23" ] }, - "javax.servlet.jsp:jsp-api": { - "locked": "2.1", + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.tez:tez-dag" ] }, - "javax.servlet:servlet-api": { - "locked": "2.5", + "org.apache.hbase:hbase-annotations": { + "locked": "1.1.1", "transitive": [ - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-protocol" ] }, - "javax.xml.bind:jaxb-api": { - "locked": "2.2.11", + "org.apache.hbase:hbase-client": { + "locked": "1.1.1", "transitive": [ - "com.sun.xml.bind:jaxb-impl", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.orc:orc-core" + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" ] }, - "jline:jline": { - "locked": "0.9.94", + "org.apache.hbase:hbase-common": { + "locked": "1.1.1", "transitive": [ - "org.apache.zookeeper:zookeeper" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" ] }, - "junit:junit": { - "locked": "4.12" - }, - "log4j:log4j": { - "locked": "1.2.17", + "org.apache.hbase:hbase-hadoop-compat": { + "locked": "1.1.1", "transitive": [ - "org.apache.hadoop:hadoop-auth", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.zookeeper:zookeeper" + "org.apache.hbase:hbase-hadoop2-compat", + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" ] }, - "org.apache.avro:avro": { - "locked": "1.9.2", + "org.apache.hbase:hbase-hadoop2-compat": { + "locked": "1.1.1", "transitive": [ - "org.apache.iceberg:iceberg-core" + "org.apache.hbase:hbase-prefix-tree", + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-llap-server" ] }, - "org.apache.commons:commons-compress": { - "locked": "1.19", + "org.apache.hbase:hbase-prefix-tree": { + "locked": "1.1.1", "transitive": [ - "org.apache.avro:avro", - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hbase:hbase-server" ] }, - "org.apache.commons:commons-math3": { - "locked": "3.1.1", + "org.apache.hbase:hbase-procedure": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-server" + ] + }, + "org.apache.hbase:hbase-protocol": { + "locked": "1.1.1", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-procedure", + "org.apache.hbase:hbase-server" ] }, - "org.apache.curator:curator-client": { - "locked": "2.7.1", + "org.apache.hbase:hbase-server": { + "locked": "1.1.1", "transitive": [ - "org.apache.curator:curator-framework", - "org.apache.hadoop:hadoop-common" + "org.apache.hive:hive-llap-server" ] }, - "org.apache.curator:curator-framework": { - "locked": "2.7.1", + "org.apache.hive.hcatalog:hive-hcatalog-core": { + "locked": "2.3.7", "transitive": [ - "org.apache.curator:curator-recipes", - "org.apache.hadoop:hadoop-auth" + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client" ] }, - "org.apache.curator:curator-recipes": { - "locked": "2.7.1", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hive.hcatalog:hive-webhcat-java-client" ] }, - "org.apache.directory.api:api-asn1-api": { - "locked": "1.0.0-M20", + "org.apache.hive.hcatalog:hive-webhcat-java-client": { + "locked": "2.3.7", "transitive": [ - "org.apache.directory.server:apacheds-kerberos-codec" + "com.klarna:hiverunner" ] }, - "org.apache.directory.api:api-util": { - "locked": "1.0.0-M20", + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", "transitive": [ - "org.apache.directory.server:apacheds-kerberos-codec" + "org.apache.hive:hive-shims" ] }, - "org.apache.directory.server:apacheds-i18n": { - "locked": "2.0.0-M15", + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", "transitive": [ - "org.apache.directory.server:apacheds-kerberos-codec" + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" ] }, - "org.apache.directory.server:apacheds-kerberos-codec": { - "locked": "2.0.0-M15", + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "org.apache.hive:hive-shims" ] }, - "org.apache.hadoop:hadoop-annotations": { - "locked": "2.7.3", + "org.apache.hive:hive-cli": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hive.hcatalog:hive-hcatalog-core" ] }, - "org.apache.hadoop:hadoop-auth": { - "locked": "2.7.3", + "org.apache.hive:hive-common": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-serde" ] }, - "org.apache.hadoop:hadoop-client": { - "locked": "2.7.3" + "org.apache.hive:hive-exec": { + "locked": "2.3.7" }, - "org.apache.hadoop:hadoop-common": { - "locked": "2.7.3", + "org.apache.hive:hive-jdbc": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client" + "com.klarna:hiverunner" ] }, - "org.apache.hadoop:hadoop-hdfs": { - "locked": "2.7.3", + "org.apache.hive:hive-llap-client": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-app": { - "locked": "2.7.3", + "org.apache.hive:hive-llap-common": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-server" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-common": { - "locked": "2.7.3", + "org.apache.hive:hive-llap-server": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.hive:hive-service" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-core": { - "locked": "2.7.3", + "org.apache.hive:hive-llap-tez": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hive:hive-llap-server" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { - "locked": "2.7.3", + "org.apache.hive:hive-metastore": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client" + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-service" ] }, - "org.apache.hadoop:hadoop-mapreduce-client-shuffle": { - "locked": "2.7.3", + "org.apache.hive:hive-serde": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-app", - "org.apache.hadoop:hadoop-mapreduce-client-jobclient" + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore" ] }, - "org.apache.hadoop:hadoop-yarn-api": { - "locked": "2.7.3", + "org.apache.hive:hive-service": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "com.klarna:hiverunner", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc" ] }, - "org.apache.hadoop:hadoop-yarn-client": { - "locked": "2.7.3", + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service" ] }, - "org.apache.hadoop:hadoop-yarn-common": { - "locked": "2.7.3", + "org.apache.hive:hive-shims": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-core", - "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde" ] }, - "org.apache.hadoop:hadoop-yarn-server-common": { - "locked": "2.7.3", + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-nodemanager": { - "locked": "2.7.3", + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.hive:hive-exec" ] }, "org.apache.htrace:htrace-core": { "locked": "3.1.0-incubating", "transitive": [ "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-server" ] }, "org.apache.httpcomponents:httpclient": { - "locked": "4.2.5", + "locked": "4.5.2", "transitive": [ - "org.apache.hadoop:hadoop-auth" + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hadoop:hadoop-auth", + "org.apache.hive:hive-jdbc", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" ] }, "org.apache.httpcomponents:httpcore": { - "locked": "4.2.4", + "locked": "4.4.4", "transitive": [ - "org.apache.httpcomponents:httpclient" + "net.java.dev.jets3t:jets3t", + "org.apache.calcite.avatica:avatica", + "org.apache.hive:hive-jdbc", + "org.apache.httpcomponents:httpclient", + "org.apache.slider:slider-core", + "org.apache.thrift:libthrift" ] }, "org.apache.iceberg:iceberg-api": { @@ -5141,9 +11266,52 @@ "org.apache.iceberg:iceberg-parquet": { "project": true }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-llap-server", "org.apache.iceberg:iceberg-orc" ] }, @@ -5193,12 +11361,151 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.slider:slider-core": { + "locked": "0.90.2-incubating", + "transitive": [ + "org.apache.hive:hive-llap-server" + ] + }, + "org.apache.tez:hadoop-shim": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-runtime-internals" + ] + }, + "org.apache.tez:tez-api": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-common", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.tez:tez-common": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-internals", + "org.apache.tez:tez-runtime-library" + ] + }, + "org.apache.tez:tez-dag": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.tez:tez-mapreduce": { + "locked": "0.9.1", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.apache.tez:tez-runtime-internals": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-dag" + ] + }, + "org.apache.tez:tez-runtime-library": { + "locked": "0.9.1", + "transitive": [ + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce" + ] + }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "org.apache.thrift:libthrift": { + "locked": "0.9.3", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, + "org.apache.twill:twill-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-common": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-api", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "org.apache.twill:twill-discovery-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.twill:twill-discovery-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core" + ] + }, + "org.apache.twill:twill-zookeeper": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.velocity:velocity": { + "locked": "1.5", + "transitive": [ + "org.apache.hive:hive-vector-code-gen" + ] + }, "org.apache.yetus:audience-annotations": { "locked": "0.11.0", "transitive": [ @@ -5208,12 +11515,32 @@ "org.apache.zookeeper:zookeeper": { "locked": "3.4.6", "transitive": [ + "org.apache.curator:apache-curator", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-common" + "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.slider:slider-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apiguardian:apiguardian-api": { + "locked": "1.1.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.jupiter:junit-jupiter-engine", + "org.junit.jupiter:junit-jupiter-params", + "org.junit.platform:junit-platform-commons", + "org.junit.platform:junit-platform-engine" ] }, "org.checkerframework:checker-qual": { @@ -5222,6 +11549,12 @@ "com.github.ben-manes.caffeine:caffeine" ] }, + "org.codehaus.groovy:groovy-all": { + "locked": "2.4.4", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, "org.codehaus.jackson:jackson-core-asl": { "locked": "1.9.13", "transitive": [ @@ -5229,6 +11562,9 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core", "org.codehaus.jackson:jackson-jaxrs", "org.codehaus.jackson:jackson-mapper-asl", "org.codehaus.jackson:jackson-xc" @@ -5238,7 +11574,9 @@ "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-server", + "org.apache.slider:slider-core" ] }, "org.codehaus.jackson:jackson-mapper-asl": { @@ -5248,6 +11586,12 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-registry", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-server", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.slider:slider-core", "org.codehaus.jackson:jackson-jaxrs", "org.codehaus.jackson:jackson-xc" ] @@ -5256,14 +11600,68 @@ "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.slider:slider-core" + ] + }, + "org.codehaus.janino:commons-compiler": { + "locked": "2.7.6", + "transitive": [ + "org.apache.calcite:calcite-core", + "org.codehaus.janino:janino" + ] + }, + "org.codehaus.janino:janino": { + "locked": "2.7.6", + "transitive": [ + "org.apache.calcite:calcite-core" ] }, "org.codehaus.jettison:jettison": { "locked": "1.1", "transitive": [ "com.sun.jersey:jersey-json", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.datanucleus:datanucleus-api-jdo": { + "locked": "4.2.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-rdbms": { + "locked": "4.1.19", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:javax.jdo": { + "locked": "3.2.0-m3", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-service" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -5271,8 +11669,10 @@ "transitive": [ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, "org.hamcrest:hamcrest-core": { @@ -5282,25 +11682,130 @@ "org.mockito:mockito-core" ] }, + "org.jamon:jamon-runtime": { + "locked": "2.3.1", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service" + ] + }, "org.jetbrains:annotations": { "locked": "17.0.0", "transitive": [ "org.apache.orc:orc-core" ] }, + "org.jruby.jcodings:jcodings": { + "locked": "1.0.8", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.jruby.joni:joni" + ] + }, + "org.jruby.joni:joni": { + "locked": "2.1.2", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.junit.jupiter:junit-jupiter": { + "locked": "5.6.0", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.junit.jupiter:junit-jupiter-api": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter", + "org.junit.jupiter:junit-jupiter-engine", + "org.junit.jupiter:junit-jupiter-params" + ] + }, + "org.junit.jupiter:junit-jupiter-engine": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter" + ] + }, + "org.junit.jupiter:junit-jupiter-params": { + "locked": "5.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter" + ] + }, + "org.junit.platform:junit-platform-commons": { + "locked": "1.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.platform:junit-platform-engine" + ] + }, + "org.junit.platform:junit-platform-engine": { + "locked": "1.6.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-engine" + ] + }, "org.mockito:mockito-core": { "locked": "1.10.19" }, "org.objenesis:objenesis": { "locked": "2.1", "transitive": [ + "com.esotericsoftware.kryo:kryo", "org.mockito:mockito-core" ] }, + "org.opentest4j:opentest4j": { + "locked": "1.2.0", + "transitive": [ + "org.junit.jupiter:junit-jupiter-api", + "org.junit.platform:junit-platform-engine" + ] + }, + "org.ow2.asm:asm-all": { + "locked": "5.0.2", + "transitive": [ + "org.apache.twill:twill-core" + ] + }, + "org.pentaho:pentaho-aggdesigner-algorithm": { + "locked": "5.1.5-jhyde", + "transitive": [ + "org.apache.calcite:calcite-core" + ] + }, + "org.reflections:reflections": { + "locked": "0.9.8", + "transitive": [ + "com.klarna:hiverunner" + ] + }, + "org.roaringbitmap:RoaringBitmap": { + "locked": "0.4.9", + "transitive": [ + "org.apache.tez:tez-runtime-library" + ] + }, "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "com.jolbox:bonecp", + "com.ning:async-http-client", + "com.yammer.metrics:metrics-core", + "com.zaxxer:HikariCP", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", + "org.apache.calcite.avatica:avatica", + "org.apache.calcite.avatica:avatica-metrics", + "org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", "org.apache.directory.api:api-util", @@ -5315,16 +11820,49 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hive.hcatalog:hive-hcatalog-core", + "org.apache.hive.hcatalog:hive-hcatalog-server-extensions", + "org.apache.hive.hcatalog:hive-webhcat-java-client", + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-cli", + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-jdbc", + "org.apache.hive:hive-llap-client", + "org.apache.hive:hive-llap-common", + "org.apache.hive:hive-llap-server", + "org.apache.hive:hive-llap-tez", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc", + "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", + "org.apache.hive:hive-vector-code-gen", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", + "org.apache.slider:slider-core", + "org.apache.tez:hadoop-shim", + "org.apache.tez:tez-api", + "org.apache.tez:tez-dag", + "org.apache.tez:tez-mapreduce", + "org.apache.tez:tez-runtime-library", + "org.apache.thrift:libthrift", + "org.apache.twill:twill-common", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", "org.apache.zookeeper:zookeeper", "org.slf4j:slf4j-simple" ] @@ -5350,6 +11888,35 @@ "org.apache.parquet:parquet-hadoop" ] }, + "oro:oro": { + "locked": "2.0.8", + "transitive": [ + "org.apache.velocity:velocity" + ] + }, + "stax:stax-api": { + "locked": "1.0.1", + "transitive": [ + "org.apache.hive:hive-exec", + "org.codehaus.jettison:jettison" + ] + }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hbase:hbase-server", + "org.apache.hive:hive-service", + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -5359,6 +11926,7 @@ "xml-apis:xml-apis": { "locked": "1.3.04", "transitive": [ + "dom4j:dom4j", "xerces:xercesImpl" ] }, diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java new file mode 100644 index 000000000000..872094eeb91a --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.StandaloneHiveRunner; +import com.klarna.hiverunner.annotations.HiveSQL; +import java.io.File; +import java.io.IOException; +import java.util.List; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertEquals; + +@RunWith(StandaloneHiveRunner.class) +public class TestHiveIcebergInputFormat { + + @HiveSQL(files = {}, autoStart = true) + private HiveShell shell; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private File tableLocation; + + @Before + public void before() throws IOException { + tableLocation = temp.newFolder(); + Schema schema = new Schema(required(1, "id", Types.LongType.get()), + optional(2, "data", Types.StringType.get())); + PartitionSpec spec = PartitionSpec.unpartitioned(); + + HadoopTables tables = new HadoopTables(); + Table table = tables.create(schema, spec, tableLocation.getAbsolutePath()); + } + + @Test + public void emptyTable() { + shell.execute("CREATE DATABASE source_db"); + shell.execute(new StringBuilder() + .append("CREATE TABLE source_db.table_a ") + .append("(id INT, data STRING) ") + .append("STORED AS ") + .append("INPUTFORMAT 'org.apache.iceberg.mr.mapred.IcebergInputFormat' ") + .append("OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' ") + .append("LOCATION '") + .append(tableLocation.getAbsolutePath()) + .append("' TBLPROPERTIES ('iceberg.catalog'='hadoop.tables'") + .append(")") + .toString()); + + List result = shell.executeStatement("SELECT id, data FROM source_db.table_a"); + + assertEquals(0, result.size()); + } + +} From f6c510835863846ec14d5a80f0266d8c502d5ab0 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Thu, 4 Jun 2020 13:54:38 +0100 Subject: [PATCH 39/51] tidy up --- build.gradle | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.gradle b/build.gradle index 19184f5b1431..311e1c2b2612 100644 --- a/build.gradle +++ b/build.gradle @@ -328,7 +328,6 @@ project(':iceberg-mr') { exclude group: 'org.apache.hive', module: 'hive-exec' exclude group: 'org.codehaus.jettison', module: 'jettison' exclude group: 'org.apache.calcite.avatica' - //exclude group: 'com.fasterxml.jackson.core' } testCompile("org.apache.hive:hive-exec::core") { @@ -340,14 +339,11 @@ project(':iceberg-mr') { exclude group: 'com.google.protobuf', module: 'protobuf-java' exclude group: 'org.apache.calcite.avatica' exclude group: 'com.google.code.findbugs', module: 'jsr305' - //exclude group: 'com.fasterxml.jackson.core' } testCompile("org.apache.calcite:calcite-core") testCompile("com.esotericsoftware.kryo:kryo:2.24.0") testCompile("com.fasterxml.jackson.core:jackson-annotations:2.6.5") - //testCompile("com.fasterxml.jackson.core:jackson-databind:2.6.0") - testCompile project(path: ':iceberg-data', configuration: 'testArtifacts') testCompile project(path: ':iceberg-api', configuration: 'testArtifacts') From a6c2b19eb2dacc1f5a53fffa7ceb89f3b9ff7051 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Fri, 5 Jun 2020 13:29:24 +0100 Subject: [PATCH 40/51] exclude pentaho --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 311e1c2b2612..eabeaba1c0b0 100644 --- a/build.gradle +++ b/build.gradle @@ -88,6 +88,7 @@ subprojects { all { exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.mortbay.jetty' + exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm' resolutionStrategy { force 'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.10.2' From 937e228b6d30f39ea826b67443094b8a7ad94558 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 8 Jun 2020 16:32:45 +0100 Subject: [PATCH 41/51] wip checkpoint --- build.gradle | 8 + mr/dependencies.lock | 1464 +++++++++++++++-- .../apache/iceberg/mr/InputFormatConfig.java | 89 +- .../mr/mapred/IcebergFilterFactory.java | 158 ++ .../iceberg/mr/mapred/IcebergInputFormat.java | 127 +- .../iceberg/mr/mapred/SystemTableUtil.java | 74 + .../iceberg/mr/mapred/TableResolver.java | 121 ++ .../mr/mapreduce/IcebergInputFormat.java | 32 +- .../iceberg/mr/mapreduce/TableResolver.java | 61 + .../iceberg/mr/BaseInputFormatTest.java | 110 -- .../org/apache/iceberg/mr/TestHelpers.java | 31 +- .../mr/mapred/TestIcebergFilterFactory.java | 191 +++ .../mr/mapred/TestIcebergInputFormat.java | 113 +- .../iceberg/mr/mapred/TestTableResolver.java | 62 + .../mr/mapreduce/TestIcebergInputFormat.java | 117 +- 15 files changed, 2332 insertions(+), 426 deletions(-) create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java delete mode 100644 mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java diff --git a/build.gradle b/build.gradle index eabeaba1c0b0..9ffce387c305 100644 --- a/build.gradle +++ b/build.gradle @@ -324,6 +324,14 @@ project(':iceberg-mr') { exclude group: 'com.google.guava' } + compileOnly("org.apache.hive:hive-metastore") { + //exclude group: 'org.apache.avro', module: 'avro' + } + + compileOnly("org.apache.hive:hive-serde") { + //exclude group: 'org.apache.avro', module: 'avro' + } + testCompile("com.klarna:hiverunner:5.2.1") { exclude group: 'javax.jms', module: 'jms' exclude group: 'org.apache.hive', module: 'hive-exec' diff --git a/mr/dependencies.lock b/mr/dependencies.lock index da03056f5794..99c48ccf7e10 100644 --- a/mr/dependencies.lock +++ b/mr/dependencies.lock @@ -586,6 +586,12 @@ } }, "compileClasspath": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -595,10 +601,61 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "ch.qos.logback:logback-classic": { + "locked": "1.0.9", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "ch.qos.logback:logback-core": { + "locked": "1.0.9", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "co.cask.tephra:tephra-api": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-core": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-hbase-compat-1.0": { + "locked": "0.6.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "com.fasterxml.jackson.core:jackson-annotations": { "locked": "2.10.2", "transitive": [ @@ -616,7 +673,9 @@ "com.fasterxml.jackson.core:jackson-databind": { "locked": "2.10.2", "transitive": [ + "io.dropwizard.metrics:metrics-json", "org.apache.avro:avro", + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-core" ] }, @@ -626,9 +685,19 @@ "org.apache.iceberg:iceberg-core" ] }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "com.github.stephenc.findbugs:findbugs-annotations": { "locked": "1.3.9-1", "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", "org.apache.iceberg:iceberg-core", @@ -640,14 +709,21 @@ "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", + "org.apache.twill:twill-api", + "org.apache.twill:twill-common", + "org.apache.twill:twill-zookeeper" ] }, "com.google.code.gson:gson": { "locked": "2.2.4", "transitive": [ + "co.cask.tephra:tephra-core", "org.apache.hadoop:hadoop-common", - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, "com.google.errorprone:error_prone_annotations": { @@ -656,9 +732,26 @@ "com.github.ben-manes.caffeine:caffeine" ] }, + "com.google.inject.extensions:guice-assistedinject": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "com.google.inject.extensions:guice-servlet": { + "locked": "3.0", + "transitive": [ + "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-common" + ] + }, "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "co.cask.tephra:tephra-core", + "com.google.inject.extensions:guice-assistedinject", + "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" @@ -677,7 +770,30 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", + "org.apache.hive:hive-metastore", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "com.jolbox:bonecp": { + "locked": "0.8.0.RELEASE", + "transitive": [ + "org.apache.hive:hive-metastore" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -700,6 +816,7 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -707,6 +824,7 @@ "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] @@ -715,6 +833,7 @@ "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -724,6 +843,18 @@ "com.sun.jersey:jersey-json" ] }, + "com.tdunning:json": { + "locked": "1.8", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "com.zaxxer:HikariCP": { + "locked": "2.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "commons-beanutils:commons-beanutils": { "locked": "1.7.0", "transitive": [ @@ -742,19 +873,27 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", "org.apache.hive:hive-exec", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -762,7 +901,8 @@ "locked": "3.2.2", "transitive": [ "commons-configuration:commons-configuration", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-common" ] }, "commons-configuration:commons-configuration": { @@ -771,12 +911,24 @@ "org.apache.hadoop:hadoop-common" ] }, + "commons-dbcp:commons-dbcp": { + "locked": "1.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "commons-digester:commons-digester": { "locked": "1.8", "transitive": [ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -790,6 +942,8 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", "org.apache.hive:hive-exec" ] }, @@ -803,8 +957,15 @@ "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", "org.apache.hive:hive-vector-code-gen", + "org.apache.orc:orc-core", "org.apache.velocity:velocity" ] }, @@ -815,7 +976,9 @@ "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -823,6 +986,9 @@ "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", "org.apache.httpcomponents:httpclient" ] }, @@ -835,6 +1001,8 @@ "commons-pool:commons-pool": { "locked": "1.6", "transitive": [ + "commons-dbcp:commons-dbcp", + "org.apache.hive:hive-metastore", "org.apache.parquet:parquet-hadoop" ] }, @@ -844,17 +1012,54 @@ "org.apache.orc:orc-core" ] }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.zookeeper:zookeeper" ] }, "io.netty:netty-all": { "locked": "4.0.23.Final", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client" + ] + }, + "it.unimi.dsi:fastutil": { + "locked": "6.5.6", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "javax.activation:activation": { + "locked": "1.1", + "transitive": [ + "javax.mail:mail", + "org.eclipse.jetty.aggregate:jetty-all" ] }, "javax.annotation:javax.annotation-api": { @@ -870,11 +1075,44 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.jdo:jdo-api": { + "locked": "3.0.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" + ] + }, + "javax.transaction:jta": { + "locked": "1.1", + "transitive": [ + "javax.jdo:jdo-api" + ] + }, + "javax.transaction:transaction-api": { + "locked": "1.1", + "transitive": [ + "org.datanucleus:javax.jdo" ] }, "javax.xml.bind:jaxb-api": { @@ -886,16 +1124,32 @@ "org.apache.orc:orc-core" ] }, - "jline:jline": { - "locked": "0.9.94", + "javolution:javolution": { + "locked": "5.5.1", "transitive": [ - "org.apache.zookeeper:zookeeper" + "org.apache.hive:hive-metastore" + ] + }, + "jline:jline": { + "locked": "2.12", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.zookeeper:zookeeper" + ] + }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" ] }, "junit:junit": { - "locked": "3.8.1", + "locked": "4.11", "transitive": [ - "jline:jline" + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol" ] }, "log4j:log4j": { @@ -905,9 +1159,24 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.antlr:ST4": { "locked": "4.0.4", "transitive": [ @@ -918,12 +1187,14 @@ "locked": "3.5.2", "transitive": [ "org.antlr:ST4", - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" ] }, "org.apache.ant:ant": { "locked": "1.9.1", "transitive": [ + "org.apache.hive:hive-common", "org.apache.hive:hive-exec", "org.apache.hive:hive-vector-code-gen" ] @@ -937,6 +1208,9 @@ "org.apache.avro:avro": { "locked": "1.9.2", "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-serde", "org.apache.iceberg:iceberg-core" ] }, @@ -946,9 +1220,16 @@ "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", "org.apache.hive:hive-exec" ] }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.commons:commons-math3": { "locked": "3.1.1", "transitive": [ @@ -983,6 +1264,12 @@ "org.apache.hadoop:hadoop-common" ] }, + "org.apache.derby:derby": { + "locked": "10.10.2.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "org.apache.directory.api:api-asn1-api": { "locked": "1.0.0-M20", "transitive": [ @@ -1007,17 +1294,39 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hadoop:hadoop-yarn-api", + "org.apache.hadoop:hadoop-yarn-common" ] }, "org.apache.hadoop:hadoop-auth": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-client" ] }, "org.apache.hadoop:hadoop-client": { @@ -1026,7 +1335,10 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" ] }, "org.apache.hadoop:hadoop-hdfs": { @@ -1053,7 +1365,9 @@ "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" ] }, "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { @@ -1109,19 +1423,76 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle" ] }, + "org.apache.hbase:hbase-annotations": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol" + ] + }, + "org.apache.hbase:hbase-client": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hbase:hbase-common": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.apache.hbase:hbase-protocol": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" + ] + }, "org.apache.hive.shims:hive-shims-common": { "locked": "2.3.7", "transitive": [ "org.apache.hive:hive-shims" ] }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.hive:hive-exec": { "locked": "2.3.7" }, + "org.apache.hive:hive-metastore": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.hive:hive-shims": { "locked": "2.3.7", "transitive": [ - "org.apache.hive:hive-exec" + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.apache.hive:hive-vector-code-gen": { @@ -1134,12 +1505,15 @@ "locked": "3.1.0-incubating", "transitive": [ "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" ] }, "org.apache.httpcomponents:httpclient": { "locked": "4.4.1", "transitive": [ + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.thrift:libthrift" ] @@ -1147,6 +1521,7 @@ "org.apache.httpcomponents:httpcore": { "locked": "4.4.1", "transitive": [ + "net.java.dev.jets3t:jets3t", "org.apache.httpcomponents:httpclient", "org.apache.thrift:libthrift" ] @@ -1196,9 +1571,45 @@ "org.apache.hive:hive-exec" ] }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", + "transitive": [ + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" + ] + }, + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" + ] + }, + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.orc:orc-core": { "locked": "1.6.3", "transitive": [ + "org.apache.hive:hive-common", "org.apache.iceberg:iceberg-orc" ] }, @@ -1248,16 +1659,79 @@ "org.apache.parquet:parquet-avro" ] }, + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.apache.parquet:parquet-jackson": { "locked": "1.11.0", "transitive": [ "org.apache.parquet:parquet-hadoop" ] }, + "org.apache.thrift:libfb303": { + "locked": "0.9.3", + "transitive": [ + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service-rpc" + ] + }, "org.apache.thrift:libthrift": { "locked": "0.9.3", "transitive": [ - "org.apache.hive.shims:hive-shims-common" + "co.cask.tephra:tephra-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" + ] + }, + "org.apache.twill:twill-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-common": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-api", + "org.apache.twill:twill-zookeeper" + ] + }, + "org.apache.twill:twill-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core" + ] + }, + "org.apache.twill:twill-discovery-api": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-core" + ] + }, + "org.apache.twill:twill-discovery-core": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core" + ] + }, + "org.apache.twill:twill-zookeeper": { + "locked": "0.6.0-incubating", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, "org.apache.velocity:velocity": { @@ -1282,8 +1756,10 @@ "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-server-common", + "org.apache.hbase:hbase-client", "org.apache.hive.shims:hive-shims-common", - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.twill:twill-zookeeper" ] }, "org.checkerframework:checker-qual": { @@ -1324,6 +1800,7 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-client", "org.codehaus.jackson:jackson-jaxrs", "org.codehaus.jackson:jackson-xc" ] @@ -1342,10 +1819,41 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, - "org.datanucleus:datanucleus-core": { - "locked": "4.1.17", - "transitive": [ - "org.apache.hive:hive-exec" + "org.datanucleus:datanucleus-api-jdo": { + "locked": "4.2.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-core": { + "locked": "4.1.17", + "transitive": [ + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-rdbms": { + "locked": "4.1.19", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:javax.jdo": { + "locked": "3.2.0-m3", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -1357,15 +1865,48 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager" ] }, + "org.hamcrest:hamcrest-core": { + "locked": "1.3", + "transitive": [ + "junit:junit" + ] + }, "org.jetbrains:annotations": { "locked": "17.0.0", "transitive": [ "org.apache.orc:orc-core" ] }, + "org.jruby.jcodings:jcodings": { + "locked": "1.0.8", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.jruby.joni:joni" + ] + }, + "org.jruby.joni:joni": { + "locked": "2.1.2", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.ow2.asm:asm-all": { + "locked": "5.0.2", + "transitive": [ + "org.apache.twill:twill-core" + ] + }, "org.slf4j:slf4j-api": { "locked": "1.7.25", "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "com.jolbox:bonecp", + "com.zaxxer:HikariCP", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", @@ -1382,8 +1923,13 @@ "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.hive:hive-vector-code-gen", "org.apache.iceberg:iceberg-api", "org.apache.iceberg:iceberg-common", @@ -1391,11 +1937,15 @@ "org.apache.iceberg:iceberg-data", "org.apache.iceberg:iceberg-orc", "org.apache.iceberg:iceberg-parquet", + "org.apache.logging.log4j:log4j-slf4j-impl", "org.apache.orc:orc-core", "org.apache.orc:orc-shims", "org.apache.parquet:parquet-common", "org.apache.parquet:parquet-format-structures", "org.apache.thrift:libthrift", + "org.apache.twill:twill-common", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", "org.apache.zookeeper:zookeeper" ] }, @@ -1429,6 +1979,18 @@ "org.apache.hive:hive-exec" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -1450,6 +2012,12 @@ } }, "compileOnly": { + "ant:ant": { + "locked": "1.6.5", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "aopalliance:aopalliance": { "locked": "1.0", "transitive": [ @@ -1459,26 +2027,120 @@ "asm:asm": { "locked": "3.1", "transitive": [ + "asm:asm-tree", "com.sun.jersey:jersey-server", "org.sonatype.sisu.inject:cglib" ] }, + "asm:asm-commons": { + "locked": "3.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "asm:asm-tree": { + "locked": "3.1", + "transitive": [ + "asm:asm-commons" + ] + }, + "ch.qos.logback:logback-classic": { + "locked": "1.0.9", + "transitive": [ + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "ch.qos.logback:logback-core": { + "locked": "1.0.9", + "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "co.cask.tephra:tephra-api": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-core", + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-core": { + "locked": "0.6.0", + "transitive": [ + "co.cask.tephra:tephra-hbase-compat-1.0", + "org.apache.hive:hive-metastore" + ] + }, + "co.cask.tephra:tephra-hbase-compat-1.0": { + "locked": "0.6.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "com.fasterxml.jackson.core:jackson-annotations": { + "locked": "2.6.0", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind" + ] + }, + "com.fasterxml.jackson.core:jackson-core": { + "locked": "2.6.5", + "transitive": [ + "com.fasterxml.jackson.core:jackson-databind" + ] + }, + "com.fasterxml.jackson.core:jackson-databind": { + "locked": "2.6.5", + "transitive": [ + "io.dropwizard.metrics:metrics-json", + "org.apache.hive:hive-common" + ] + }, + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter": { + "locked": "0.1.2", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "com.github.stephenc.findbugs:findbugs-annotations": { + "locked": "1.3.9-1", + "transitive": [ + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol" + ] + }, "com.google.code.findbugs:jsr305": { "locked": "3.0.0", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hive:hive-serde", + "org.apache.twill:twill-api", + "org.apache.twill:twill-common", + "org.apache.twill:twill-zookeeper" ] }, "com.google.code.gson:gson": { "locked": "2.2.4", "transitive": [ + "co.cask.tephra:tephra-core", "org.apache.hadoop:hadoop-common", - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, "com.google.guava:guava": { "locked": "16.0.1", "transitive": [ + "co.cask.tephra:tephra-core", + "com.jolbox:bonecp", "org.apache.curator:curator-client", "org.apache.curator:curator-framework", "org.apache.curator:curator-recipes", @@ -1487,14 +2149,30 @@ "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" + ] + }, + "com.google.inject.extensions:guice-assistedinject": { + "locked": "3.0", + "transitive": [ + "co.cask.tephra:tephra-core" ] }, "com.google.inject.extensions:guice-servlet": { "locked": "3.0", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-resourcemanager" @@ -1503,6 +2181,8 @@ "com.google.inject:guice": { "locked": "3.0", "transitive": [ + "co.cask.tephra:tephra-core", + "com.google.inject.extensions:guice-assistedinject", "com.google.inject.extensions:guice-servlet", "com.sun.jersey.contribs:jersey-guice", "org.apache.hadoop:hadoop-yarn-common", @@ -1523,8 +2203,33 @@ "org.apache.hadoop:hadoop-mapreduce-client-shuffle", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", + "org.apache.hive:hive-metastore", + "org.apache.orc:orc-core" + ] + }, + "com.jamesmurty.utils:java-xmlbuilder": { + "locked": "0.4", + "transitive": [ + "net.java.dev.jets3t:jets3t" + ] + }, + "com.jcraft:jsch": { + "locked": "0.1.42", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "com.jolbox:bonecp": { + "locked": "0.8.0.RELEASE", + "transitive": [ + "org.apache.hive:hive-metastore" ] }, "com.sun.jersey.contribs:jersey-guice": { @@ -1551,6 +2256,7 @@ "com.sun.jersey:jersey-client", "com.sun.jersey:jersey-json", "com.sun.jersey:jersey-server", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", @@ -1560,6 +2266,7 @@ "com.sun.jersey:jersey-json": { "locked": "1.9", "transitive": [ + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", "org.apache.hadoop:hadoop-yarn-server-nodemanager", @@ -1570,6 +2277,7 @@ "locked": "1.9", "transitive": [ "com.sun.jersey.contribs:jersey-guice", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common" ] }, @@ -1579,37 +2287,63 @@ "com.sun.jersey:jersey-json" ] }, - "commons-beanutils:commons-beanutils": { - "locked": "1.7.0", + "com.tdunning:json": { + "locked": "1.8", "transitive": [ - "commons-digester:commons-digester" + "org.apache.hive:hive-common" ] }, - "commons-beanutils:commons-beanutils-core": { - "locked": "1.8.0", + "com.thoughtworks.paranamer:paranamer": { + "locked": "2.3", "transitive": [ - "commons-configuration:commons-configuration" + "org.apache.avro:avro" ] }, - "commons-cli:commons-cli": { - "locked": "1.2", - "transitive": [ + "com.zaxxer:HikariCP": { + "locked": "2.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "commons-beanutils:commons-beanutils": { + "locked": "1.7.0", + "transitive": [ + "commons-digester:commons-digester" + ] + }, + "commons-beanutils:commons-beanutils-core": { + "locked": "1.8.0", + "transitive": [ + "commons-configuration:commons-configuration" + ] + }, + "commons-cli:commons-cli": { + "locked": "1.2", + "transitive": [ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-client", - "org.apache.hadoop:hadoop-yarn-common" + "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service-rpc" ] }, "commons-codec:commons-codec": { "locked": "1.9", "transitive": [ "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-auth", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", "org.apache.hive:hive-exec", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.httpcomponents:httpclient" ] }, @@ -1618,7 +2352,8 @@ "transitive": [ "commons-configuration:commons-configuration", "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice" + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hbase:hbase-common" ] }, "commons-configuration:commons-configuration": { @@ -1627,12 +2362,24 @@ "org.apache.hadoop:hadoop-common" ] }, + "commons-dbcp:commons-dbcp": { + "locked": "1.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "commons-digester:commons-digester": { "locked": "1.8", "transitive": [ "commons-configuration:commons-configuration" ] }, + "commons-el:commons-el": { + "locked": "1.0", + "transitive": [ + "tomcat:jasper-runtime" + ] + }, "commons-httpclient:commons-httpclient": { "locked": "3.1", "transitive": [ @@ -1647,6 +2394,8 @@ "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", "org.apache.hive:hive-exec" ] }, @@ -1661,9 +2410,16 @@ "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-nodemanager", "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", "org.apache.hive.shims:hive-shims-0.23", "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-storage-api", "org.apache.hive:hive-vector-code-gen", + "org.apache.orc:orc-core", "org.apache.velocity:velocity" ] }, @@ -1674,7 +2430,9 @@ "commons-beanutils:commons-beanutils-core", "commons-configuration:commons-configuration", "commons-digester:commons-digester", + "commons-el:commons-el", "commons-httpclient:commons-httpclient", + "net.java.dev.jets3t:jets3t", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-api", @@ -1685,6 +2443,9 @@ "org.apache.hadoop:hadoop-yarn-server-nodemanager", "org.apache.hadoop:hadoop-yarn-server-resourcemanager", "org.apache.hadoop:hadoop-yarn-server-web-proxy", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", "org.apache.httpcomponents:httpclient" ] }, @@ -1694,23 +2455,74 @@ "org.apache.hadoop:hadoop-common" ] }, + "commons-pool:commons-pool": { + "locked": "1.5.4", + "transitive": [ + "commons-dbcp:commons-dbcp", + "org.apache.hive:hive-metastore" + ] + }, + "io.airlift:aircompressor": { + "locked": "0.8", + "transitive": [ + "org.apache.orc:orc-core" + ] + }, + "io.airlift:slice": { + "locked": "0.29", + "transitive": [ + "io.airlift:aircompressor" + ] + }, + "io.dropwizard.metrics:metrics-core": { + "locked": "3.1.2", + "transitive": [ + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-json": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "io.dropwizard.metrics:metrics-jvm": { + "locked": "3.1.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "io.netty:netty": { "locked": "3.7.0.Final", "transitive": [ "org.apache.hadoop:hadoop-hdfs", + "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.zookeeper:zookeeper" ] }, "io.netty:netty-all": { "locked": "4.0.23.Final", "transitive": [ - "org.apache.hadoop:hadoop-hdfs" + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client" + ] + }, + "it.unimi.dsi:fastutil": { + "locked": "6.5.6", + "transitive": [ + "co.cask.tephra:tephra-core" ] }, "javax.activation:activation": { "locked": "1.1", "transitive": [ - "javax.xml.bind:jaxb-api" + "javax.mail:mail", + "javax.xml.bind:jaxb-api", + "org.eclipse.jetty.aggregate:jetty-all" ] }, "javax.inject:javax.inject": { @@ -1720,17 +2532,50 @@ "com.sun.jersey.contribs:jersey-guice" ] }, + "javax.jdo:jdo-api": { + "locked": "3.0.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "javax.mail:mail": { + "locked": "1.4.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "javax.servlet.jsp:jsp-api": { "locked": "2.1", "transitive": [ "org.apache.hadoop:hadoop-common" ] }, + "javax.servlet:jsp-api": { + "locked": "2.0", + "transitive": [ + "tomcat:jasper-compiler" + ] + }, "javax.servlet:servlet-api": { "locked": "2.5", "transitive": [ + "javax.servlet:jsp-api", + "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", - "org.apache.hadoop:hadoop-yarn-server-nodemanager" + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "tomcat:jasper-runtime" + ] + }, + "javax.transaction:jta": { + "locked": "1.1", + "transitive": [ + "javax.jdo:jdo-api" + ] + }, + "javax.transaction:transaction-api": { + "locked": "1.1", + "transitive": [ + "org.datanucleus:javax.jdo" ] }, "javax.xml.bind:jaxb-api": { @@ -1749,16 +2594,32 @@ "javax.xml.bind:jaxb-api" ] }, + "javolution:javolution": { + "locked": "5.5.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "jline:jline": { - "locked": "0.9.94", + "locked": "2.12", "transitive": [ + "org.apache.hive:hive-common", "org.apache.zookeeper:zookeeper" ] }, + "joda-time:joda-time": { + "locked": "2.8.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "junit:junit": { - "locked": "3.8.1", + "locked": "4.11", "transitive": [ - "jline:jline" + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol" ] }, "log4j:log4j": { @@ -1770,9 +2631,24 @@ "org.apache.hadoop:hadoop-yarn-client", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-annotations", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol", "org.apache.zookeeper:zookeeper" ] }, + "net.java.dev.jets3t:jets3t": { + "locked": "0.9.0", + "transitive": [ + "org.apache.hadoop:hadoop-common" + ] + }, + "net.sf.opencsv:opencsv": { + "locked": "2.3", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, "org.antlr:ST4": { "locked": "4.0.4", "transitive": [ @@ -1783,12 +2659,14 @@ "locked": "3.5.2", "transitive": [ "org.antlr:ST4", - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" ] }, "org.apache.ant:ant": { "locked": "1.9.1", "transitive": [ + "org.apache.hive:hive-common", "org.apache.hive:hive-exec", "org.apache.hive:hive-vector-code-gen" ] @@ -1799,14 +2677,30 @@ "org.apache.ant:ant" ] }, + "org.apache.avro:avro": { + "locked": "1.7.7", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", + "org.apache.hive:hive-serde" + ] + }, "org.apache.commons:commons-compress": { "locked": "1.9", "transitive": [ + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hive:hive-common", "org.apache.hive:hive-exec" ] }, + "org.apache.commons:commons-lang3": { + "locked": "3.1", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, "org.apache.commons:commons-math3": { "locked": "3.1.1", "transitive": [ @@ -1841,6 +2735,12 @@ "org.apache.hadoop:hadoop-common" ] }, + "org.apache.derby:derby": { + "locked": "10.10.2.0", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "org.apache.directory.api:api-asn1-api": { "locked": "1.0.0-M20", "transitive": [ @@ -1865,11 +2765,30 @@ "org.apache.hadoop:hadoop-auth" ] }, + "org.apache.geronimo.specs:geronimo-annotation_1.0_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jaspic_1.0_spec": { + "locked": "1.0", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, + "org.apache.geronimo.specs:geronimo-jta_1.1_spec": { + "locked": "1.1.1", + "transitive": [ + "org.eclipse.jetty.aggregate:jetty-all" + ] + }, "org.apache.hadoop:hadoop-annotations": { "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-mapreduce-client-core", "org.apache.hadoop:hadoop-yarn-api", "org.apache.hadoop:hadoop-yarn-common", "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", @@ -1880,7 +2799,8 @@ "org.apache.hadoop:hadoop-auth": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-common" + "org.apache.hadoop:hadoop-common", + "org.apache.hbase:hbase-client" ] }, "org.apache.hadoop:hadoop-client": { @@ -1889,7 +2809,10 @@ "org.apache.hadoop:hadoop-common": { "locked": "2.7.3", "transitive": [ - "org.apache.hadoop:hadoop-client" + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "org.apache.hadoop:hadoop-client", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" ] }, "org.apache.hadoop:hadoop-hdfs": { @@ -1916,7 +2839,9 @@ "locked": "2.7.3", "transitive": [ "org.apache.hadoop:hadoop-client", - "org.apache.hadoop:hadoop-mapreduce-client-common" + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" ] }, "org.apache.hadoop:hadoop-mapreduce-client-jobclient": { @@ -1964,107 +2889,272 @@ "org.apache.hadoop:hadoop-yarn-server-web-proxy" ] }, - "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { - "locked": "2.7.2", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-common": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-common", + "org.apache.hadoop:hadoop-mapreduce-client-shuffle", + "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", + "org.apache.hadoop:hadoop-yarn-server-nodemanager", + "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hadoop:hadoop-yarn-server-web-proxy" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-nodemanager": { + "locked": "2.7.3", + "transitive": [ + "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23" + ] + }, + "org.apache.hadoop:hadoop-yarn-server-web-proxy": { + "locked": "2.7.2", + "transitive": [ + "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + ] + }, + "org.apache.hbase:hbase-annotations": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common", + "org.apache.hbase:hbase-protocol" + ] + }, + "org.apache.hbase:hbase-client": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hbase:hbase-common": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.apache.hbase:hbase-protocol": { + "locked": "1.1.1", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" + ] + }, + "org.apache.hive.shims:hive-shims-0.23": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive.shims:hive-shims-0.23", + "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive.shims:hive-shims-scheduler": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-shims" + ] + }, + "org.apache.hive:hive-common": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-exec": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-metastore": { + "locked": "2.3.7" + }, + "org.apache.hive:hive-serde": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.apache.hive:hive-service-rpc": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-shims": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-common", + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde" + ] + }, + "org.apache.hive:hive-storage-api": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.apache.hive:hive-vector-code-gen": { + "locked": "2.3.7", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.htrace:htrace-core": { + "locked": "3.1.0-incubating", + "transitive": [ + "org.apache.hadoop:hadoop-common", + "org.apache.hadoop:hadoop-hdfs", + "org.apache.hbase:hbase-client", + "org.apache.hbase:hbase-common" + ] + }, + "org.apache.httpcomponents:httpclient": { + "locked": "4.4.1", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.hadoop:hadoop-auth", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.httpcomponents:httpcore": { + "locked": "4.4.1", + "transitive": [ + "net.java.dev.jets3t:jets3t", + "org.apache.httpcomponents:httpclient", + "org.apache.thrift:libthrift" + ] + }, + "org.apache.ivy:ivy": { + "locked": "2.4.0", + "transitive": [ + "org.apache.hive:hive-exec" + ] + }, + "org.apache.logging.log4j:log4j-1.2-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-common": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-api": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-common", - "org.apache.hadoop:hadoop-mapreduce-client-shuffle", - "org.apache.hadoop:hadoop-yarn-server-applicationhistoryservice", - "org.apache.hadoop:hadoop-yarn-server-nodemanager", - "org.apache.hadoop:hadoop-yarn-server-resourcemanager", - "org.apache.hadoop:hadoop-yarn-server-web-proxy" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-core", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.logging.log4j:log4j-web" ] }, - "org.apache.hadoop:hadoop-yarn-server-nodemanager": { - "locked": "2.7.3", + "org.apache.logging.log4j:log4j-core": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-mapreduce-client-shuffle" + "org.apache.logging.log4j:log4j-1.2-api", + "org.apache.logging.log4j:log4j-web" ] }, - "org.apache.hadoop:hadoop-yarn-server-resourcemanager": { - "locked": "2.7.2", + "org.apache.logging.log4j:log4j-slf4j-impl": { + "locked": "2.6.2", "transitive": [ - "org.apache.hive.shims:hive-shims-0.23" + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-common" ] }, - "org.apache.hadoop:hadoop-yarn-server-web-proxy": { - "locked": "2.7.2", + "org.apache.logging.log4j:log4j-web": { + "locked": "2.6.2", "transitive": [ - "org.apache.hadoop:hadoop-yarn-server-resourcemanager" + "org.apache.hive:hive-common" ] }, - "org.apache.hive.shims:hive-shims-0.23": { - "locked": "2.3.7", + "org.apache.orc:orc-core": { + "locked": "1.3.4", "transitive": [ - "org.apache.hive:hive-shims" + "org.apache.hive:hive-common" ] }, - "org.apache.hive.shims:hive-shims-common": { - "locked": "2.3.7", + "org.apache.parquet:parquet-hadoop-bundle": { + "locked": "1.8.1", "transitive": [ - "org.apache.hive.shims:hive-shims-0.23", - "org.apache.hive.shims:hive-shims-scheduler", - "org.apache.hive:hive-shims" + "org.apache.hive:hive-serde" ] }, - "org.apache.hive.shims:hive-shims-scheduler": { - "locked": "2.3.7", + "org.apache.thrift:libfb303": { + "locked": "0.9.3", "transitive": [ - "org.apache.hive:hive-shims" + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-service-rpc" ] }, - "org.apache.hive:hive-exec": { - "locked": "2.3.7" - }, - "org.apache.hive:hive-shims": { - "locked": "2.3.7", + "org.apache.thrift:libthrift": { + "locked": "0.9.3", "transitive": [ - "org.apache.hive:hive-exec" + "co.cask.tephra:tephra-core", + "org.apache.hive.shims:hive-shims-common", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", + "org.apache.thrift:libfb303" ] }, - "org.apache.hive:hive-vector-code-gen": { - "locked": "2.3.7", + "org.apache.twill:twill-api": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hive:hive-exec" + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper" ] }, - "org.apache.htrace:htrace-core": { - "locked": "3.1.0-incubating", + "org.apache.twill:twill-common": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hadoop:hadoop-common", - "org.apache.hadoop:hadoop-hdfs" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-api", + "org.apache.twill:twill-zookeeper" ] }, - "org.apache.httpcomponents:httpclient": { - "locked": "4.4.1", + "org.apache.twill:twill-core": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hadoop:hadoop-auth", - "org.apache.thrift:libthrift" + "co.cask.tephra:tephra-core" ] }, - "org.apache.httpcomponents:httpcore": { - "locked": "4.4.1", + "org.apache.twill:twill-discovery-api": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.httpcomponents:httpclient", - "org.apache.thrift:libthrift" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-api", + "org.apache.twill:twill-discovery-core" ] }, - "org.apache.ivy:ivy": { - "locked": "2.4.0", + "org.apache.twill:twill-discovery-core": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hive:hive-exec" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core" ] }, - "org.apache.thrift:libthrift": { - "locked": "0.9.3", + "org.apache.twill:twill-zookeeper": { + "locked": "0.6.0-incubating", "transitive": [ - "org.apache.hive.shims:hive-shims-common" + "co.cask.tephra:tephra-core", + "org.apache.twill:twill-core", + "org.apache.twill:twill-discovery-core" ] }, "org.apache.velocity:velocity": { @@ -2084,8 +3174,10 @@ "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-yarn-server-common", "org.apache.hadoop:hadoop-yarn-server-resourcemanager", + "org.apache.hbase:hbase-client", "org.apache.hive.shims:hive-shims-common", - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.twill:twill-zookeeper" ] }, "org.codehaus.groovy:groovy-all": { @@ -2098,6 +3190,7 @@ "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", @@ -2117,9 +3210,11 @@ "locked": "1.9.13", "transitive": [ "com.sun.jersey:jersey-json", + "org.apache.avro:avro", "org.apache.hadoop:hadoop-common", "org.apache.hadoop:hadoop-hdfs", "org.apache.hadoop:hadoop-yarn-common", + "org.apache.hbase:hbase-client", "org.codehaus.jackson:jackson-jaxrs", "org.codehaus.jackson:jackson-xc" ] @@ -2140,10 +3235,41 @@ "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, + "org.datanucleus:datanucleus-api-jdo": { + "locked": "4.2.4", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, "org.datanucleus:datanucleus-core": { "locked": "4.1.17", "transitive": [ - "org.apache.hive:hive-exec" + "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:datanucleus-rdbms": { + "locked": "4.1.19", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.datanucleus:javax.jdo": { + "locked": "3.2.0-m3", + "transitive": [ + "org.apache.hive:hive-metastore" + ] + }, + "org.eclipse.jetty.aggregate:jetty-all": { + "locked": "7.6.0.v20120127", + "transitive": [ + "org.apache.hive:hive-common" + ] + }, + "org.eclipse.jetty.orbit:javax.servlet": { + "locked": "3.0.0.v201112011016", + "transitive": [ + "org.apache.hive:hive-common" ] }, "org.fusesource.leveldbjni:leveldbjni-all": { @@ -2157,9 +3283,49 @@ "org.apache.hadoop:hadoop-yarn-server-resourcemanager" ] }, + "org.hamcrest:hamcrest-core": { + "locked": "1.3", + "transitive": [ + "junit:junit" + ] + }, + "org.jruby.jcodings:jcodings": { + "locked": "1.0.8", + "transitive": [ + "org.apache.hbase:hbase-client", + "org.jruby.joni:joni" + ] + }, + "org.jruby.joni:joni": { + "locked": "2.1.2", + "transitive": [ + "org.apache.hbase:hbase-client" + ] + }, + "org.openjdk.jol:jol-core": { + "locked": "0.2", + "transitive": [ + "io.airlift:slice" + ] + }, + "org.ow2.asm:asm-all": { + "locked": "5.0.2", + "transitive": [ + "org.apache.twill:twill-core" + ] + }, "org.slf4j:slf4j-api": { - "locked": "1.7.12", + "locked": "1.7.21", "transitive": [ + "ch.qos.logback:logback-classic", + "co.cask.tephra:tephra-core", + "com.github.joshelser:dropwizard-metrics-hadoop-metrics2-reporter", + "com.jolbox:bonecp", + "com.zaxxer:HikariCP", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-json", + "io.dropwizard.metrics:metrics-jvm", + "org.apache.avro:avro", "org.apache.curator:curator-client", "org.apache.directory.api:api-asn1-api", "org.apache.directory.api:api-util", @@ -2178,10 +3344,20 @@ "org.apache.hive.shims:hive-shims-0.23", "org.apache.hive.shims:hive-shims-common", "org.apache.hive.shims:hive-shims-scheduler", + "org.apache.hive:hive-common", "org.apache.hive:hive-exec", + "org.apache.hive:hive-metastore", + "org.apache.hive:hive-serde", + "org.apache.hive:hive-service-rpc", "org.apache.hive:hive-shims", + "org.apache.hive:hive-storage-api", "org.apache.hive:hive-vector-code-gen", + "org.apache.logging.log4j:log4j-slf4j-impl", + "org.apache.orc:orc-core", "org.apache.thrift:libthrift", + "org.apache.twill:twill-common", + "org.apache.twill:twill-core", + "org.apache.twill:twill-zookeeper", "org.apache.zookeeper:zookeeper" ] }, @@ -2191,6 +3367,12 @@ "com.google.inject:guice" ] }, + "org.xerial.snappy:snappy-java": { + "locked": "1.0.5", + "transitive": [ + "org.apache.avro:avro" + ] + }, "oro:oro": { "locked": "2.0.8", "transitive": [ @@ -2203,6 +3385,18 @@ "org.apache.hive:hive-exec" ] }, + "tomcat:jasper-compiler": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, + "tomcat:jasper-runtime": { + "locked": "5.5.23", + "transitive": [ + "org.apache.hive:hive-service-rpc" + ] + }, "xerces:xercesImpl": { "locked": "2.9.1", "transitive": [ @@ -3738,8 +4932,7 @@ "org.apache.tez:tez-dag", "org.apache.tez:tez-mapreduce", "org.apache.tez:tez-runtime-library", - "org.apache.velocity:velocity", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.velocity:velocity" ] }, "commons-logging:commons-logging": { @@ -3772,8 +4965,7 @@ "org.apache.hbase:hbase-protocol", "org.apache.hbase:hbase-server", "org.apache.httpcomponents:httpclient", - "org.apache.slider:slider-core", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.slider:slider-core" ] }, "commons-net:commons-net": { @@ -5236,12 +6428,6 @@ "org.apache.twill:twill-core" ] }, - "org.pentaho:pentaho-aggdesigner-algorithm": { - "locked": "5.1.5-jhyde", - "transitive": [ - "org.apache.calcite:calcite-core" - ] - }, "org.reflections:reflections": { "locked": "0.9.8", "transitive": [ @@ -5932,8 +7118,7 @@ "org.apache.tez:tez-dag", "org.apache.tez:tez-mapreduce", "org.apache.tez:tez-runtime-library", - "org.apache.velocity:velocity", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.velocity:velocity" ] }, "commons-logging:commons-logging": { @@ -5963,8 +7148,7 @@ "org.apache.hbase:hbase-protocol", "org.apache.hbase:hbase-server", "org.apache.httpcomponents:httpclient", - "org.apache.slider:slider-core", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.slider:slider-core" ] }, "commons-net:commons-net": { @@ -7342,12 +8526,6 @@ "org.apache.twill:twill-core" ] }, - "org.pentaho:pentaho-aggdesigner-algorithm": { - "locked": "5.1.5-jhyde", - "transitive": [ - "org.apache.calcite:calcite-core" - ] - }, "org.reflections:reflections": { "locked": "0.9.8", "transitive": [ @@ -8056,8 +9234,7 @@ "org.apache.tez:tez-dag", "org.apache.tez:tez-mapreduce", "org.apache.tez:tez-runtime-library", - "org.apache.velocity:velocity", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.velocity:velocity" ] }, "commons-logging:commons-logging": { @@ -8090,8 +9267,7 @@ "org.apache.hbase:hbase-protocol", "org.apache.hbase:hbase-server", "org.apache.httpcomponents:httpclient", - "org.apache.slider:slider-core", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.slider:slider-core" ] }, "commons-net:commons-net": { @@ -9554,12 +10730,6 @@ "org.apache.twill:twill-core" ] }, - "org.pentaho:pentaho-aggdesigner-algorithm": { - "locked": "5.1.5-jhyde", - "transitive": [ - "org.apache.calcite:calcite-core" - ] - }, "org.reflections:reflections": { "locked": "0.9.8", "transitive": [ @@ -10272,8 +11442,7 @@ "org.apache.tez:tez-dag", "org.apache.tez:tez-mapreduce", "org.apache.tez:tez-runtime-library", - "org.apache.velocity:velocity", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.velocity:velocity" ] }, "commons-logging:commons-logging": { @@ -10306,8 +11475,7 @@ "org.apache.hbase:hbase-protocol", "org.apache.hbase:hbase-server", "org.apache.httpcomponents:httpclient", - "org.apache.slider:slider-core", - "org.pentaho:pentaho-aggdesigner-algorithm" + "org.apache.slider:slider-core" ] }, "commons-net:commons-net": { @@ -11770,12 +12938,6 @@ "org.apache.twill:twill-core" ] }, - "org.pentaho:pentaho-aggdesigner-algorithm": { - "locked": "5.1.5-jhyde", - "transitive": [ - "org.apache.calcite:calcite-core" - ] - }, "org.reflections:reflections": { "locked": "0.9.8", "transitive": [ diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java index e37776bad730..e462704efd95 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -19,21 +19,12 @@ package org.apache.iceberg.mr; -import java.net.URI; -import java.net.URISyntaxException; import java.util.function.Function; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.TableScan; import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.common.DynConstructors; import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; public class InputFormatConfig { @@ -54,8 +45,15 @@ private InputFormatConfig() {} public static final String LOCALITY = "iceberg.mr.locality"; public static final String CATALOG = "iceberg.mr.catalog"; - // configuration value set by Hive to contain Table location + public static final String CATALOG_NAME = "iceberg.catalog"; + public static final String HADOOP_CATALOG = "hadoop.catalog"; + public static final String HADOOP_TABLES = "hadoop.tables"; + public static final String HIVE_CATALOG = "hive.catalog"; + public static final String ICEBERG_SNAPSHOTS_TABLE_SUFFIX = ".snapshots"; + public static final String SNAPSHOT_TABLE = "iceberg.snapshots.table"; + public static final String SNAPSHOT_TABLE_SUFFIX = "__snapshots"; public static final String TABLE_LOCATION = "location"; + public static final String TABLE_NAME = "name"; public static class ConfigBuilder { private final Configuration conf; @@ -69,13 +67,6 @@ public ConfigBuilder(Configuration conf) { conf.setBoolean(LOCALITY, false); } - public ConfigBuilder readFrom(String path) { - conf.set(TABLE_PATH, path); - Table table = findTable(conf); - conf.set(TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - return this; - } - public ConfigBuilder filter(Expression expression) { conf.set(FILTER_EXPRESSION, SerializationUtil.serializeToBase64(expression)); return this; @@ -136,68 +127,4 @@ public ConfigBuilder skipResidualFiltering() { } } - public static Table findTable(Configuration conf) { - // TODO: below is naive for Hive, we need to replace it with something more like - // https://github.com/ExpediaGroup/hiveberg/blob/master/src/main/java/com/expediagroup/hiveberg/ - // TableResolverUtil.java - String tableLocation = conf.get(TABLE_LOCATION); - if (tableLocation != null) { - HadoopTables tables = new HadoopTables(conf); - try { - URI location = new URI(tableLocation); - return tables.load(location.getPath()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Unable to create URI for table location: '" + tableLocation + "'", e); - } - } - - String path = conf.get(TABLE_PATH); - Preconditions.checkArgument(path != null, TABLE_PATH + " or " + TABLE_LOCATION + " should not be null"); - if (path.contains("/")) { - HadoopTables tables = new HadoopTables(conf); - return tables.load(path); - } - - String catalogFuncClass = conf.get(InputFormatConfig.CATALOG); - if (catalogFuncClass != null) { - Function catalogFunc = (Function) DynConstructors - .builder(Function.class) - .impl(catalogFuncClass) - .build() - .newInstance(); - Catalog catalog = catalogFunc.apply(conf); - TableIdentifier tableIdentifier = TableIdentifier.parse(path); - return catalog.loadTable(tableIdentifier); - } else { - throw new IllegalArgumentException("No custom catalog specified to load table " + path); - } - } - - public static TableScan createTableScan(Configuration conf, Table table) { - TableScan scan = table.newScan().caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); - long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); - if (snapshotId != -1) { - scan = scan.useSnapshot(snapshotId); - } - long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); - if (asOfTime != -1) { - scan = scan.asOfTime(asOfTime); - } - long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); - if (splitSize > 0) { - scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); - } - String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); - if (schemaStr != null) { - scan.project(SchemaParser.fromJson(schemaStr)); - } - - // TODO add a filter parser to get rid of Serialization - Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); - if (filter != null) { - scan = scan.filter(filter); - } - return scan; - } - } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java new file mode 100644 index 000000000000..46a276d3298f --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.List; +import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; + +import static org.apache.iceberg.expressions.Expressions.and; +import static org.apache.iceberg.expressions.Expressions.equal; +import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.in; +import static org.apache.iceberg.expressions.Expressions.isNull; +import static org.apache.iceberg.expressions.Expressions.lessThan; +import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual; +import static org.apache.iceberg.expressions.Expressions.not; +import static org.apache.iceberg.expressions.Expressions.notNull; +import static org.apache.iceberg.expressions.Expressions.or; + +public class IcebergFilterFactory { + + private IcebergFilterFactory() { + } + + public static Expression generateFilterExpression(SearchArgument sarg) { + List leaves = sarg.getLeaves(); + List childNodes = sarg.getExpression().getChildren(); + + switch (sarg.getExpression().getOperator()) { + case OR: + ExpressionTree orLeft = childNodes.get(0); + ExpressionTree orRight = childNodes.get(1); + return or(translate(orLeft, leaves), translate(orRight, leaves)); + case AND: + ExpressionTree andLeft = childNodes.get(0); + ExpressionTree andRight = childNodes.get(1); + if (childNodes.size() > 2) { + Expression[] evaluatedChildren = getLeftoverLeaves(childNodes, leaves); + return and( + translate(andLeft, leaves), translate(andRight, leaves), evaluatedChildren); + } else { + return and(translate(andLeft, leaves), translate(andRight, leaves)); + } + case NOT: + return not(translateLeaf(sarg.getLeaves().get(0))); + case LEAF: + return translateLeaf(sarg.getLeaves().get(0)); + case CONSTANT: + return null; + default: + throw new IllegalStateException("Unknown operator: " + sarg.getExpression().getOperator()); + } + } + + /** + * Remove first 2 nodes already evaluated and return an array of the evaluated leftover nodes. + * @param allChildNodes All child nodes to be evaluated for the AND expression. + * @param leaves All instances of the leaf nodes. + * @return Array of leftover evaluated nodes. + */ + private static Expression[] getLeftoverLeaves(List allChildNodes, List leaves) { + allChildNodes.remove(0); + allChildNodes.remove(0); + + Expression[] evaluatedLeaves = new Expression[allChildNodes.size()]; + for (int i = 0; i < allChildNodes.size(); i++) { + Expression filter = translate(allChildNodes.get(i), leaves); + evaluatedLeaves[i] = filter; + } + return evaluatedLeaves; + } + + /** + * Recursive method to traverse down the ExpressionTree to evaluate each expression and its leaf nodes. + * @param tree Current ExpressionTree where the 'top' node is being evaluated. + * @param leaves List of all leaf nodes within the tree. + * @return Expression that is translated from the Hive SearchArgument. + */ + private static Expression translate(ExpressionTree tree, List leaves) { + switch (tree.getOperator()) { + case OR: + return or(translate(tree.getChildren().get(0), leaves), + translate(tree.getChildren().get(1), leaves)); + case AND: + if (tree.getChildren().size() > 2) { + Expression[] evaluatedChildren = getLeftoverLeaves(tree.getChildren(), leaves); + return and(translate(tree.getChildren().get(0), leaves), + translate(tree.getChildren().get(1), leaves), evaluatedChildren); + } else { + return and(translate(tree.getChildren().get(0), leaves), + translate(tree.getChildren().get(1), leaves)); + } + case NOT: + return not(translate(tree.getChildren().get(0), leaves)); + case LEAF: + return translateLeaf(leaves.get(tree.getLeaf())); + case CONSTANT: + //We are unsure of how the CONSTANT case works, so using the approach of: + //https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ + // ParquetFilterPredicateConverter.java#L116 + return null; + default: + throw new IllegalStateException("Unknown operator: " + tree.getOperator()); + } + } + + /** + * Translate leaf nodes from Hive operator to Iceberg operator. + * @param leaf Leaf node + * @return Expression fully translated from Hive PredicateLeaf + */ + private static Expression translateLeaf(PredicateLeaf leaf) { + String column = leaf.getColumnName(); + if (column.equals("snapshot__id")) { + return Expressions.alwaysTrue(); + } + switch (leaf.getOperator()) { + case EQUALS: + return equal(column, leaf.getLiteral()); + case NULL_SAFE_EQUALS: + return equal(notNull(column).ref().name(), leaf.getLiteral()); //TODO: Unsure.. + case LESS_THAN: + return lessThan(column, leaf.getLiteral()); + case LESS_THAN_EQUALS: + return lessThanOrEqual(column, leaf.getLiteral()); + case IN: + return in(column, leaf.getLiteralList()); + case BETWEEN: + return and(greaterThanOrEqual(column, leaf.getLiteralList().get(0)), + lessThanOrEqual(column, leaf.getLiteralList().get(1))); + case IS_NULL: + return isNull(column); + default: + throw new IllegalStateException("Unknown operator: " + leaf.getOperator()); + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 7b073051f523..8b987587ff50 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -22,13 +22,19 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.net.URI; import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; @@ -38,12 +44,15 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotsTable; import org.apache.iceberg.Table; -import org.apache.iceberg.TableScan; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.SerializationUtil; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,22 +65,71 @@ public class IcebergInputFormat implements InputFormat, CombineHiveI private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); private Table table; + private long currentSnapshotId; + private String virtualSnapshotIdColumnName; @Override public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { - table = InputFormatConfig.findTable(conf); - TableScan scan = InputFormatConfig.createTableScan(conf, table); - CloseableIterable taskIterable = scan.planTasks(); - List tasks = (List) StreamSupport - .stream(taskIterable.spliterator(), false) - .collect(Collectors.toList()); - return createSplits(tasks, table.location()); + table = TableResolver.resolveTableFromJob(conf); + URI location = TableResolver.pathAsURI(conf.get(InputFormatConfig.TABLE_LOCATION)); + List tasks = planTasks(conf); + return createSplits(tasks, location.toString()); } - private InputSplit[] createSplits(List tasks, String location) { + private List planTasks(JobConf conf) { + // Set defaults for virtual column + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot != null) { + currentSnapshotId = currentSnapshot.snapshotId(); + } + virtualSnapshotIdColumnName = SystemTableUtil.getVirtualColumnName(conf); + + String[] readColumns = ColumnProjectionUtils.getReadColumnNames(conf); + List tasks; + if (conf.get(TableScanDesc.FILTER_EXPR_CONF_STR) == null) { + tasks = Lists.newArrayList(table + .newScan() + .select(readColumns) + .planTasks()); + } else { + ExprNodeGenericFuncDesc exprNodeDesc = SerializationUtilities + .deserializeObject(conf.get(TableScanDesc.FILTER_EXPR_CONF_STR), ExprNodeGenericFuncDesc.class); + SearchArgument sarg = ConvertAstToSearchArg.create(conf, exprNodeDesc); + Expression filter = IcebergFilterFactory.generateFilterExpression(sarg); + + long snapshotIdToScan = extractSnapshotID(conf, exprNodeDesc); + + tasks = Lists.newArrayList(table + .newScan() + .useSnapshot(snapshotIdToScan) + .select(readColumns) + .filter(filter) + .planTasks()); + } + return tasks; + } + + /** + * Search all the leaves of the expression for the 'snapshot_id' column and extract value. + * If snapshot_id column not found, return current table snapshot ID. + */ + private long extractSnapshotID(Configuration conf, ExprNodeGenericFuncDesc exprNodeDesc) { + SearchArgument sarg = ConvertAstToSearchArg.create(conf, exprNodeDesc); + List leaves = sarg.getLeaves(); + for (PredicateLeaf leaf : leaves) { + if (leaf.getColumnName().equals(virtualSnapshotIdColumnName)) { + currentSnapshotId = (long) leaf.getLiteral(); + return (long) leaf.getLiteral(); + } + } + currentSnapshotId = table.currentSnapshot().snapshotId(); + return table.currentSnapshot().snapshotId(); + } + + private InputSplit[] createSplits(List tasks, String name) { InputSplit[] splits = new InputSplit[tasks.size()]; for (int i = 0; i < tasks.size(); i++) { - splits[i] = new IcebergSplit(tasks.get(i), location); + splits[i] = new IcebergSplit(tasks.get(i), name); } return splits; } @@ -115,23 +173,32 @@ private void nextTask() { recordIterator = reader.iterator(); } + private Record resolveAppropriateRecordForTableType() { + if (table instanceof SnapshotsTable) { + return currentRecord; + } else { + return SystemTableUtil.recordWithVirtualColumn(currentRecord, currentSnapshotId, table.schema(), + virtualSnapshotIdColumnName); + } + } + @Override public boolean next(Void key, IcebergWritable value) { if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); - value.setRecord(currentRecord); + value.setRecord(resolveAppropriateRecordForTableType()); return true; } if (tasks.hasNext()) { - try { + /*try { reader.close(); } catch (IOException e) { LOG.error("Error closing reader", e); - } + }*/ nextTask(); currentRecord = recordIterator.next(); - value.setRecord(currentRecord); + value.setRecord(resolveAppropriateRecordForTableType()); return true; } return false; @@ -146,7 +213,11 @@ public Void createKey() { public IcebergWritable createValue() { IcebergWritable record = new IcebergWritable(); record.setRecord(currentRecord); - record.setSchema(table.schema()); + if (table instanceof SnapshotsTable) { + record.setSchema(table.schema()); + } else { + record.setSchema(SystemTableUtil.schemaWithVirtualColumn(table.schema(), virtualSnapshotIdColumnName)); + } return record; } @@ -172,8 +243,6 @@ public float getProgress() throws IOException { */ private static class IcebergSplit extends FileSplit { - private static final String[] ANYWHERE = new String[]{"*"}; - private CombinedScanTask task; private String partitionLocation; @@ -192,7 +261,7 @@ public long getLength() { @Override public String[] getLocations() throws IOException { - return ANYWHERE; + return new String[0]; } @Override @@ -207,13 +276,13 @@ public long getStart() { @Override public void write(DataOutput out) throws IOException { - byte[] data = SerializationUtil.serializeToBytes(this.task); - out.writeInt(data.length); - out.write(data); + byte[] dataTask = SerializationUtil.serializeToBytes(this.task); + out.writeInt(dataTask.length); + out.write(dataTask); - byte[] tableLocation = SerializationUtil.serializeToBytes(this.partitionLocation); - out.writeInt(tableLocation.length); - out.write(tableLocation); + byte[] tableName = SerializationUtil.serializeToBytes(this.partitionLocation); + out.writeInt(tableName.length); + out.write(tableName); } @Override @@ -222,9 +291,9 @@ public void readFields(DataInput in) throws IOException { in.readFully(data); this.task = SerializationUtil.deserializeFromBytes(data); - byte[] location = new byte[in.readInt()]; - in.readFully(location); - this.partitionLocation = SerializationUtil.deserializeFromBytes(location); + byte[] name = new byte[in.readInt()]; + in.readFully(name); + this.partitionLocation = SerializationUtil.deserializeFromBytes(name); } public CombinedScanTask getTask() { diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java new file mode 100644 index 000000000000..17f5b9fb07b2 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Types; + +public class SystemTableUtil { + + static final String VIRTUAL_COLUMN_NAME = "iceberg.hive.snapshot.virtual.column.name"; + + private static final String DEFAULT_SNAPSHOT_ID_COLUMN_NAME = "snapshot__id"; + + private SystemTableUtil() {} + + protected static Schema schemaWithVirtualColumn(Schema schema, String columnName) { + List columns = new ArrayList<>(schema.columns()); + columns.add(Types.NestedField.optional(Integer.MAX_VALUE, columnName, Types.LongType.get())); + return new Schema(columns); + } + + protected static Record recordWithVirtualColumn(Record record, long snapshotId, Schema oldSchema, + String columnName) { + Schema newSchema = schemaWithVirtualColumn(oldSchema, columnName); + Record newRecord = GenericRecord.create(newSchema); + for (Types.NestedField field : oldSchema.columns()) { + newRecord.setField(field.name(), record.getField(field.name())); + } + newRecord.setField(columnName, snapshotId); + return newRecord; + } + + protected static String getVirtualColumnName(Configuration conf) { + String virtualColumnName = conf.get(VIRTUAL_COLUMN_NAME); + if (virtualColumnName == null) { + return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; + } else { + return virtualColumnName; + } + } + + protected static String getVirtualColumnName(Properties properties) { + String virtualColumnName = properties.getProperty(VIRTUAL_COLUMN_NAME); + if (virtualColumnName == null) { + return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; + } else { + return virtualColumnName; + } + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java new file mode 100644 index 000000000000..b6e2d879cbe7 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Properties; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; + +final class TableResolver { + + private TableResolver() { + } + + static Table resolveTableFromJob(JobConf conf) throws IOException { + Properties properties = new Properties(); + properties.setProperty(InputFormatConfig.CATALOG_NAME, extractProperty(conf, InputFormatConfig.CATALOG_NAME)); + if (conf.get(InputFormatConfig.CATALOG_NAME).equals(InputFormatConfig.HADOOP_CATALOG)) { + properties.setProperty(InputFormatConfig.SNAPSHOT_TABLE, conf.get(InputFormatConfig.SNAPSHOT_TABLE, "true")); + } + properties.setProperty(InputFormatConfig.TABLE_LOCATION, extractProperty(conf, InputFormatConfig.TABLE_LOCATION)); + properties.setProperty(InputFormatConfig.TABLE_NAME, extractProperty(conf, InputFormatConfig.TABLE_NAME)); + return resolveTableFromConfiguration(conf, properties); + } + + static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { + String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME); + URI tableLocation = pathAsURI(properties.getProperty(InputFormatConfig.TABLE_LOCATION)); + if (catalogName == null) { + throw new IllegalArgumentException("Catalog property: 'iceberg.catalog' not set in JobConf"); + } + switch (catalogName) { + case InputFormatConfig.HADOOP_TABLES: + HadoopTables tables = new HadoopTables(conf); + return tables.load(tableLocation.getPath()); + case InputFormatConfig.HADOOP_CATALOG: + String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); + TableIdentifier id = TableIdentifier.parse(tableName); + if (tableName.endsWith(InputFormatConfig.SNAPSHOT_TABLE_SUFFIX)) { + if (!Boolean.parseBoolean(properties.getProperty(InputFormatConfig.SNAPSHOT_TABLE, + Boolean.TRUE.toString()))) { + String tablePath = id.toString().replaceAll("\\.", "/"); + URI warehouseLocation = pathAsURI(tableLocation.getPath().replaceAll(tablePath, "")); + HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); + return catalog.loadTable(id); + } else { + return resolveMetadataTable(conf, tableLocation.getPath(), tableName); + } + } else { + URI warehouseLocation = pathAsURI(extractWarehousePath(tableLocation.getPath(), tableName)); + HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); + return catalog.loadTable(id); + } + case InputFormatConfig.HIVE_CATALOG: + //TODO Implement HiveCatalog + return null; + } + return null; + } + + static Table resolveMetadataTable(Configuration conf, String location, String tableName) throws IOException { + URI warehouseLocation = pathAsURI(extractWarehousePath(location, tableName)); + HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); + String baseTableName = StringUtils.removeEnd(tableName, InputFormatConfig.SNAPSHOT_TABLE_SUFFIX); + + TableIdentifier snapshotsId = TableIdentifier.parse(baseTableName + + InputFormatConfig.ICEBERG_SNAPSHOTS_TABLE_SUFFIX); + return catalog.loadTable(snapshotsId); + } + + static URI pathAsURI(String path) throws IOException { + if (path == null) { + throw new IllegalArgumentException("Path is null."); + } + try { + return new URI(path); + } catch (URISyntaxException e) { + throw new IOException("Unable to create URI for table location: '" + path + "'", e); + } + } + + protected static String extractProperty(JobConf conf, String key) { + String value = conf.get(key); + if (value == null) { + throw new IllegalArgumentException("Property not set in JobConf: " + key); + } + return value; + } + + protected static String extractWarehousePath(String location, String tableName) { + String tablePath = tableName.replaceAll("\\.", "/").replaceAll( + InputFormatConfig.SNAPSHOT_TABLE_SUFFIX, ""); + return location.replaceAll(tablePath, ""); + } + +} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java index 901404571241..11035f57b210 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java @@ -43,6 +43,7 @@ import org.apache.iceberg.SchemaParser; import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; @@ -97,8 +98,8 @@ public List getSplits(JobContext context) { } Configuration conf = context.getConfiguration(); - Table table = InputFormatConfig.findTable(conf); - TableScan scan = InputFormatConfig.createTableScan(conf, table); + Table table = TableResolver.findTable(conf); + TableScan scan = createTableScan(conf, table); splits = Lists.newArrayList(); boolean applyResidual = !conf.getBoolean(InputFormatConfig.SKIP_RESIDUAL_FILTERING, false); @@ -118,6 +119,33 @@ public List getSplits(JobContext context) { return splits; } + private TableScan createTableScan(Configuration conf, Table table) { + TableScan scan = table.newScan().caseSensitive(conf.getBoolean(InputFormatConfig.CASE_SENSITIVE, true)); + long snapshotId = conf.getLong(InputFormatConfig.SNAPSHOT_ID, -1); + if (snapshotId != -1) { + scan = scan.useSnapshot(snapshotId); + } + long asOfTime = conf.getLong(InputFormatConfig.AS_OF_TIMESTAMP, -1); + if (asOfTime != -1) { + scan = scan.asOfTime(asOfTime); + } + long splitSize = conf.getLong(InputFormatConfig.SPLIT_SIZE, 0); + if (splitSize > 0) { + scan = scan.option(TableProperties.SPLIT_SIZE, String.valueOf(splitSize)); + } + String schemaStr = conf.get(InputFormatConfig.READ_SCHEMA); + if (schemaStr != null) { + scan.project(SchemaParser.fromJson(schemaStr)); + } + + // TODO add a filter parser to get rid of Serialization + Expression filter = SerializationUtil.deserializeFromBase64(conf.get(InputFormatConfig.FILTER_EXPRESSION)); + if (filter != null) { + scan = scan.filter(filter); + } + return scan; + } + private static void checkResiduals(CombinedScanTask task) { task.files().forEach(fileScanTask -> { Expression residual = fileScanTask.residual(); diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java new file mode 100644 index 000000000000..88f2b2e8bc29 --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapreduce; + +import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.common.DynConstructors; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +final class TableResolver { + + private TableResolver() { + } + + public static Table findTable(Configuration conf) { + String path = conf.get(InputFormatConfig.TABLE_PATH); + System.out.println("YYY PATH : " + path); + Preconditions.checkArgument(path != null, "Table path should not be null"); + if (path.contains("/")) { + HadoopTables tables = new HadoopTables(conf); + return tables.load(path); + } + + String catalogFuncClass = conf.get(InputFormatConfig.CATALOG); + if (catalogFuncClass != null) { + Function catalogFunc = (Function) + DynConstructors.builder(Function.class) + .impl(catalogFuncClass) + .build() + .newInstance(); + Catalog catalog = catalogFunc.apply(conf); + TableIdentifier tableIdentifier = TableIdentifier.parse(path); + return catalog.loadTable(tableIdentifier); + } else { + throw new IllegalArgumentException("No custom catalog specified to load table " + path); + } + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java b/mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java deleted file mode 100644 index 23da724e3888..000000000000 --- a/mr/src/test/java/org/apache/iceberg/mr/BaseInputFormatTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr; - -import java.io.File; -import java.io.IOException; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.data.RandomGenericData; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.mr.TestHelpers.Row; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import static org.apache.iceberg.mr.TestHelpers.writeFile; -import static org.apache.iceberg.types.Types.NestedField.required; - - -@RunWith(Parameterized.class) -public abstract class BaseInputFormatTest { - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Parameterized.Parameters - public static Object[][] parameters() { - return new Object[][]{ - new Object[]{"parquet"}, - new Object[]{"avro"}, - new Object[]{"orc"} - }; - } - - protected static final Schema SCHEMA = new Schema( - required(1, "data", Types.StringType.get()), - required(2, "id", Types.LongType.get()), - required(3, "date", Types.StringType.get())); - - protected static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) - .identity("date") - .bucket("id", 1) - .build(); - - protected Configuration conf = new Configuration(); - protected HadoopTables tables = new HadoopTables(conf); - - protected FileFormat fileFormat; - - protected abstract void runAndValidate(File tableLocation, List expectedRecords) throws IOException; - - @Test - public void testUnpartitionedTable() throws Exception { - File tableLocation = temp.newFolder(fileFormat.name()); - Table table = tables - .create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), tableLocation.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, null, fileFormat, expectedRecords); - table.newAppend().appendFile(dataFile).commit(); - runAndValidate(tableLocation, expectedRecords); - } - - @Test - public void testPartitionedTable() throws Exception { - File tableLocation = temp.newFolder(fileFormat.name()); - Assert.assertTrue(tableLocation.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - tableLocation.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - - runAndValidate(tableLocation, expectedRecords); - } - -} diff --git a/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java b/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java index bddddd6afefc..8db577f00ed4 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java +++ b/mr/src/test/java/org/apache/iceberg/mr/TestHelpers.java @@ -26,9 +26,11 @@ import org.apache.iceberg.DataFiles; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; +import org.apache.iceberg.Schema; import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.avro.DataWriter; import org.apache.iceberg.data.orc.GenericOrcWriter; @@ -36,10 +38,11 @@ import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.orc.ORC; import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.types.Types; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; -/** - * - */ public class TestHelpers { private TestHelpers() {} @@ -125,4 +128,26 @@ public static DataFile writeFile(File targetFile, return builder.build(); } + /** + * Based on: https://github.com/apache/incubator-iceberg/blob/master/ + * spark/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java + */ + public static Record createSimpleRecord(long id, String data) { + Schema schema = new Schema(required(1, "id", Types.StringType.get()), + optional(2, "data", Types.LongType.get())); + GenericRecord record = GenericRecord.create(schema); + record.setField("id", id); + record.setField("data", data); + return record; + } + + public static Record createCustomRecord(Schema schema, List dataValues) { + GenericRecord record = GenericRecord.create(schema); + List fields = schema.columns(); + for (int i = 0; i < fields.size(); i++) { + record.setField(fields.get(i).name(), dataValues.get(i)); + } + return record; + } + } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java new file mode 100644 index 000000000000..f6d53379fd7b --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; +import org.apache.iceberg.expressions.And; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Not; +import org.apache.iceberg.expressions.Or; +import org.apache.iceberg.expressions.UnboundPredicate; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestIcebergFilterFactory { + + @Test + public void testEqualsOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startAnd().equals("salary", PredicateLeaf.Type.LONG, 3000L).end().build(); + + UnboundPredicate expected = Expressions.equal("salary", 3000L); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.literal(), expected.literal()); + assertEquals(actual.ref().name(), expected.ref().name()); + } + + @Test + public void testNotEqualsOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startNot().equals("salary", PredicateLeaf.Type.LONG, 3000L).end().build(); + + Not expected = (Not) Expressions.not(Expressions.equal("salary", 3000L)); + Not actual = (Not) IcebergFilterFactory.generateFilterExpression(arg); + + UnboundPredicate childExpressionActual = (UnboundPredicate) actual.child(); + UnboundPredicate childExpressionExpected = Expressions.equal("salary", 3000L); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.child().op(), expected.child().op()); + assertEquals(childExpressionActual.ref().name(), childExpressionExpected.ref().name()); + assertEquals(childExpressionActual.literal(), childExpressionExpected.literal()); + } + + @Test + public void testLessThanOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startAnd().lessThan("salary", PredicateLeaf.Type.LONG, 3000L).end().build(); + + UnboundPredicate expected = Expressions.lessThan("salary", 3000L); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.literal(), expected.literal()); + assertEquals(actual.ref().name(), expected.ref().name()); + } + + @Test + public void testLessThanEqualsOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startAnd().lessThanEquals("salary", PredicateLeaf.Type.LONG, 3000L).end().build(); + + UnboundPredicate expected = Expressions.lessThanOrEqual("salary", 3000L); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.literal(), expected.literal()); + assertEquals(actual.ref().name(), expected.ref().name()); + } + + @Test + public void testInOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startAnd().in("salary", PredicateLeaf.Type.LONG, 3000L, 4000L).end().build(); + + UnboundPredicate expected = Expressions.in("salary", 3000L, 4000L); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.literals(), expected.literals()); + assertEquals(actual.ref().name(), expected.ref().name()); + } + + @Test + public void testBetweenOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startAnd() + .between("salary", PredicateLeaf.Type.LONG, 3000L, 4000L).end().build(); + + And expected = (And) Expressions.and(Expressions.greaterThanOrEqual("salary", 3000L), + Expressions.lessThanOrEqual("salary", 3000L)); + And actual = (And) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.left().op(), expected.left().op()); + assertEquals(actual.right().op(), expected.right().op()); + } + + @Test + public void testIsNullOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startAnd().isNull("salary", PredicateLeaf.Type.LONG).end().build(); + + UnboundPredicate expected = Expressions.isNull("salary"); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.ref().name(), expected.ref().name()); + } + + @Test + public void testAndOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startAnd() + .equals("salary", PredicateLeaf.Type.LONG, 3000L) + .equals("salary", PredicateLeaf.Type.LONG, 4000L) + .end().build(); + + And expected = (And) Expressions + .and(Expressions.equal("salary", 3000L), Expressions.equal("salary", 4000L)); + And actual = (And) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.left().op(), expected.left().op()); + assertEquals(actual.right().op(), expected.right().op()); + } + + @Test + public void tesOrOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startOr() + .equals("salary", PredicateLeaf.Type.LONG, 3000L) + .equals("salary", PredicateLeaf.Type.LONG, 4000L) + .end().build(); + + Or expected = (Or) Expressions + .or(Expressions.equal("salary", 3000L), Expressions.equal("salary", 4000L)); + Or actual = (Or) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.left().op(), expected.left().op()); + assertEquals(actual.right().op(), expected.right().op()); + } + + @Test + public void testManyAndOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startAnd() + .equals("salary", PredicateLeaf.Type.LONG, 3000L) + .equals("job", PredicateLeaf.Type.LONG, 4000L) + .equals("name", PredicateLeaf.Type.LONG, 9000L) + .end() + .build(); + + And expected = (And) Expressions.and( + Expressions.equal("salary", 3000L), + Expressions.equal("job", 4000L), + Expressions.equal("name", 9000L)); + + And actual = (And) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.right().op(), expected.right().op()); + assertEquals(actual.left().op(), expected.left().op()); + } +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 02c7e568a3a2..9c71aeafc1dc 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -19,56 +19,123 @@ package org.apache.iceberg.mr.mapred; +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.StandaloneHiveRunner; +import com.klarna.hiverunner.annotations.HiveSQL; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; +import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; import org.apache.iceberg.data.Record; -import org.apache.iceberg.mr.BaseInputFormatTest; +import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; -import org.junit.Assert; +import org.apache.iceberg.mr.TestHelpers; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; -public class TestIcebergInputFormat extends BaseInputFormatTest { +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; - private static final Logger LOG = LoggerFactory.getLogger(TestIcebergInputFormat.class); +@RunWith(StandaloneHiveRunner.class) +public class TestIcebergInputFormat { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @HiveSQL(files = {}, autoStart = true) + private HiveShell shell; private IcebergInputFormat inputFormat = new IcebergInputFormat(); + private File tableLocation; + private JobConf conf = new JobConf(); + + @Before + public void before() throws IOException { + tableLocation = temp.newFolder(); + Schema schema = new Schema(required(1, "id", Types.LongType.get()), + optional(2, "data", Types.StringType.get())); + PartitionSpec spec = PartitionSpec.unpartitioned(); + + HadoopTables tables = new HadoopTables(); + Table table = tables.create(schema, spec, tableLocation.getAbsolutePath()); + + List data = new ArrayList<>(); + data.add(TestHelpers.createSimpleRecord(1L, "Michael")); + data.add(TestHelpers.createSimpleRecord(2L, "Andy")); + data.add(TestHelpers.createSimpleRecord(3L, "Berta")); - public TestIcebergInputFormat(String fileFormat) { - this.fileFormat = FileFormat.valueOf(fileFormat.toUpperCase(Locale.ENGLISH)); + DataFile fileA = TestHelpers.writeFile(temp.newFile(), table, null, FileFormat.PARQUET, data); + table.newAppend().appendFile(fileA).commit(); + } + + @Test + public void testGetSplits() throws IOException { + IcebergInputFormat format = new IcebergInputFormat(); + conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + conf.set(InputFormatConfig.TABLE_NAME, "source_db.table_a"); + InputSplit[] splits = format.getSplits(conf, 1); + assertEquals(splits.length, 1); } @Test(expected = IllegalArgumentException.class) public void testGetSplitsNoLocation() throws IOException { - JobConf jobConf = new JobConf(); - inputFormat.getSplits(jobConf, 1); + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + conf.set(InputFormatConfig.TABLE_NAME, "source_db.table_a"); + inputFormat.getSplits(conf, 1); } @Test(expected = IllegalArgumentException.class) - public void testGetSplitsInvalidLocationUri() throws IOException { - JobConf jobConf = new JobConf(); - jobConf.set(InputFormatConfig.TABLE_LOCATION, "http:"); - inputFormat.getSplits(jobConf, 1); + public void testGetSplitsNoCatalog() throws IOException { + conf.set(InputFormatConfig.TABLE_LOCATION, "file:" + tableLocation.getAbsolutePath()); + conf.set(InputFormatConfig.TABLE_NAME, "source_db.table_a"); + inputFormat.getSplits(conf, 1); } - @Override - protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { - JobConf jobConf = new JobConf(); - jobConf.set(InputFormatConfig.TABLE_LOCATION, "file:" + tableLocation); - validate(jobConf, expectedRecords); + @Test(expected = IllegalArgumentException.class) + public void testGetSplitsNoName() throws IOException { + conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + conf.set(InputFormatConfig.TABLE_LOCATION, "file:" + tableLocation.getAbsolutePath()); + inputFormat.getSplits(conf, 1); } - private void validate(JobConf jobConf, List expectedRecords) throws IOException { - List actualRecords = readRecords(jobConf); - Assert.assertEquals(expectedRecords, actualRecords); + @Ignore("Requires SerDe") + @Test + public void testInputFormat() { + shell.execute("CREATE DATABASE source_db"); + shell.execute(new StringBuilder() + .append("CREATE TABLE source_db.table_a ") + .append("ROW FORMAT SERDE 'org.apache.iceberg.mr.mapred.IcebergSerDe' ") + .append("STORED AS ") + .append("INPUTFORMAT 'org.apache.iceberg.mr.mapred.IcebergInputFormat' ") + .append("OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' ") + .append("LOCATION '") + .append(tableLocation.getAbsolutePath()) + .append("' TBLPROPERTIES ('iceberg.catalog'='hadoop.tables'") + .append(")") + .toString()); + + List result = shell.executeStatement("SELECT id, data FROM source_db.table_a"); + + assertEquals(3, result.size()); + assertArrayEquals(new Object[]{1L, "Michael"}, result.get(0)); + assertArrayEquals(new Object[]{2L, "Andy"}, result.get(1)); + assertArrayEquals(new Object[]{3L, "Berta"}, result.get(2)); } private List readRecords(JobConf jobConf) throws IOException { diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java new file mode 100644 index 000000000000..e4b9c87a51c9 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred; + +import org.apache.hadoop.mapred.JobConf; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestTableResolver { + + @Test + public void extractWarehouseLocationRegularTable() { + // This is the style of input expected from HiveConf + String testLocation = "some/folder/database/table_a"; + String testTableName = "database.table_a"; + + String expected = "some/folder/"; + String result = TableResolver.extractWarehousePath(testLocation, testTableName); + + assertEquals(expected, result); + } + + @Test + public void extractPropertyFromJobConf() { + JobConf conf = new JobConf(); + String key = "iceberg.catalog"; + String value = "hadoop.tables"; + + conf.set(key, value); + + String result = TableResolver.extractProperty(conf, key); + + assertEquals(value, result); + } + + @Test(expected = IllegalArgumentException.class) + public void extractNonExistentProperty() { + JobConf conf = new JobConf(); + String key = "iceberg.catalog"; + + TableResolver.extractProperty(conf, key); + } + +} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index b3187dafcf46..4999df86538e 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -39,6 +39,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.Catalog; @@ -47,7 +48,7 @@ import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.mr.BaseInputFormatTest; +import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.TestHelpers.Row; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; @@ -57,30 +58,96 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import static org.apache.iceberg.mr.TestHelpers.writeFile; +import static org.apache.iceberg.types.Types.NestedField.required; +@Ignore("") @RunWith(Parameterized.class) -public class TestIcebergInputFormat extends BaseInputFormatTest { +public class TestIcebergInputFormat { + + static final Schema SCHEMA = new Schema( + required(1, "data", Types.StringType.get()), + required(2, "id", Types.LongType.get()), + required(3, "date", Types.StringType.get())); + + static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) + .identity("date") + .bucket("id", 1) + .build(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + private HadoopTables tables; + private Configuration conf; + private final FileFormat fileFormat; + + @Parameterized.Parameters + public static Object[][] parameters() { + return new Object[][]{ + new Object[]{"parquet"}, + new Object[]{"avro"}, + new Object[]{"orc"} + }; + } public TestIcebergInputFormat(String format) { this.fileFormat = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); } - @Override + @Before + public void before() { + conf = new Configuration(); + tables = new HadoopTables(conf); + } + + private void readFrom(String path) { + conf.set(InputFormatConfig.TABLE_PATH, path); + System.out.println("XXX PATH " + InputFormatConfig.TABLE_PATH + " : " + path); + Table table = TableResolver.findTable(conf); + conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + } + protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { Job job = Job.getInstance(conf); + readFrom(tableLocation.toString()); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(tableLocation.toString()); validate(job, expectedRecords); } - //TODO: try move as many methods below into base class (once functionality is implemented in - // mapred InputFormat) + @Test + public void testUnpartitionedTable() throws Exception { + File tableLocation = temp.newFolder(fileFormat.name()); + Table table = tables + .create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), tableLocation.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + DataFile dataFile = writeFile(temp.newFile(), table, null, fileFormat, expectedRecords); + table.newAppend().appendFile(dataFile).commit(); + runAndValidate(tableLocation, expectedRecords); + } + + @Test + public void testPartitionedTable() throws Exception { + File tableLocation = temp.newFolder(fileFormat.name()); + Assert.assertTrue(tableLocation.delete()); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), + tableLocation.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + expectedRecords.get(0).set(2, "2020-03-20"); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + + runAndValidate(tableLocation, expectedRecords); + } + @Test public void testFilterExp() throws Exception { File location = temp.newFolder(fileFormat.name()); @@ -100,8 +167,8 @@ public void testFilterExp() throws Exception { .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()) - .filter(Expressions.equal("date", "2020-03-20")); + readFrom(location.toString()); + configBuilder.filter(Expressions.equal("date", "2020-03-20")); validate(job, expectedRecords); } @@ -130,8 +197,8 @@ public void testResiduals() throws Exception { .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()) - .filter(Expressions.and( + readFrom(location.toString()); + configBuilder.filter(Expressions.and( Expressions.equal("date", "2020-03-20"), Expressions.equal("id", 123))); validate(job, expectedRecords); @@ -139,7 +206,8 @@ public void testResiduals() throws Exception { // skip residual filtering job = Job.getInstance(conf); configBuilder = IcebergInputFormat.configure(job); - configBuilder.skipResidualFiltering().readFrom(location.toString()) + readFrom(location.toString()); + configBuilder.skipResidualFiltering() .filter(Expressions.and( Expressions.equal("date", "2020-03-20"), Expressions.equal("id", 123))); @@ -162,9 +230,8 @@ public void testProjection() throws Exception { Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .readFrom(location.toString()) - .project(projectedSchema); + readFrom(location.toString()); + configBuilder.project(projectedSchema); List outputRecords = readRecords(job.getConfiguration()); Assert.assertEquals(inputRecords.size(), outputRecords.size()); Assert.assertEquals(projectedSchema.asStruct(), outputRecords.get(0).struct()); @@ -237,9 +304,8 @@ private void validateIdentityPartitionProjections( String tablePath, Schema projectedSchema, List inputRecords) throws Exception { Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .readFrom(tablePath) - .project(projectedSchema); + readFrom(tablePath); + configBuilder.project(projectedSchema); List actualRecords = readRecords(job.getConfiguration()); Set fieldNames = TypeUtil.indexByName(projectedSchema.asStruct()).keySet(); @@ -273,9 +339,8 @@ public void testSnapshotReads() throws Exception { Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .readFrom(location.toString()) - .snapshotId(snapshotId); + readFrom(location.toString()); + configBuilder.snapshotId(snapshotId); validate(job, expectedRecords); } @@ -293,8 +358,7 @@ public void testLocality() throws Exception { .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); - + readFrom(location.toString()); for (InputSplit split : splits(job.getConfiguration())) { Assert.assertArrayEquals(IcebergInputFormat.IcebergSplit.ANYWHERE, split.getLocations()); } @@ -330,9 +394,8 @@ public void testCustomCatalog() throws Exception { Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .catalogFunc(HadoopCatalogFunc.class) - .readFrom(tableIdentifier.toString()); + readFrom(tableIdentifier.toString()); + configBuilder.catalogFunc(HadoopCatalogFunc.class); validate(job, expectedRecords); } From 382b3bb8bff07e53e24c06ca5edfe772e0d6ec5f Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 8 Jun 2020 16:33:03 +0100 Subject: [PATCH 42/51] debugging failing tests --- .../mr/mapreduce/TestIcebergInputFormat2.java | 484 ++++++++++++++++++ 1 file changed, 484 insertions(+) create mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java new file mode 100644 index 000000000000..10a443bc91d7 --- /dev/null +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java @@ -0,0 +1,484 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapreduce; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; +import org.apache.iceberg.*; +import org.apache.iceberg.TestHelpers.Row; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.RandomGenericData; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.avro.DataWriter; +import org.apache.iceberg.data.orc.GenericOrcWriter; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.orc.ORC; +import org.apache.iceberg.parquet.Parquet; +import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.apache.iceberg.mr.TestHelpers.writeFile; + +@RunWith(Parameterized.class) +public class TestIcebergInputFormat2 { + static final Schema SCHEMA = new Schema( + required(1, "data", Types.StringType.get()), + required(2, "id", Types.LongType.get()), + required(3, "date", Types.StringType.get())); + + static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) + .identity("date") + .bucket("id", 1) + .build(); + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + private HadoopTables tables; + private Configuration conf; + + @Parameterized.Parameters + public static Object[][] parameters() { + return new Object[][]{ + new Object[]{"parquet"}, + new Object[]{"avro"}, + new Object[]{"orc"} + }; + } + + private final FileFormat format; + + public TestIcebergInputFormat2(String format) { + this.format = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); + } + + @Before + public void before() { + conf = new Configuration(); + tables = new HadoopTables(conf); + } + + private void readFrom(String path) { + conf.set(InputFormatConfig.TABLE_PATH, path); + System.out.println("XXX PATH " + InputFormatConfig.TABLE_PATH + " : " + path); + Table table = TableResolver.findTable(conf); + conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema())); + } + + @Test + public void testUnpartitionedTable() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + DataFile dataFile = writeFile(temp.newFile(), table, null, format, expectedRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + readFrom(location.toString()); + validate(job, expectedRecords); + } +/* + @Test + public void testPartitionedTable() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + expectedRecords.get(0).set(2, "2020-03-20"); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()); + validate(job, expectedRecords); + } + + @Test + public void testFilterExp() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); + expectedRecords.get(0).set(2, "2020-03-20"); + expectedRecords.get(1).set(2, "2020-03-20"); + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, + RandomGenericData.generate(table.schema(), 2, 0L)); + table.newAppend() + .appendFile(dataFile1) + .appendFile(dataFile2) + .commit(); + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()) + .filter(Expressions.equal("date", "2020-03-20")); + validate(job, expectedRecords); + } + + @Test + public void testResiduals() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List writeRecords = RandomGenericData.generate(table.schema(), 2, 0L); + writeRecords.get(0).set(1, 123L); + writeRecords.get(0).set(2, "2020-03-20"); + writeRecords.get(1).set(1, 456L); + writeRecords.get(1).set(2, "2020-03-20"); + + List expectedRecords = new ArrayList<>(); + expectedRecords.add(writeRecords.get(0)); + + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, writeRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, + RandomGenericData.generate(table.schema(), 2, 0L)); + table.newAppend() + .appendFile(dataFile1) + .appendFile(dataFile2) + .commit(); + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); + validate(job, expectedRecords); + + // skip residual filtering + job = Job.getInstance(conf); + configBuilder = IcebergInputFormat.configure(job); + configBuilder.skipResidualFiltering().readFrom(location.toString()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); + validate(job, writeRecords); + } + + @Test + public void testFailedResidualFiltering() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); + expectedRecords.get(0).set(2, "2020-03-20"); + expectedRecords.get(1).set(2, "2020-03-20"); + + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + table.newAppend() + .appendFile(dataFile1) + .commit(); + + Job jobShouldFail1 = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); + configBuilder.useHiveRows().readFrom(location.toString()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 0))); + AssertHelpers.assertThrows( + "Residuals are not evaluated today for Iceberg Generics In memory model of HIVE", + UnsupportedOperationException.class, "Filter expression ref(name=\"id\") == 0 is not completely satisfied.", + () -> validate(jobShouldFail1, expectedRecords)); + + Job jobShouldFail2 = Job.getInstance(conf); + configBuilder = IcebergInputFormat.configure(jobShouldFail2); + configBuilder.usePigTuples().readFrom(location.toString()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 0))); + AssertHelpers.assertThrows( + "Residuals are not evaluated today for Iceberg Generics In memory model of PIG", + UnsupportedOperationException.class, "Filter expression ref(name=\"id\") == 0 is not completely satisfied.", + () -> validate(jobShouldFail2, expectedRecords)); + } + + @Test + public void testProjection() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Schema projectedSchema = TypeUtil.select(SCHEMA, ImmutableSet.of(1)); + Table table = tables.create(SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List inputRecords = RandomGenericData.generate(table.schema(), 1, 0L); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, inputRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder + .readFrom(location.toString()) + .project(projectedSchema); + List outputRecords = readRecords(job.getConfiguration()); + Assert.assertEquals(inputRecords.size(), outputRecords.size()); + Assert.assertEquals(projectedSchema.asStruct(), outputRecords.get(0).struct()); + } + + private static final Schema LOG_SCHEMA = new Schema( + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "date", Types.StringType.get()), + Types.NestedField.optional(3, "level", Types.StringType.get()), + Types.NestedField.optional(4, "message", Types.StringType.get()) + ); + + private static final PartitionSpec IDENTITY_PARTITION_SPEC = + PartitionSpec.builderFor(LOG_SCHEMA).identity("date").identity("level").build(); + + @Test + public void testIdentityPartitionProjections() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(LOG_SCHEMA, IDENTITY_PARTITION_SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + + List inputRecords = RandomGenericData.generate(LOG_SCHEMA, 10, 0); + Integer idx = 0; + AppendFiles append = table.newAppend(); + for (Record record : inputRecords) { + record.set(1, "2020-03-2" + idx); + record.set(2, idx.toString()); + append.appendFile(writeFile(temp.newFile(), table, Row.of("2020-03-2" + idx, idx.toString()), format, ImmutableList.of(record))); + idx += 1; + } + append.commit(); + + // individual fields + validateIdentityPartitionProjections(location.toString(), withColumns("date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("id"), inputRecords); + // field pairs + validateIdentityPartitionProjections(location.toString(), withColumns("date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("date", "level"), inputRecords); + // out-of-order pairs + validateIdentityPartitionProjections(location.toString(), withColumns("message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("level", "date"), inputRecords); + // full projection + validateIdentityPartitionProjections(location.toString(), LOG_SCHEMA, inputRecords); + // out-of-order triplets + validateIdentityPartitionProjections(location.toString(), withColumns("date", "level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("level", "date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("date", "message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("level", "message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("message", "date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), withColumns("message", "level", "date"), inputRecords); + } + + private static Schema withColumns(String... names) { + Map indexByName = TypeUtil.indexByName(LOG_SCHEMA.asStruct()); + Set projectedIds = Sets.newHashSet(); + for (String name : names) { + projectedIds.add(indexByName.get(name)); + } + return TypeUtil.select(LOG_SCHEMA, projectedIds); + } + + private void validateIdentityPartitionProjections( + String tablePath, Schema projectedSchema, List inputRecords) throws Exception { + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder + .readFrom(tablePath) + .project(projectedSchema); + List actualRecords = readRecords(job.getConfiguration()); + + Set fieldNames = TypeUtil.indexByName(projectedSchema.asStruct()).keySet(); + for (int pos = 0; pos < inputRecords.size(); pos++) { + Record inputRecord = inputRecords.get(pos); + Record actualRecord = actualRecords.get(pos); + Assert.assertEquals("Projected schema should match", projectedSchema.asStruct(), actualRecord.struct()); + for (String name : fieldNames) { + Assert.assertEquals( + "Projected field " + name + " should match", inputRecord.getField(name), actualRecord.getField(name)); + } + } + } + + @Test + public void testSnapshotReads() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + table.newAppend() + .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) + .commit(); + long snapshotId = table.currentSnapshot().snapshotId(); + table.newAppend() + .appendFile(writeFile(table, null, format, RandomGenericData.generate(table.schema(), 1, 0L))) + .commit(); + + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder + .readFrom(location.toString()) + .snapshotId(snapshotId); + + validate(job, expectedRecords); + } + + @Test + public void testLocality() throws Exception { + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + table.newAppend() + .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) + .commit(); + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()); + + for (InputSplit split : splits(job.getConfiguration())) { + Assert.assertArrayEquals(IcebergInputFormat.IcebergSplit.ANYWHERE, split.getLocations()); + } + + configBuilder.preferLocality(); + for (InputSplit split : splits(job.getConfiguration())) { + Assert.assertArrayEquals(new String[]{"localhost"}, split.getLocations()); + } + } + + public static class HadoopCatalogFunc implements Function { + @Override + public Catalog apply(Configuration conf) { + return new HadoopCatalog(conf, conf.get("warehouse.location")); + } + } + + @Test + public void testCustomCatalog() throws Exception { + conf = new Configuration(); + conf.set("warehouse.location", temp.newFolder("hadoop_catalog").getAbsolutePath()); + + Catalog catalog = new HadoopCatalogFunc().apply(conf); + TableIdentifier tableIdentifier = TableIdentifier.of("db", "t"); + Table table = catalog.createTable(tableIdentifier, SCHEMA, SPEC, + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name())); + List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); + expectedRecords.get(0).set(2, "2020-03-20"); + DataFile dataFile = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder + .catalogFunc(HadoopCatalogFunc.class) + .readFrom(tableIdentifier.toString()); + validate(job, expectedRecords); + } +*/ + private static void validate(Job job, List expectedRecords) { + List actualRecords = readRecords(job.getConfiguration()); + Assert.assertEquals(expectedRecords, actualRecords); + } + + private static List splits(Configuration conf) { + TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); + IcebergInputFormat icebergInputFormat = new IcebergInputFormat<>(); + return icebergInputFormat.getSplits(context); + } + + private static List readRecords(Configuration conf) { + TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); + IcebergInputFormat icebergInputFormat = new IcebergInputFormat<>(); + List splits = icebergInputFormat.getSplits(context); + return + FluentIterable + .from(splits) + .transformAndConcat(split -> readRecords(icebergInputFormat, split, context)) + .toList(); + } + + private static Iterable readRecords( + IcebergInputFormat inputFormat, InputSplit split, TaskAttemptContext context) { + RecordReader recordReader = inputFormat.createRecordReader(split, context); + List records = new ArrayList<>(); + try { + recordReader.initialize(split, context); + while (recordReader.nextKeyValue()) { + records.add(recordReader.getCurrentValue()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return records; + } + +} From 98436910a3e75229a5450e3a97a07ab98138df89 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 8 Jun 2020 20:52:22 +0100 Subject: [PATCH 43/51] fix tests --- .../apache/iceberg/mr/InputFormatConfig.java | 10 + .../iceberg/mr/mapreduce/TableResolver.java | 1 - .../mr/mapreduce/TestIcebergInputFormat.java | 297 ++++++----- .../mr/mapreduce/TestIcebergInputFormat2.java | 484 ------------------ 4 files changed, 167 insertions(+), 625 deletions(-) delete mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java index e462704efd95..386898f8ee30 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java +++ b/mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java @@ -77,6 +77,16 @@ public ConfigBuilder project(Schema schema) { return this; } + public ConfigBuilder schema(Schema schema) { + conf.set(TABLE_SCHEMA, SchemaParser.toJson(schema)); + return this; + } + + public ConfigBuilder readFrom(String path) { + conf.set(TABLE_PATH, path); + return this; + } + public ConfigBuilder reuseContainers(boolean reuse) { conf.setBoolean(InputFormatConfig.REUSE_CONTAINERS, reuse); return this; diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java index 88f2b2e8bc29..7775e7ea9409 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapreduce/TableResolver.java @@ -36,7 +36,6 @@ private TableResolver() { public static Table findTable(Configuration conf) { String path = conf.get(InputFormatConfig.TABLE_PATH); - System.out.println("YYY PATH : " + path); Preconditions.checkArgument(path != null, "Table path should not be null"); if (path.contains("/")) { HadoopTables tables = new HadoopTables(conf); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java index 4999df86538e..8c0c9e81aa04 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat.java @@ -20,7 +20,6 @@ package org.apache.iceberg.mr.mapreduce; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -39,9 +38,9 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; -import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestHelpers.Row; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.RandomGenericData; @@ -50,7 +49,6 @@ import org.apache.iceberg.hadoop.HadoopCatalog; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.mr.TestHelpers.Row; import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -58,7 +56,9 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -import org.junit.*; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -66,25 +66,23 @@ import static org.apache.iceberg.mr.TestHelpers.writeFile; import static org.apache.iceberg.types.Types.NestedField.required; -@Ignore("") @RunWith(Parameterized.class) public class TestIcebergInputFormat { - - static final Schema SCHEMA = new Schema( + private static final Schema SCHEMA = new Schema( required(1, "data", Types.StringType.get()), required(2, "id", Types.LongType.get()), required(3, "date", Types.StringType.get())); - static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) + private static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) .identity("date") .bucket("id", 1) .build(); @Rule public TemporaryFolder temp = new TemporaryFolder(); - private HadoopTables tables; - private Configuration conf; - private final FileFormat fileFormat; + + private Configuration conf = new Configuration(); + private HadoopTables tables = new HadoopTables(conf); @Parameterized.Parameters public static Object[][] parameters() { @@ -95,90 +93,82 @@ public static Object[][] parameters() { }; } - public TestIcebergInputFormat(String format) { - this.fileFormat = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); - } - - @Before - public void before() { - conf = new Configuration(); - tables = new HadoopTables(conf); - } - - private void readFrom(String path) { - conf.set(InputFormatConfig.TABLE_PATH, path); - System.out.println("XXX PATH " + InputFormatConfig.TABLE_PATH + " : " + path); - Table table = TableResolver.findTable(conf); - conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - } + private final FileFormat format; - protected void runAndValidate(File tableLocation, List expectedRecords) throws IOException { - Job job = Job.getInstance(conf); - readFrom(tableLocation.toString()); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - validate(job, expectedRecords); + public TestIcebergInputFormat(String format) { + this.format = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); } @Test public void testUnpartitionedTable() throws Exception { - File tableLocation = temp.newFolder(fileFormat.name()); - Table table = tables - .create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), tableLocation.toString()); + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); + Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, null, fileFormat, expectedRecords); - table.newAppend().appendFile(dataFile).commit(); - runAndValidate(tableLocation, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, null, format, expectedRecords); + table.newAppend() + .appendFile(dataFile) + .commit(); + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); + validate(job, expectedRecords); } @Test public void testPartitionedTable() throws Exception { - File tableLocation = temp.newFolder(fileFormat.name()); - Assert.assertTrue(tableLocation.delete()); + File location = temp.newFolder(format.name()); + Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - tableLocation.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); table.newAppend() .appendFile(dataFile) .commit(); - runAndValidate(tableLocation, expectedRecords); + Job job = Job.getInstance(conf); + InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); + configBuilder.readFrom(location.toString()).schema(table.schema()); + validate(job, expectedRecords); } @Test public void testFilterExp() throws Exception { - File location = temp.newFolder(fileFormat.name()); + File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); expectedRecords.get(0).set(2, "2020-03-20"); expectedRecords.get(1).set(2, "2020-03-20"); - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); - DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), fileFormat, - RandomGenericData.generate(table.schema(), 2, 0L)); + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, + RandomGenericData.generate(table.schema(), 2, 0L)); table.newAppend() - .appendFile(dataFile1) - .appendFile(dataFile2) - .commit(); + .appendFile(dataFile1) + .appendFile(dataFile2) + .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); - configBuilder.filter(Expressions.equal("date", "2020-03-20")); + configBuilder.readFrom(location.toString()) + .schema(table.schema()) + .filter(Expressions.equal("date", "2020-03-20")); validate(job, expectedRecords); } @Test public void testResiduals() throws Exception { - File location = temp.newFolder(fileFormat.name()); + File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List writeRecords = RandomGenericData.generate(table.schema(), 2, 0L); writeRecords.get(0).set(1, 123L); writeRecords.get(0).set(2, "2020-03-20"); @@ -188,72 +178,75 @@ public void testResiduals() throws Exception { List expectedRecords = new ArrayList<>(); expectedRecords.add(writeRecords.get(0)); - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, writeRecords); - DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), fileFormat, - RandomGenericData.generate(table.schema(), 2, 0L)); + DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, writeRecords); + DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, + RandomGenericData.generate(table.schema(), 2, 0L)); table.newAppend() - .appendFile(dataFile1) - .appendFile(dataFile2) - .commit(); + .appendFile(dataFile1) + .appendFile(dataFile2) + .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); - configBuilder.filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); + configBuilder.readFrom(location.toString()) + .schema(table.schema()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); validate(job, expectedRecords); // skip residual filtering job = Job.getInstance(conf); configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); - configBuilder.skipResidualFiltering() - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); + configBuilder.skipResidualFiltering().readFrom(location.toString()) + .schema(table.schema()) + .filter(Expressions.and( + Expressions.equal("date", "2020-03-20"), + Expressions.equal("id", 123))); validate(job, writeRecords); } @Test public void testProjection() throws Exception { - File location = temp.newFolder(fileFormat.name()); + File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Schema projectedSchema = TypeUtil.select(SCHEMA, ImmutableSet.of(1)); Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List inputRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, inputRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, inputRecords); table.newAppend() - .appendFile(dataFile) - .commit(); + .appendFile(dataFile) + .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); - configBuilder.project(projectedSchema); + configBuilder + .readFrom(location.toString()) + .project(projectedSchema) + .schema(table.schema()); List outputRecords = readRecords(job.getConfiguration()); Assert.assertEquals(inputRecords.size(), outputRecords.size()); Assert.assertEquals(projectedSchema.asStruct(), outputRecords.get(0).struct()); } private static final Schema LOG_SCHEMA = new Schema( - Types.NestedField.optional(1, "id", Types.IntegerType.get()), - Types.NestedField.optional(2, "date", Types.StringType.get()), - Types.NestedField.optional(3, "level", Types.StringType.get()), - Types.NestedField.optional(4, "message", Types.StringType.get()) + Types.NestedField.optional(1, "id", Types.IntegerType.get()), + Types.NestedField.optional(2, "date", Types.StringType.get()), + Types.NestedField.optional(3, "level", Types.StringType.get()), + Types.NestedField.optional(4, "message", Types.StringType.get()) ); private static final PartitionSpec IDENTITY_PARTITION_SPEC = - PartitionSpec.builderFor(LOG_SCHEMA).identity("date").identity("level").build(); + PartitionSpec.builderFor(LOG_SCHEMA).identity("date").identity("level").build(); @Test public void testIdentityPartitionProjections() throws Exception { - File location = temp.newFolder(fileFormat.name()); + File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = tables.create(LOG_SCHEMA, IDENTITY_PARTITION_SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List inputRecords = RandomGenericData.generate(LOG_SCHEMA, 10, 0); Integer idx = 0; @@ -261,34 +254,50 @@ public void testIdentityPartitionProjections() throws Exception { for (Record record : inputRecords) { record.set(1, "2020-03-2" + idx); record.set(2, idx.toString()); - append.appendFile(writeFile(temp.newFile(), table, Row.of("2020-03-2" + idx, idx.toString()), - fileFormat, ImmutableList.of(record))); + append.appendFile(writeFile(temp.newFile(), table, Row.of("2020-03-2" + idx, idx.toString()), format, + ImmutableList.of(record))); idx += 1; } append.commit(); // individual fields - validateIdentityPartitionProjections(location.toString(), withColumns("date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("id"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("id"), inputRecords); // field pairs - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "level"), inputRecords); // out-of-order pairs - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "date"), inputRecords); // full projection - validateIdentityPartitionProjections(location.toString(), LOG_SCHEMA, inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), LOG_SCHEMA, inputRecords); // out-of-order triplets - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "level", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "date", "message"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("date", "message", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("level", "message", "date"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "date", "level"), inputRecords); + validateIdentityPartitionProjections(location.toString(), table.schema(), + withColumns("message", "level", "date"), inputRecords); } private static Schema withColumns(String... names) { @@ -301,11 +310,13 @@ private static Schema withColumns(String... names) { } private void validateIdentityPartitionProjections( - String tablePath, Schema projectedSchema, List inputRecords) throws Exception { + String tablePath, Schema tableSchema, Schema projectedSchema, List inputRecords) throws Exception { Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(tablePath); - configBuilder.project(projectedSchema); + configBuilder + .readFrom(tablePath) + .schema(tableSchema) + .project(projectedSchema); List actualRecords = readRecords(job.getConfiguration()); Set fieldNames = TypeUtil.indexByName(projectedSchema.asStruct()).keySet(); @@ -315,50 +326,54 @@ private void validateIdentityPartitionProjections( Assert.assertEquals("Projected schema should match", projectedSchema.asStruct(), actualRecord.struct()); for (String name : fieldNames) { Assert.assertEquals( - "Projected field " + name + " should match", inputRecord.getField(name), actualRecord.getField(name)); + "Projected field " + name + " should match", inputRecord.getField(name), + actualRecord.getField(name)); } } } @Test public void testSnapshotReads() throws Exception { - File location = temp.newFolder(fileFormat.name()); + File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, fileFormat, expectedRecords)) - .commit(); + .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) + .commit(); long snapshotId = table.currentSnapshot().snapshotId(); table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, fileFormat, - RandomGenericData.generate(table.schema(), 1, 0L))) - .commit(); + .appendFile(writeFile(temp.newFile(), table, null, format, + RandomGenericData.generate(table.schema(), 1, 0L))) + .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); - configBuilder.snapshotId(snapshotId); + configBuilder + .schema(table.schema()) + .readFrom(location.toString()) + .snapshotId(snapshotId); validate(job, expectedRecords); } @Test public void testLocality() throws Exception { - File location = temp.newFolder(fileFormat.name()); + File location = temp.newFolder(format.name()); Assert.assertTrue(location.delete()); Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name()), - location.toString()); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), + location.toString()); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, fileFormat, expectedRecords)) - .commit(); + .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) + .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); + configBuilder.readFrom(location.toString()).schema(table.schema()); + for (InputSplit split : splits(job.getConfiguration())) { Assert.assertArrayEquals(IcebergInputFormat.IcebergSplit.ANYWHERE, split.getLocations()); } @@ -384,18 +399,20 @@ public void testCustomCatalog() throws Exception { Catalog catalog = new HadoopCatalogFunc().apply(conf); TableIdentifier tableIdentifier = TableIdentifier.of("db", "t"); Table table = catalog.createTable(tableIdentifier, SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.name())); + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name())); List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), fileFormat, expectedRecords); + DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); table.newAppend() - .appendFile(dataFile) - .commit(); + .appendFile(dataFile) + .commit(); Job job = Job.getInstance(conf); InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(tableIdentifier.toString()); - configBuilder.catalogFunc(HadoopCatalogFunc.class); + configBuilder + .catalogFunc(HadoopCatalogFunc.class) + .schema(table.schema()) + .readFrom(tableIdentifier.toString()); validate(job, expectedRecords); } @@ -415,14 +432,14 @@ private static List readRecords(Configuration conf) { IcebergInputFormat icebergInputFormat = new IcebergInputFormat<>(); List splits = icebergInputFormat.getSplits(context); return - FluentIterable - .from(splits) - .transformAndConcat(split -> readRecords(icebergInputFormat, split, context)) - .toList(); + FluentIterable + .from(splits) + .transformAndConcat(split -> readRecords(icebergInputFormat, split, context)) + .toList(); } private static Iterable readRecords( - IcebergInputFormat inputFormat, InputSplit split, TaskAttemptContext context) { + IcebergInputFormat inputFormat, InputSplit split, TaskAttemptContext context) { RecordReader recordReader = inputFormat.createRecordReader(split, context); List records = new ArrayList<>(); try { diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java b/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java deleted file mode 100644 index 10a443bc91d7..000000000000 --- a/mr/src/test/java/org/apache/iceberg/mr/mapreduce/TestIcebergInputFormat2.java +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapreduce; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; -import org.apache.iceberg.*; -import org.apache.iceberg.TestHelpers.Row; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.RandomGenericData; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.avro.DataWriter; -import org.apache.iceberg.data.orc.GenericOrcWriter; -import org.apache.iceberg.data.parquet.GenericParquetWriter; -import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.hadoop.HadoopTables; -import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.mr.InputFormatConfig; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.relocated.com.google.common.collect.FluentIterable; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; -import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.types.TypeUtil; -import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.apache.iceberg.mr.TestHelpers.writeFile; - -@RunWith(Parameterized.class) -public class TestIcebergInputFormat2 { - static final Schema SCHEMA = new Schema( - required(1, "data", Types.StringType.get()), - required(2, "id", Types.LongType.get()), - required(3, "date", Types.StringType.get())); - - static final PartitionSpec SPEC = PartitionSpec.builderFor(SCHEMA) - .identity("date") - .bucket("id", 1) - .build(); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - private HadoopTables tables; - private Configuration conf; - - @Parameterized.Parameters - public static Object[][] parameters() { - return new Object[][]{ - new Object[]{"parquet"}, - new Object[]{"avro"}, - new Object[]{"orc"} - }; - } - - private final FileFormat format; - - public TestIcebergInputFormat2(String format) { - this.format = FileFormat.valueOf(format.toUpperCase(Locale.ENGLISH)); - } - - @Before - public void before() { - conf = new Configuration(); - tables = new HadoopTables(conf); - } - - private void readFrom(String path) { - conf.set(InputFormatConfig.TABLE_PATH, path); - System.out.println("XXX PATH " + InputFormatConfig.TABLE_PATH + " : " + path); - Table table = TableResolver.findTable(conf); - conf.set(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(table.schema())); - } - - @Test - public void testUnpartitionedTable() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, null, format, expectedRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - readFrom(location.toString()); - validate(job, expectedRecords); - } -/* - @Test - public void testPartitionedTable() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); - validate(job, expectedRecords); - } - - @Test - public void testFilterExp() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - expectedRecords.get(1).set(2, "2020-03-20"); - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); - DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, - RandomGenericData.generate(table.schema(), 2, 0L)); - table.newAppend() - .appendFile(dataFile1) - .appendFile(dataFile2) - .commit(); - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()) - .filter(Expressions.equal("date", "2020-03-20")); - validate(job, expectedRecords); - } - - @Test - public void testResiduals() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List writeRecords = RandomGenericData.generate(table.schema(), 2, 0L); - writeRecords.get(0).set(1, 123L); - writeRecords.get(0).set(2, "2020-03-20"); - writeRecords.get(1).set(1, 456L); - writeRecords.get(1).set(2, "2020-03-20"); - - List expectedRecords = new ArrayList<>(); - expectedRecords.add(writeRecords.get(0)); - - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, writeRecords); - DataFile dataFile2 = writeFile(temp.newFile(), table, Row.of("2020-03-21", 0), format, - RandomGenericData.generate(table.schema(), 2, 0L)); - table.newAppend() - .appendFile(dataFile1) - .appendFile(dataFile2) - .commit(); - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); - validate(job, expectedRecords); - - // skip residual filtering - job = Job.getInstance(conf); - configBuilder = IcebergInputFormat.configure(job); - configBuilder.skipResidualFiltering().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 123))); - validate(job, writeRecords); - } - - @Test - public void testFailedResidualFiltering() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - expectedRecords.get(1).set(2, "2020-03-20"); - - DataFile dataFile1 = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, expectedRecords); - table.newAppend() - .appendFile(dataFile1) - .commit(); - - Job jobShouldFail1 = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(jobShouldFail1); - configBuilder.useHiveRows().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 0))); - AssertHelpers.assertThrows( - "Residuals are not evaluated today for Iceberg Generics In memory model of HIVE", - UnsupportedOperationException.class, "Filter expression ref(name=\"id\") == 0 is not completely satisfied.", - () -> validate(jobShouldFail1, expectedRecords)); - - Job jobShouldFail2 = Job.getInstance(conf); - configBuilder = IcebergInputFormat.configure(jobShouldFail2); - configBuilder.usePigTuples().readFrom(location.toString()) - .filter(Expressions.and( - Expressions.equal("date", "2020-03-20"), - Expressions.equal("id", 0))); - AssertHelpers.assertThrows( - "Residuals are not evaluated today for Iceberg Generics In memory model of PIG", - UnsupportedOperationException.class, "Filter expression ref(name=\"id\") == 0 is not completely satisfied.", - () -> validate(jobShouldFail2, expectedRecords)); - } - - @Test - public void testProjection() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Schema projectedSchema = TypeUtil.select(SCHEMA, ImmutableSet.of(1)); - Table table = tables.create(SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List inputRecords = RandomGenericData.generate(table.schema(), 1, 0L); - DataFile dataFile = writeFile(temp.newFile(), table, Row.of("2020-03-20", 0), format, inputRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .readFrom(location.toString()) - .project(projectedSchema); - List outputRecords = readRecords(job.getConfiguration()); - Assert.assertEquals(inputRecords.size(), outputRecords.size()); - Assert.assertEquals(projectedSchema.asStruct(), outputRecords.get(0).struct()); - } - - private static final Schema LOG_SCHEMA = new Schema( - Types.NestedField.optional(1, "id", Types.IntegerType.get()), - Types.NestedField.optional(2, "date", Types.StringType.get()), - Types.NestedField.optional(3, "level", Types.StringType.get()), - Types.NestedField.optional(4, "message", Types.StringType.get()) - ); - - private static final PartitionSpec IDENTITY_PARTITION_SPEC = - PartitionSpec.builderFor(LOG_SCHEMA).identity("date").identity("level").build(); - - @Test - public void testIdentityPartitionProjections() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(LOG_SCHEMA, IDENTITY_PARTITION_SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - - List inputRecords = RandomGenericData.generate(LOG_SCHEMA, 10, 0); - Integer idx = 0; - AppendFiles append = table.newAppend(); - for (Record record : inputRecords) { - record.set(1, "2020-03-2" + idx); - record.set(2, idx.toString()); - append.appendFile(writeFile(temp.newFile(), table, Row.of("2020-03-2" + idx, idx.toString()), format, ImmutableList.of(record))); - idx += 1; - } - append.commit(); - - // individual fields - validateIdentityPartitionProjections(location.toString(), withColumns("date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("id"), inputRecords); - // field pairs - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level"), inputRecords); - // out-of-order pairs - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date"), inputRecords); - // full projection - validateIdentityPartitionProjections(location.toString(), LOG_SCHEMA, inputRecords); - // out-of-order triplets - validateIdentityPartitionProjections(location.toString(), withColumns("date", "level", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "date", "message"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("date", "message", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("level", "message", "date"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "date", "level"), inputRecords); - validateIdentityPartitionProjections(location.toString(), withColumns("message", "level", "date"), inputRecords); - } - - private static Schema withColumns(String... names) { - Map indexByName = TypeUtil.indexByName(LOG_SCHEMA.asStruct()); - Set projectedIds = Sets.newHashSet(); - for (String name : names) { - projectedIds.add(indexByName.get(name)); - } - return TypeUtil.select(LOG_SCHEMA, projectedIds); - } - - private void validateIdentityPartitionProjections( - String tablePath, Schema projectedSchema, List inputRecords) throws Exception { - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .readFrom(tablePath) - .project(projectedSchema); - List actualRecords = readRecords(job.getConfiguration()); - - Set fieldNames = TypeUtil.indexByName(projectedSchema.asStruct()).keySet(); - for (int pos = 0; pos < inputRecords.size(); pos++) { - Record inputRecord = inputRecords.get(pos); - Record actualRecord = actualRecords.get(pos); - Assert.assertEquals("Projected schema should match", projectedSchema.asStruct(), actualRecord.struct()); - for (String name : fieldNames) { - Assert.assertEquals( - "Projected field " + name + " should match", inputRecord.getField(name), actualRecord.getField(name)); - } - } - } - - @Test - public void testSnapshotReads() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) - .commit(); - long snapshotId = table.currentSnapshot().snapshotId(); - table.newAppend() - .appendFile(writeFile(table, null, format, RandomGenericData.generate(table.schema(), 1, 0L))) - .commit(); - - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .readFrom(location.toString()) - .snapshotId(snapshotId); - - validate(job, expectedRecords); - } - - @Test - public void testLocality() throws Exception { - File location = temp.newFolder(format.name()); - Assert.assertTrue(location.delete()); - Table table = tables.create(SCHEMA, PartitionSpec.unpartitioned(), - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()), - location.toString()); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - table.newAppend() - .appendFile(writeFile(temp.newFile(), table, null, format, expectedRecords)) - .commit(); - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder.readFrom(location.toString()); - - for (InputSplit split : splits(job.getConfiguration())) { - Assert.assertArrayEquals(IcebergInputFormat.IcebergSplit.ANYWHERE, split.getLocations()); - } - - configBuilder.preferLocality(); - for (InputSplit split : splits(job.getConfiguration())) { - Assert.assertArrayEquals(new String[]{"localhost"}, split.getLocations()); - } - } - - public static class HadoopCatalogFunc implements Function { - @Override - public Catalog apply(Configuration conf) { - return new HadoopCatalog(conf, conf.get("warehouse.location")); - } - } - - @Test - public void testCustomCatalog() throws Exception { - conf = new Configuration(); - conf.set("warehouse.location", temp.newFolder("hadoop_catalog").getAbsolutePath()); - - Catalog catalog = new HadoopCatalogFunc().apply(conf); - TableIdentifier tableIdentifier = TableIdentifier.of("db", "t"); - Table table = catalog.createTable(tableIdentifier, SCHEMA, SPEC, - ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name())); - List expectedRecords = RandomGenericData.generate(table.schema(), 1, 0L); - expectedRecords.get(0).set(2, "2020-03-20"); - DataFile dataFile = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords); - table.newAppend() - .appendFile(dataFile) - .commit(); - - Job job = Job.getInstance(conf); - InputFormatConfig.ConfigBuilder configBuilder = IcebergInputFormat.configure(job); - configBuilder - .catalogFunc(HadoopCatalogFunc.class) - .readFrom(tableIdentifier.toString()); - validate(job, expectedRecords); - } -*/ - private static void validate(Job job, List expectedRecords) { - List actualRecords = readRecords(job.getConfiguration()); - Assert.assertEquals(expectedRecords, actualRecords); - } - - private static List splits(Configuration conf) { - TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); - IcebergInputFormat icebergInputFormat = new IcebergInputFormat<>(); - return icebergInputFormat.getSplits(context); - } - - private static List readRecords(Configuration conf) { - TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID()); - IcebergInputFormat icebergInputFormat = new IcebergInputFormat<>(); - List splits = icebergInputFormat.getSplits(context); - return - FluentIterable - .from(splits) - .transformAndConcat(split -> readRecords(icebergInputFormat, split, context)) - .toList(); - } - - private static Iterable readRecords( - IcebergInputFormat inputFormat, InputSplit split, TaskAttemptContext context) { - RecordReader recordReader = inputFormat.createRecordReader(split, context); - List records = new ArrayList<>(); - try { - recordReader.initialize(split, context); - while (recordReader.nextKeyValue()) { - records.add(recordReader.getCurrentValue()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - return records; - } - -} From 7570d0625a11d6236d9c2e6ccff9b0699c8fd213 Mon Sep 17 00:00:00 2001 From: awoodhead Date: Mon, 8 Jun 2020 21:31:43 +0100 Subject: [PATCH 44/51] tidy up --- build.gradle | 9 ++------- .../org/apache/iceberg/mr/mapred/IcebergInputFormat.java | 5 ----- .../iceberg/mr/mapred/TestHiveIcebergInputFormat.java | 1 + 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/build.gradle b/build.gradle index 9ffce387c305..116e4df54d66 100644 --- a/build.gradle +++ b/build.gradle @@ -324,13 +324,8 @@ project(':iceberg-mr') { exclude group: 'com.google.guava' } - compileOnly("org.apache.hive:hive-metastore") { - //exclude group: 'org.apache.avro', module: 'avro' - } - - compileOnly("org.apache.hive:hive-serde") { - //exclude group: 'org.apache.avro', module: 'avro' - } + compileOnly "org.apache.hive:hive-metastore" + compileOnly "org.apache.hive:hive-serde" testCompile("com.klarna:hiverunner:5.2.1") { exclude group: 'javax.jms', module: 'jms' diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 8b987587ff50..9c3d4935cae8 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -191,11 +191,6 @@ public boolean next(Void key, IcebergWritable value) { } if (tasks.hasNext()) { - /*try { - reader.close(); - } catch (IOException e) { - LOG.error("Error closing reader", e); - }*/ nextTask(); currentRecord = recordIterator.next(); value.setRecord(resolveAppropriateRecordForTableType()); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java index 872094eeb91a..80ec34ec0487 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestHiveIcebergInputFormat.java @@ -82,4 +82,5 @@ public void emptyTable() { assertEquals(0, result.size()); } + //TODO: when HiveSerde and StorageHandlers merged in, move over additional tests from Hiveberg } From 2fce73547b15e556884065f57b8cfe1772ec2afd Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Wed, 10 Jun 2020 14:51:39 +0100 Subject: [PATCH 45/51] Add SnapshotIterable --- .../iceberg/mr/mapred/IcebergInputFormat.java | 9 ++- .../mr/mapred/iterables/SnapshotIterable.java | 69 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index 9c3d4935cae8..acf1af6a21a3 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -52,6 +52,7 @@ import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.SerializationUtil; +import org.apache.iceberg.mr.mapred.iterables.SnapshotIterable; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -161,7 +162,12 @@ public IcebergRecordReader(InputSplit split, JobConf conf) throws IOException { private void initialise() { tasks = split.getTask().files().iterator(); - nextTask(); + if (table instanceof SnapshotsTable) { + reader = new SnapshotIterable(table); + recordIterator = reader.iterator(); + } else { + nextTask(); + } } private void nextTask() { @@ -182,6 +188,7 @@ private Record resolveAppropriateRecordForTableType() { } } + @Override public boolean next(Void key, IcebergWritable value) { if (recordIterator.hasNext()) { diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java new file mode 100644 index 000000000000..4a4a7d66fd4a --- /dev/null +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.mapred.iterables; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.CloseableIterator; + +/** + * Creates an Iterable of Records with all snapshot metadata that can be used with the RecordReader. + */ +public class SnapshotIterable implements CloseableIterable { + + private Table table; + + public SnapshotIterable(Table table) { + this.table = table; + } + + public CloseableIterator iterator() { + Iterable snapshots = table.snapshots(); + List snapRecords = new ArrayList<>(); + snapshots.forEach(snapshot -> snapRecords.add(createSnapshotRecord(snapshot))); + + return (CloseableIterator) snapRecords.iterator(); + } + + /** + * Populates a Record with snapshot metadata. + */ + private Record createSnapshotRecord(Snapshot snapshot) { + Record snapRecord = GenericRecord.create(table.schema()); + snapRecord.setField("committed_at", snapshot.timestampMillis()); + snapRecord.setField("snapshot_id", snapshot.snapshotId()); + snapRecord.setField("parent_id", snapshot.parentId()); + snapRecord.setField("operation", snapshot.operation()); + snapRecord.setField("manifest_list", snapshot.manifestListLocation()); + snapRecord.setField("summary", snapshot.summary()); + return snapRecord; + } + + @Override + public void close() throws IOException { + iterator().close(); + } +} From 52a6a174bb2b1aaada7bdb373d22fe538de6896f Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Fri, 12 Jun 2020 18:23:18 +0100 Subject: [PATCH 46/51] Clean up FilterFactory --- .../mr/mapred/IcebergFilterFactory.java | 77 +++++-------------- .../iceberg/mr/mapred/SystemTableUtil.java | 2 +- .../mr/mapred/TestIcebergFilterFactory.java | 21 +++++ 3 files changed, 40 insertions(+), 60 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java index 46a276d3298f..a4e3cf754ac4 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -39,55 +39,13 @@ public class IcebergFilterFactory { - private IcebergFilterFactory() { - } + private IcebergFilterFactory() {} public static Expression generateFilterExpression(SearchArgument sarg) { List leaves = sarg.getLeaves(); List childNodes = sarg.getExpression().getChildren(); - switch (sarg.getExpression().getOperator()) { - case OR: - ExpressionTree orLeft = childNodes.get(0); - ExpressionTree orRight = childNodes.get(1); - return or(translate(orLeft, leaves), translate(orRight, leaves)); - case AND: - ExpressionTree andLeft = childNodes.get(0); - ExpressionTree andRight = childNodes.get(1); - if (childNodes.size() > 2) { - Expression[] evaluatedChildren = getLeftoverLeaves(childNodes, leaves); - return and( - translate(andLeft, leaves), translate(andRight, leaves), evaluatedChildren); - } else { - return and(translate(andLeft, leaves), translate(andRight, leaves)); - } - case NOT: - return not(translateLeaf(sarg.getLeaves().get(0))); - case LEAF: - return translateLeaf(sarg.getLeaves().get(0)); - case CONSTANT: - return null; - default: - throw new IllegalStateException("Unknown operator: " + sarg.getExpression().getOperator()); - } - } - - /** - * Remove first 2 nodes already evaluated and return an array of the evaluated leftover nodes. - * @param allChildNodes All child nodes to be evaluated for the AND expression. - * @param leaves All instances of the leaf nodes. - * @return Array of leftover evaluated nodes. - */ - private static Expression[] getLeftoverLeaves(List allChildNodes, List leaves) { - allChildNodes.remove(0); - allChildNodes.remove(0); - - Expression[] evaluatedLeaves = new Expression[allChildNodes.size()]; - for (int i = 0; i < allChildNodes.size(); i++) { - Expression filter = translate(allChildNodes.get(i), leaves); - evaluatedLeaves[i] = filter; - } - return evaluatedLeaves; + return translate(sarg.getExpression(), leaves, childNodes); } /** @@ -96,29 +54,30 @@ private static Expression[] getLeftoverLeaves(List allChildNodes * @param leaves List of all leaf nodes within the tree. * @return Expression that is translated from the Hive SearchArgument. */ - private static Expression translate(ExpressionTree tree, List leaves) { + private static Expression translate(ExpressionTree tree, List leaves, + List childNodes) { switch (tree.getOperator()) { case OR: - return or(translate(tree.getChildren().get(0), leaves), - translate(tree.getChildren().get(1), leaves)); + Expression orResult = Expressions.alwaysFalse(); + for (ExpressionTree child : childNodes) { + orResult = or(orResult, translate(child, leaves, childNodes)); + } + return orResult; case AND: - if (tree.getChildren().size() > 2) { - Expression[] evaluatedChildren = getLeftoverLeaves(tree.getChildren(), leaves); - return and(translate(tree.getChildren().get(0), leaves), - translate(tree.getChildren().get(1), leaves), evaluatedChildren); - } else { - return and(translate(tree.getChildren().get(0), leaves), - translate(tree.getChildren().get(1), leaves)); + Expression result = Expressions.alwaysTrue(); + for (ExpressionTree child : childNodes) { + result = and(result, translate(child, leaves, childNodes)); } + return result; case NOT: - return not(translate(tree.getChildren().get(0), leaves)); + return not(translate(tree.getChildren().get(0), leaves, childNodes)); case LEAF: return translateLeaf(leaves.get(tree.getLeaf())); case CONSTANT: //We are unsure of how the CONSTANT case works, so using the approach of: //https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ // ParquetFilterPredicateConverter.java#L116 - return null; + throw new UnsupportedOperationException("CONSTANT operator is not supported"); default: throw new IllegalStateException("Unknown operator: " + tree.getOperator()); } @@ -131,7 +90,7 @@ private static Expression translate(ExpressionTree tree, List lea */ private static Expression translateLeaf(PredicateLeaf leaf) { String column = leaf.getColumnName(); - if (column.equals("snapshot__id")) { + if (column.equals(SystemTableUtil.DEFAULT_SNAPSHOT_ID_COLUMN_NAME)) { return Expressions.alwaysTrue(); } switch (leaf.getOperator()) { @@ -147,12 +106,12 @@ private static Expression translateLeaf(PredicateLeaf leaf) { return in(column, leaf.getLiteralList()); case BETWEEN: return and(greaterThanOrEqual(column, leaf.getLiteralList().get(0)), - lessThanOrEqual(column, leaf.getLiteralList().get(1))); + lessThanOrEqual(column, leaf.getLiteralList().get(1))); case IS_NULL: return isNull(column); default: throw new IllegalStateException("Unknown operator: " + leaf.getOperator()); } } - } + diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java index 17f5b9fb07b2..b68e2f812ae1 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java @@ -32,7 +32,7 @@ public class SystemTableUtil { static final String VIRTUAL_COLUMN_NAME = "iceberg.hive.snapshot.virtual.column.name"; - private static final String DEFAULT_SNAPSHOT_ID_COLUMN_NAME = "snapshot__id"; + protected static final String DEFAULT_SNAPSHOT_ID_COLUMN_NAME = "snapshot__id"; private SystemTableUtil() {} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java index f6d53379fd7b..1ccdc6d424ea 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java @@ -188,4 +188,25 @@ public void testManyAndOperand() { assertEquals(actual.right().op(), expected.right().op()); assertEquals(actual.left().op(), expected.left().op()); } + + @Test + public void testManyOrOperand() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startOr() + .equals("salary", PredicateLeaf.Type.LONG, 3000L) + .equals("job", PredicateLeaf.Type.LONG, 4000L) + .equals("name", PredicateLeaf.Type.LONG, 9000L) + .end() + .build(); + + Or expected = (Or) Expressions.or(Expressions.or(Expressions.equal("salary", 3000L), + Expressions.equal("job", 4000L)), Expressions.equal("name", 9000L)); + + Or actual = (Or) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.right().op(), expected.right().op()); + assertEquals(actual.left().op(), expected.left().op()); + } } From d2eee30d571c0b39df094c2f5b206c6fcdd9be8b Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 15 Jun 2020 14:53:04 +0100 Subject: [PATCH 47/51] Convert Hive types --- .../mr/mapred/IcebergFilterFactory.java | 79 +++++++++++++++---- .../mr/mapred/TestIcebergFilterFactory.java | 40 +++++++++- 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java index a4e3cf754ac4..868a3f3ba07f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -19,12 +19,18 @@ package org.apache.iceberg.mr.mapred; +import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Timestamp; import java.util.List; +import java.util.Set; import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import static org.apache.iceberg.expressions.Expressions.and; import static org.apache.iceberg.expressions.Expressions.equal; @@ -41,11 +47,14 @@ public class IcebergFilterFactory { private IcebergFilterFactory() {} - public static Expression generateFilterExpression(SearchArgument sarg) { - List leaves = sarg.getLeaves(); - List childNodes = sarg.getExpression().getChildren(); + private static final Set HIVE_TYPES_TO_CONVERT = ImmutableSet.of( + PredicateLeaf.Type.DATE, + PredicateLeaf.Type.DECIMAL, + PredicateLeaf.Type.TIMESTAMP + ); - return translate(sarg.getExpression(), leaves, childNodes); + public static Expression generateFilterExpression(SearchArgument sarg) { + return translate(sarg.getExpression(), sarg.getLeaves()); } /** @@ -54,23 +63,23 @@ public static Expression generateFilterExpression(SearchArgument sarg) { * @param leaves List of all leaf nodes within the tree. * @return Expression that is translated from the Hive SearchArgument. */ - private static Expression translate(ExpressionTree tree, List leaves, - List childNodes) { + private static Expression translate(ExpressionTree tree, List leaves) { + List childNodes = tree.getChildren(); switch (tree.getOperator()) { case OR: Expression orResult = Expressions.alwaysFalse(); for (ExpressionTree child : childNodes) { - orResult = or(orResult, translate(child, leaves, childNodes)); + orResult = or(orResult, translate(child, leaves)); } return orResult; case AND: Expression result = Expressions.alwaysTrue(); for (ExpressionTree child : childNodes) { - result = and(result, translate(child, leaves, childNodes)); + result = and(result, translate(child, leaves)); } return result; case NOT: - return not(translate(tree.getChildren().get(0), leaves, childNodes)); + return not(translate(tree.getChildren().get(0), leaves)); case LEAF: return translateLeaf(leaves.get(tree.getLeaf())); case CONSTANT: @@ -95,23 +104,61 @@ private static Expression translateLeaf(PredicateLeaf leaf) { } switch (leaf.getOperator()) { case EQUALS: - return equal(column, leaf.getLiteral()); + return equal(column, leafToIcebergType(leaf)); case NULL_SAFE_EQUALS: - return equal(notNull(column).ref().name(), leaf.getLiteral()); //TODO: Unsure.. + return equal(notNull(column).ref().name(), leafToIcebergType(leaf)); //TODO: Unsure.. case LESS_THAN: - return lessThan(column, leaf.getLiteral()); + return lessThan(column, leafToIcebergType(leaf)); case LESS_THAN_EQUALS: - return lessThanOrEqual(column, leaf.getLiteral()); + return lessThanOrEqual(column, leafToIcebergType(leaf)); case IN: - return in(column, leaf.getLiteralList()); + //TODO: 'in' doesn't support literals of Date or Timestamp - test for type here? + return in(column, hiveLiteralListToIcebergType(leaf.getLiteralList())); case BETWEEN: - return and(greaterThanOrEqual(column, leaf.getLiteralList().get(0)), - lessThanOrEqual(column, leaf.getLiteralList().get(1))); + List icebergLiterals = hiveLiteralListToIcebergType(leaf.getLiteralList()); + return and(greaterThanOrEqual(column, icebergLiterals.get(0)), + lessThanOrEqual(column, icebergLiterals.get(1))); case IS_NULL: return isNull(column); default: throw new IllegalStateException("Unknown operator: " + leaf.getOperator()); } } + + private static Object leafToIcebergType(PredicateLeaf leaf) { + switch (leaf.getType()) { + case LONG: + return leaf.getLiteral(); + case FLOAT: + return leaf.getLiteral(); + case STRING: + return leaf.getLiteral(); + case DATE: + return ((Date) leaf.getLiteral()).toLocalDate(); + case DECIMAL: + HiveDecimalWritable leafValue = (HiveDecimalWritable) leaf.getLiteral(); + return BigDecimal.valueOf(leafValue.doubleValue()); + case TIMESTAMP: + return ((Timestamp) leaf.getLiteral()).toLocalDateTime(); + case BOOLEAN: + return leaf.getLiteral(); + default: + throw new IllegalStateException("Unknown type: " + leaf.getType()); + } + } + + private static List hiveLiteralListToIcebergType(List hiveLiteralTypes) { + for (int i = 0; i < hiveLiteralTypes.size(); i++) { + Object type = hiveLiteralTypes.get(i); + if (type instanceof HiveDecimalWritable) { + hiveLiteralTypes.set(i, BigDecimal.valueOf(((HiveDecimalWritable) type).doubleValue())); + } else if (type instanceof Date) { + hiveLiteralTypes.set(i, ((Date) type).toLocalDate()); + } else if (type instanceof Timestamp) { + hiveLiteralTypes.set(i, ((Timestamp) type).toLocalDateTime()); + } + } + return hiveLiteralTypes; + } } diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java index 1ccdc6d424ea..d43724939727 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java @@ -19,9 +19,11 @@ package org.apache.iceberg.mr.mapred; +import java.math.BigDecimal; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.iceberg.expressions.And; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.Not; @@ -102,6 +104,21 @@ public void testInOperand() { assertEquals(actual.ref().name(), expected.ref().name()); } + @Test + public void testInOperandWithDecimal() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder.startAnd().in("date", PredicateLeaf.Type.DECIMAL, + new HiveDecimalWritable("12.14"), new HiveDecimalWritable("13.15")).end().build(); + + UnboundPredicate expected = Expressions.in("date", BigDecimal.valueOf(12.14), BigDecimal.valueOf(13.15)); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.literals(), expected.literals()); + assertEquals(actual.ref().name(), expected.ref().name()); + } + + @Test public void testBetweenOperand() { SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); @@ -149,7 +166,7 @@ public void testAndOperand() { } @Test - public void tesOrOperand() { + public void testOrOperand() { SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); SearchArgument arg = builder .startOr() @@ -209,4 +226,25 @@ public void testManyOrOperand() { assertEquals(actual.right().op(), expected.right().op()); assertEquals(actual.left().op(), expected.left().op()); } + + @Test + public void testNestedFilter() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startOr() + .equals("job", PredicateLeaf.Type.STRING, "dev") + .startAnd() + .equals("id", PredicateLeaf.Type.LONG, 3L) + .equals("dept", PredicateLeaf.Type.STRING, "300") + .end() + .end() + .build(); + + And expected = (And) Expressions.and(Expressions.or(Expressions.equal("job", "dev"), Expressions.equal( + "id", 3L)), Expressions.or(Expressions.equal("job", "dev"), Expressions.equal("dept", "300"))); + And actual = (And) IcebergFilterFactory.generateFilterExpression(arg); + assertEquals(actual.op(), expected.op()); + assertEquals(actual.right().op(), expected.right().op()); + assertEquals(actual.left().op(), expected.left().op()); + } } From 00dcf766c75f0d3b7a9e148ceb3948471b287fcf Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 15 Jun 2020 15:20:07 +0100 Subject: [PATCH 48/51] Remove system tables code --- .../mr/mapred/IcebergFilterFactory.java | 14 +--- .../iceberg/mr/mapred/IcebergInputFormat.java | 64 ++-------------- .../iceberg/mr/mapred/SystemTableUtil.java | 74 ------------------- .../iceberg/mr/mapred/TableResolver.java | 74 +++---------------- .../mr/mapred/iterables/SnapshotIterable.java | 69 ----------------- .../mr/mapred/TestIcebergInputFormat.java | 8 -- .../iceberg/mr/mapred/TestTableResolver.java | 62 ---------------- 7 files changed, 19 insertions(+), 346 deletions(-) delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java delete mode 100644 mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java delete mode 100644 mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java index 868a3f3ba07f..c240a690ab5f 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -23,14 +23,12 @@ import java.sql.Date; import java.sql.Timestamp; import java.util.List; -import java.util.Set; import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import static org.apache.iceberg.expressions.Expressions.and; import static org.apache.iceberg.expressions.Expressions.equal; @@ -47,12 +45,6 @@ public class IcebergFilterFactory { private IcebergFilterFactory() {} - private static final Set HIVE_TYPES_TO_CONVERT = ImmutableSet.of( - PredicateLeaf.Type.DATE, - PredicateLeaf.Type.DECIMAL, - PredicateLeaf.Type.TIMESTAMP - ); - public static Expression generateFilterExpression(SearchArgument sarg) { return translate(sarg.getExpression(), sarg.getLeaves()); } @@ -99,9 +91,6 @@ private static Expression translate(ExpressionTree tree, List lea */ private static Expression translateLeaf(PredicateLeaf leaf) { String column = leaf.getColumnName(); - if (column.equals(SystemTableUtil.DEFAULT_SNAPSHOT_ID_COLUMN_NAME)) { - return Expressions.alwaysTrue(); - } switch (leaf.getOperator()) { case EQUALS: return equal(column, leafToIcebergType(leaf)); @@ -136,8 +125,7 @@ private static Object leafToIcebergType(PredicateLeaf leaf) { case DATE: return ((Date) leaf.getLiteral()).toLocalDate(); case DECIMAL: - HiveDecimalWritable leafValue = (HiveDecimalWritable) leaf.getLiteral(); - return BigDecimal.valueOf(leafValue.doubleValue()); + return BigDecimal.valueOf(((HiveDecimalWritable) leaf.getLiteral()).doubleValue()); case TIMESTAMP: return ((Timestamp) leaf.getLiteral()).toLocalDateTime(); case BOOLEAN: diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java index acf1af6a21a3..847410ae0669 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergInputFormat.java @@ -22,7 +22,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.net.URI; import java.util.Iterator; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -30,7 +29,6 @@ import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; -import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; @@ -44,15 +42,12 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.Schema; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.SnapshotsTable; import org.apache.iceberg.Table; import org.apache.iceberg.data.Record; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.SerializationUtil; -import org.apache.iceberg.mr.mapred.iterables.SnapshotIterable; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,25 +61,16 @@ public class IcebergInputFormat implements InputFormat, CombineHiveI private static final Logger LOG = LoggerFactory.getLogger(IcebergInputFormat.class); private Table table; - private long currentSnapshotId; - private String virtualSnapshotIdColumnName; @Override public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { table = TableResolver.resolveTableFromJob(conf); - URI location = TableResolver.pathAsURI(conf.get(InputFormatConfig.TABLE_LOCATION)); + String location = conf.get(InputFormatConfig.TABLE_LOCATION); List tasks = planTasks(conf); - return createSplits(tasks, location.toString()); + return createSplits(tasks, location); } private List planTasks(JobConf conf) { - // Set defaults for virtual column - Snapshot currentSnapshot = table.currentSnapshot(); - if (currentSnapshot != null) { - currentSnapshotId = currentSnapshot.snapshotId(); - } - virtualSnapshotIdColumnName = SystemTableUtil.getVirtualColumnName(conf); - String[] readColumns = ColumnProjectionUtils.getReadColumnNames(conf); List tasks; if (conf.get(TableScanDesc.FILTER_EXPR_CONF_STR) == null) { @@ -98,11 +84,8 @@ private List planTasks(JobConf conf) { SearchArgument sarg = ConvertAstToSearchArg.create(conf, exprNodeDesc); Expression filter = IcebergFilterFactory.generateFilterExpression(sarg); - long snapshotIdToScan = extractSnapshotID(conf, exprNodeDesc); - tasks = Lists.newArrayList(table .newScan() - .useSnapshot(snapshotIdToScan) .select(readColumns) .filter(filter) .planTasks()); @@ -110,23 +93,6 @@ private List planTasks(JobConf conf) { return tasks; } - /** - * Search all the leaves of the expression for the 'snapshot_id' column and extract value. - * If snapshot_id column not found, return current table snapshot ID. - */ - private long extractSnapshotID(Configuration conf, ExprNodeGenericFuncDesc exprNodeDesc) { - SearchArgument sarg = ConvertAstToSearchArg.create(conf, exprNodeDesc); - List leaves = sarg.getLeaves(); - for (PredicateLeaf leaf : leaves) { - if (leaf.getColumnName().equals(virtualSnapshotIdColumnName)) { - currentSnapshotId = (long) leaf.getLiteral(); - return (long) leaf.getLiteral(); - } - } - currentSnapshotId = table.currentSnapshot().snapshotId(); - return table.currentSnapshot().snapshotId(); - } - private InputSplit[] createSplits(List tasks, String name) { InputSplit[] splits = new InputSplit[tasks.size()]; for (int i = 0; i < tasks.size(); i++) { @@ -162,12 +128,7 @@ public IcebergRecordReader(InputSplit split, JobConf conf) throws IOException { private void initialise() { tasks = split.getTask().files().iterator(); - if (table instanceof SnapshotsTable) { - reader = new SnapshotIterable(table); - recordIterator = reader.iterator(); - } else { - nextTask(); - } + nextTask(); } private void nextTask() { @@ -179,28 +140,19 @@ private void nextTask() { recordIterator = reader.iterator(); } - private Record resolveAppropriateRecordForTableType() { - if (table instanceof SnapshotsTable) { - return currentRecord; - } else { - return SystemTableUtil.recordWithVirtualColumn(currentRecord, currentSnapshotId, table.schema(), - virtualSnapshotIdColumnName); - } - } - @Override public boolean next(Void key, IcebergWritable value) { if (recordIterator.hasNext()) { currentRecord = recordIterator.next(); - value.setRecord(resolveAppropriateRecordForTableType()); + value.setRecord(currentRecord); return true; } if (tasks.hasNext()) { nextTask(); currentRecord = recordIterator.next(); - value.setRecord(resolveAppropriateRecordForTableType()); + value.setRecord(currentRecord); return true; } return false; @@ -215,11 +167,7 @@ public Void createKey() { public IcebergWritable createValue() { IcebergWritable record = new IcebergWritable(); record.setRecord(currentRecord); - if (table instanceof SnapshotsTable) { - record.setSchema(table.schema()); - } else { - record.setSchema(SystemTableUtil.schemaWithVirtualColumn(table.schema(), virtualSnapshotIdColumnName)); - } + record.setSchema(table.schema()); return record; } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java deleted file mode 100644 index b68e2f812ae1..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/SystemTableUtil.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.Schema; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.types.Types; - -public class SystemTableUtil { - - static final String VIRTUAL_COLUMN_NAME = "iceberg.hive.snapshot.virtual.column.name"; - - protected static final String DEFAULT_SNAPSHOT_ID_COLUMN_NAME = "snapshot__id"; - - private SystemTableUtil() {} - - protected static Schema schemaWithVirtualColumn(Schema schema, String columnName) { - List columns = new ArrayList<>(schema.columns()); - columns.add(Types.NestedField.optional(Integer.MAX_VALUE, columnName, Types.LongType.get())); - return new Schema(columns); - } - - protected static Record recordWithVirtualColumn(Record record, long snapshotId, Schema oldSchema, - String columnName) { - Schema newSchema = schemaWithVirtualColumn(oldSchema, columnName); - Record newRecord = GenericRecord.create(newSchema); - for (Types.NestedField field : oldSchema.columns()) { - newRecord.setField(field.name(), record.getField(field.name())); - } - newRecord.setField(columnName, snapshotId); - return newRecord; - } - - protected static String getVirtualColumnName(Configuration conf) { - String virtualColumnName = conf.get(VIRTUAL_COLUMN_NAME); - if (virtualColumnName == null) { - return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; - } else { - return virtualColumnName; - } - } - - protected static String getVirtualColumnName(Properties properties) { - String virtualColumnName = properties.getProperty(VIRTUAL_COLUMN_NAME); - if (virtualColumnName == null) { - return DEFAULT_SNAPSHOT_ID_COLUMN_NAME; - } else { - return virtualColumnName; - } - } - -} diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java index b6e2d879cbe7..600f7f48e770 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/TableResolver.java @@ -20,17 +20,14 @@ package org.apache.iceberg.mr.mapred; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Properties; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.JobConf; import org.apache.iceberg.Table; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.mr.InputFormatConfig; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; final class TableResolver { @@ -39,68 +36,28 @@ private TableResolver() { static Table resolveTableFromJob(JobConf conf) throws IOException { Properties properties = new Properties(); - properties.setProperty(InputFormatConfig.CATALOG_NAME, extractProperty(conf, InputFormatConfig.CATALOG_NAME)); - if (conf.get(InputFormatConfig.CATALOG_NAME).equals(InputFormatConfig.HADOOP_CATALOG)) { - properties.setProperty(InputFormatConfig.SNAPSHOT_TABLE, conf.get(InputFormatConfig.SNAPSHOT_TABLE, "true")); - } + properties.setProperty(InputFormatConfig.CATALOG_NAME, + conf.get(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES)); //Default to HadoopTables properties.setProperty(InputFormatConfig.TABLE_LOCATION, extractProperty(conf, InputFormatConfig.TABLE_LOCATION)); properties.setProperty(InputFormatConfig.TABLE_NAME, extractProperty(conf, InputFormatConfig.TABLE_NAME)); return resolveTableFromConfiguration(conf, properties); } static Table resolveTableFromConfiguration(Configuration conf, Properties properties) throws IOException { - String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME); - URI tableLocation = pathAsURI(properties.getProperty(InputFormatConfig.TABLE_LOCATION)); - if (catalogName == null) { - throw new IllegalArgumentException("Catalog property: 'iceberg.catalog' not set in JobConf"); - } + String catalogName = properties.getProperty(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); + String tableLocation = properties.getProperty(InputFormatConfig.TABLE_LOCATION); + String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); + Preconditions.checkNotNull(tableLocation, "Table location is not set."); + Preconditions.checkNotNull(tableName, "Table name is not set."); switch (catalogName) { case InputFormatConfig.HADOOP_TABLES: HadoopTables tables = new HadoopTables(conf); - return tables.load(tableLocation.getPath()); - case InputFormatConfig.HADOOP_CATALOG: - String tableName = properties.getProperty(InputFormatConfig.TABLE_NAME); - TableIdentifier id = TableIdentifier.parse(tableName); - if (tableName.endsWith(InputFormatConfig.SNAPSHOT_TABLE_SUFFIX)) { - if (!Boolean.parseBoolean(properties.getProperty(InputFormatConfig.SNAPSHOT_TABLE, - Boolean.TRUE.toString()))) { - String tablePath = id.toString().replaceAll("\\.", "/"); - URI warehouseLocation = pathAsURI(tableLocation.getPath().replaceAll(tablePath, "")); - HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); - return catalog.loadTable(id); - } else { - return resolveMetadataTable(conf, tableLocation.getPath(), tableName); - } - } else { - URI warehouseLocation = pathAsURI(extractWarehousePath(tableLocation.getPath(), tableName)); - HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); - return catalog.loadTable(id); - } + return tables.load(tableLocation); case InputFormatConfig.HIVE_CATALOG: //TODO Implement HiveCatalog return null; - } - return null; - } - - static Table resolveMetadataTable(Configuration conf, String location, String tableName) throws IOException { - URI warehouseLocation = pathAsURI(extractWarehousePath(location, tableName)); - HadoopCatalog catalog = new HadoopCatalog(conf, warehouseLocation.getPath()); - String baseTableName = StringUtils.removeEnd(tableName, InputFormatConfig.SNAPSHOT_TABLE_SUFFIX); - - TableIdentifier snapshotsId = TableIdentifier.parse(baseTableName + - InputFormatConfig.ICEBERG_SNAPSHOTS_TABLE_SUFFIX); - return catalog.loadTable(snapshotsId); - } - - static URI pathAsURI(String path) throws IOException { - if (path == null) { - throw new IllegalArgumentException("Path is null."); - } - try { - return new URI(path); - } catch (URISyntaxException e) { - throw new IOException("Unable to create URI for table location: '" + path + "'", e); + default: + throw new NoSuchTableException("Table does not exist at location: " + tableLocation); } } @@ -111,11 +68,4 @@ protected static String extractProperty(JobConf conf, String key) { } return value; } - - protected static String extractWarehousePath(String location, String tableName) { - String tablePath = tableName.replaceAll("\\.", "/").replaceAll( - InputFormatConfig.SNAPSHOT_TABLE_SUFFIX, ""); - return location.replaceAll(tablePath, ""); - } - } diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java deleted file mode 100644 index 4a4a7d66fd4a..000000000000 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/iterables/SnapshotIterable.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred.iterables; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.Table; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.CloseableIterator; - -/** - * Creates an Iterable of Records with all snapshot metadata that can be used with the RecordReader. - */ -public class SnapshotIterable implements CloseableIterable { - - private Table table; - - public SnapshotIterable(Table table) { - this.table = table; - } - - public CloseableIterator iterator() { - Iterable snapshots = table.snapshots(); - List snapRecords = new ArrayList<>(); - snapshots.forEach(snapshot -> snapRecords.add(createSnapshotRecord(snapshot))); - - return (CloseableIterator) snapRecords.iterator(); - } - - /** - * Populates a Record with snapshot metadata. - */ - private Record createSnapshotRecord(Snapshot snapshot) { - Record snapRecord = GenericRecord.create(table.schema()); - snapRecord.setField("committed_at", snapshot.timestampMillis()); - snapRecord.setField("snapshot_id", snapshot.snapshotId()); - snapRecord.setField("parent_id", snapshot.parentId()); - snapRecord.setField("operation", snapshot.operation()); - snapRecord.setField("manifest_list", snapshot.manifestListLocation()); - snapRecord.setField("summary", snapshot.summary()); - return snapRecord; - } - - @Override - public void close() throws IOException { - iterator().close(); - } -} diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java index 9c71aeafc1dc..6f49f981d792 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergInputFormat.java @@ -87,7 +87,6 @@ public void before() throws IOException { public void testGetSplits() throws IOException { IcebergInputFormat format = new IcebergInputFormat(); conf.set(InputFormatConfig.TABLE_LOCATION, tableLocation.getAbsolutePath()); - conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); conf.set(InputFormatConfig.TABLE_NAME, "source_db.table_a"); InputSplit[] splits = format.getSplits(conf, 1); assertEquals(splits.length, 1); @@ -100,13 +99,6 @@ public void testGetSplitsNoLocation() throws IOException { inputFormat.getSplits(conf, 1); } - @Test(expected = IllegalArgumentException.class) - public void testGetSplitsNoCatalog() throws IOException { - conf.set(InputFormatConfig.TABLE_LOCATION, "file:" + tableLocation.getAbsolutePath()); - conf.set(InputFormatConfig.TABLE_NAME, "source_db.table_a"); - inputFormat.getSplits(conf, 1); - } - @Test(expected = IllegalArgumentException.class) public void testGetSplitsNoName() throws IOException { conf.set(InputFormatConfig.CATALOG_NAME, InputFormatConfig.HADOOP_TABLES); diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java deleted file mode 100644 index e4b9c87a51c9..000000000000 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestTableResolver.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.mr.mapred; - -import org.apache.hadoop.mapred.JobConf; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class TestTableResolver { - - @Test - public void extractWarehouseLocationRegularTable() { - // This is the style of input expected from HiveConf - String testLocation = "some/folder/database/table_a"; - String testTableName = "database.table_a"; - - String expected = "some/folder/"; - String result = TableResolver.extractWarehousePath(testLocation, testTableName); - - assertEquals(expected, result); - } - - @Test - public void extractPropertyFromJobConf() { - JobConf conf = new JobConf(); - String key = "iceberg.catalog"; - String value = "hadoop.tables"; - - conf.set(key, value); - - String result = TableResolver.extractProperty(conf, key); - - assertEquals(value, result); - } - - @Test(expected = IllegalArgumentException.class) - public void extractNonExistentProperty() { - JobConf conf = new JobConf(); - String key = "iceberg.catalog"; - - TableResolver.extractProperty(conf, key); - } - -} From cb764b9162551d1b1cc15ce3a49cf479582d6a7e Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 15 Jun 2020 16:21:41 +0100 Subject: [PATCH 49/51] Add type conversion test --- .../mr/mapred/IcebergFilterFactory.java | 8 ++--- .../mr/mapred/TestIcebergFilterFactory.java | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java index c240a690ab5f..d59278e2c99b 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -123,11 +123,11 @@ private static Object leafToIcebergType(PredicateLeaf leaf) { case STRING: return leaf.getLiteral(); case DATE: - return ((Date) leaf.getLiteral()).toLocalDate(); + return ((Timestamp) leaf.getLiteral()).getTime(); case DECIMAL: return BigDecimal.valueOf(((HiveDecimalWritable) leaf.getLiteral()).doubleValue()); case TIMESTAMP: - return ((Timestamp) leaf.getLiteral()).toLocalDateTime(); + return ((Timestamp) leaf.getLiteral()).getTime(); case BOOLEAN: return leaf.getLiteral(); default: @@ -141,9 +141,9 @@ private static List hiveLiteralListToIcebergType(List hiveLitera if (type instanceof HiveDecimalWritable) { hiveLiteralTypes.set(i, BigDecimal.valueOf(((HiveDecimalWritable) type).doubleValue())); } else if (type instanceof Date) { - hiveLiteralTypes.set(i, ((Date) type).toLocalDate()); + hiveLiteralTypes.set(i, ((Timestamp) type).getTime()); } else if (type instanceof Timestamp) { - hiveLiteralTypes.set(i, ((Timestamp) type).toLocalDateTime()); + hiveLiteralTypes.set(i, ((Timestamp) type).getTime()); } } return hiveLiteralTypes; diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java index d43724939727..f0df19405463 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java @@ -20,6 +20,11 @@ package org.apache.iceberg.mr.mapred; import java.math.BigDecimal; +import java.sql.Date; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; @@ -247,4 +252,35 @@ public void testNestedFilter() { assertEquals(actual.right().op(), expected.right().op()); assertEquals(actual.left().op(), expected.left().op()); } + + @Test + public void testTypeConversion() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startAnd() + .equals("date", PredicateLeaf.Type.DATE, Date.valueOf("2020-06-15")) + .equals("timestamp", PredicateLeaf.Type.TIMESTAMP, Timestamp.valueOf("2016-11-16 06:43:19.77")) + .equals("decimal", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("12.12")) + .equals("string", PredicateLeaf.Type.STRING, "hello world") + .equals("long", PredicateLeaf.Type.LONG, 3020L) + .equals("float", PredicateLeaf.Type.FLOAT, 4400D) + .equals("boolean", PredicateLeaf.Type.BOOLEAN, true) + .end() + .build(); + + And expected = (And) Expressions.and( + Expressions.equal("date", Date.valueOf("2020-06-15").getTime()), + Expressions.equal("timestamp", Timestamp.valueOf("2016-11-16 06:43:19.77").getTime()), + Expressions.equal("decimal", BigDecimal.valueOf(12.12)), + Expressions.equal("string", "hello world"), + Expressions.equal("long", 3020L), + Expressions.equal("float", 4400D), + Expressions.equal("boolean", true)); + + And actual = (And) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(expected.toString(), actual.toString()); + assertEquals(expected.op(), actual.op()); + + } } From 0f0b39d767e35116c7e842166369bfdf282a789b Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Mon, 15 Jun 2020 16:59:24 +0100 Subject: [PATCH 50/51] Removing old todo comment --- .../java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java | 1 - 1 file changed, 1 deletion(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java index d59278e2c99b..dc257781e765 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -101,7 +101,6 @@ private static Expression translateLeaf(PredicateLeaf leaf) { case LESS_THAN_EQUALS: return lessThanOrEqual(column, leafToIcebergType(leaf)); case IN: - //TODO: 'in' doesn't support literals of Date or Timestamp - test for type here? return in(column, hiveLiteralListToIcebergType(leaf.getLiteralList())); case BETWEEN: List icebergLiterals = hiveLiteralListToIcebergType(leaf.getLiteralList()); From a6eb6c786b782b1bfb77a7f951e38b699f81463a Mon Sep 17 00:00:00 2001 From: Christine Mathiesen Date: Tue, 16 Jun 2020 14:52:54 +0100 Subject: [PATCH 51/51] Timestamps in microseconds --- .../mr/mapred/IcebergFilterFactory.java | 55 +++++++++++-------- .../mr/mapred/TestIcebergFilterFactory.java | 55 ++++++++++++++++--- 2 files changed, 79 insertions(+), 31 deletions(-) diff --git a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java index dc257781e765..ada7b78fa94a 100644 --- a/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java +++ b/mr/src/main/java/org/apache/iceberg/mr/mapred/IcebergFilterFactory.java @@ -101,9 +101,9 @@ private static Expression translateLeaf(PredicateLeaf leaf) { case LESS_THAN_EQUALS: return lessThanOrEqual(column, leafToIcebergType(leaf)); case IN: - return in(column, hiveLiteralListToIcebergType(leaf.getLiteralList())); + return in(column, (List) leafToIcebergType(leaf)); case BETWEEN: - List icebergLiterals = hiveLiteralListToIcebergType(leaf.getLiteralList()); + List icebergLiterals = leaf.getLiteralList(); return and(greaterThanOrEqual(column, icebergLiterals.get(0)), lessThanOrEqual(column, icebergLiterals.get(1))); case IS_NULL: @@ -116,36 +116,43 @@ private static Expression translateLeaf(PredicateLeaf leaf) { private static Object leafToIcebergType(PredicateLeaf leaf) { switch (leaf.getType()) { case LONG: - return leaf.getLiteral(); + return leaf.getLiteral() != null ? leaf.getLiteral() : leaf.getLiteralList(); case FLOAT: - return leaf.getLiteral(); + return leaf.getLiteral() != null ? leaf.getLiteral() : leaf.getLiteralList(); case STRING: - return leaf.getLiteral(); + return leaf.getLiteral() != null ? leaf.getLiteral() : leaf.getLiteralList(); case DATE: - return ((Timestamp) leaf.getLiteral()).getTime(); + //Hive converts a Date type to a Timestamp internally when retrieving literal + if (leaf.getLiteral() != null) { + return ((Timestamp) leaf.getLiteral()).toLocalDateTime().toLocalDate().toEpochDay(); + } else { + //But not when retrieving the literalList + List icebergValues = leaf.getLiteralList(); + icebergValues.replaceAll(value -> ((Date) value).toLocalDate().toEpochDay()); + return icebergValues; + } case DECIMAL: - return BigDecimal.valueOf(((HiveDecimalWritable) leaf.getLiteral()).doubleValue()); + if (leaf.getLiteral() != null) { + return BigDecimal.valueOf(((HiveDecimalWritable) leaf.getLiteral()).doubleValue()); + } else { + List icebergValues = leaf.getLiteralList(); + icebergValues.replaceAll(value -> BigDecimal.valueOf(((HiveDecimalWritable) value).doubleValue())); + return icebergValues; + } case TIMESTAMP: - return ((Timestamp) leaf.getLiteral()).getTime(); + if (leaf.getLiteral() != null) { + Timestamp timestamp = (Timestamp) leaf.getLiteral(); + return timestamp.toInstant().getEpochSecond() * 1000000 + timestamp.getNanos() / 1000; + } else { + List icebergValues = leaf.getLiteralList(); + icebergValues.replaceAll(value -> ( + (Timestamp) value).toInstant().getEpochSecond() * 1000000 + ((Timestamp) value).getNanos() / 1000); + return icebergValues; + } case BOOLEAN: - return leaf.getLiteral(); + return leaf.getLiteral() != null ? leaf.getLiteral() : leaf.getLiteralList(); default: throw new IllegalStateException("Unknown type: " + leaf.getType()); } } - - private static List hiveLiteralListToIcebergType(List hiveLiteralTypes) { - for (int i = 0; i < hiveLiteralTypes.size(); i++) { - Object type = hiveLiteralTypes.get(i); - if (type instanceof HiveDecimalWritable) { - hiveLiteralTypes.set(i, BigDecimal.valueOf(((HiveDecimalWritable) type).doubleValue())); - } else if (type instanceof Date) { - hiveLiteralTypes.set(i, ((Timestamp) type).getTime()); - } else if (type instanceof Timestamp) { - hiveLiteralTypes.set(i, ((Timestamp) type).getTime()); - } - } - return hiveLiteralTypes; - } } - diff --git a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java index f0df19405463..4c5338e13233 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java +++ b/mr/src/test/java/org/apache/iceberg/mr/mapred/TestIcebergFilterFactory.java @@ -23,8 +23,6 @@ import java.sql.Date; import java.sql.Timestamp; import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneOffset; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; @@ -97,7 +95,7 @@ public void testLessThanEqualsOperand() { } @Test - public void testInOperand() { + public void testInOperandWithLong() { SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); SearchArgument arg = builder.startAnd().in("salary", PredicateLeaf.Type.LONG, 3000L, 4000L).end().build(); @@ -112,10 +110,10 @@ public void testInOperand() { @Test public void testInOperandWithDecimal() { SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); - SearchArgument arg = builder.startAnd().in("date", PredicateLeaf.Type.DECIMAL, + SearchArgument arg = builder.startAnd().in("decimal", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("12.14"), new HiveDecimalWritable("13.15")).end().build(); - UnboundPredicate expected = Expressions.in("date", BigDecimal.valueOf(12.14), BigDecimal.valueOf(13.15)); + UnboundPredicate expected = Expressions.in("decimal", BigDecimal.valueOf(12.14), BigDecimal.valueOf(13.15)); UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); assertEquals(actual.op(), expected.op()); @@ -123,6 +121,48 @@ public void testInOperandWithDecimal() { assertEquals(actual.ref().name(), expected.ref().name()); } + @Test + public void testInOperandWithDate() { + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startAnd() + .in("date", PredicateLeaf.Type.DATE, + Date.valueOf("2020-06-15"), Date.valueOf("2021-06-15")) + .end() + .build(); + + UnboundPredicate expected = Expressions.in("date", LocalDate.of(2020, 6, 15).toEpochDay(), + LocalDate.of(2021, 6, 15).toEpochDay()); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(actual.op(), expected.op()); + assertEquals(actual.literals(), expected.literals()); + assertEquals(actual.ref().name(), expected.ref().name()); + assertEquals(expected.toString(), actual.toString()); + } + + @Test + public void testInOperandWithTimestamp() { + Timestamp timestampHiveFilterOne = Timestamp.valueOf("2016-11-16 06:43:19.77"); + Timestamp timestampHiveFilterTwo = Timestamp.valueOf("2017-11-16 06:43:19.77"); + + SearchArgument.Builder builder = SearchArgumentFactory.newBuilder(); + SearchArgument arg = builder + .startAnd() + .in("timestamp", PredicateLeaf.Type.TIMESTAMP, timestampHiveFilterOne, timestampHiveFilterTwo) + .end() + .build(); + + UnboundPredicate expected = Expressions.in("timestamp", + 1479278599770000L, 1510814599770000L); + UnboundPredicate actual = (UnboundPredicate) IcebergFilterFactory.generateFilterExpression(arg); + + assertEquals(expected.op(), actual.op()); + assertEquals(expected.literals(), actual.literals()); + assertEquals(expected.ref().name(), actual.ref().name()); + assertEquals(expected.toString(), actual.toString()); + } + @Test public void testBetweenOperand() { @@ -268,9 +308,10 @@ public void testTypeConversion() { .end() .build(); + Timestamp timestamp = Timestamp.valueOf("2016-11-16 06:43:19.77"); And expected = (And) Expressions.and( - Expressions.equal("date", Date.valueOf("2020-06-15").getTime()), - Expressions.equal("timestamp", Timestamp.valueOf("2016-11-16 06:43:19.77").getTime()), + Expressions.equal("date", LocalDate.of(2020, 6, 15).toEpochDay()), + Expressions.equal("timestamp", 1479278599770000L), Expressions.equal("decimal", BigDecimal.valueOf(12.12)), Expressions.equal("string", "hello world"), Expressions.equal("long", 3020L),