diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml index cdf8fbee109..184300fe54e 100644 --- a/contrib/storage-hive/core/pom.xml +++ b/contrib/storage-hive/core/pom.xml @@ -63,12 +63,6 @@ org.apache.hive hive-hbase-handler - - - org.apache.hive - hive-exec - - org.apache.hbase diff --git a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java index 4a75ed336cd..a6e588b84d8 100644 --- a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java +++ b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java @@ -130,7 +130,7 @@ public int next() { while (!recordsInspector.isBatchFull() && hasNextValue(recordsInspector.getValueHolder())) { Object value = recordsInspector.getNextValue(); if (value != null) { - Object deSerializedValue = partitionSerDe.deserialize((Writable) value); + Object deSerializedValue = partitionDeserializer.deserialize((Writable) value); if (partTblObjectInspectorConverter != null) { deSerializedValue = partTblObjectInspectorConverter.convert(deSerializedValue); } @@ -159,7 +159,7 @@ public int next() { try { int recordCount = 0; while (recordCount < TARGET_RECORD_COUNT && hasNextValue(value)) { - Object deSerializedValue = partitionSerDe.deserialize((Writable) value); + Object deSerializedValue = partitionDeserializer.deserialize((Writable) value); if (partTblObjectInspectorConverter != null) { deSerializedValue = partTblObjectInspectorConverter.convert(deSerializedValue); } diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java index d0259ca1413..b11ef3b7a26 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java @@ -264,6 +264,7 @@ public List run() throws Exception { final List splits = Lists.newArrayList(); final JobConf job = new JobConf(hiveConf); HiveUtilities.addConfToJob(job, properties); + HiveUtilities.verifyAndAddTransactionalProperties(job, sd); job.setInputFormat(HiveUtilities.getInputFormatClass(job, sd, hiveReadEntry.getTable())); final Path path = new Path(sd.getLocation()); final FileSystem fs = path.getFileSystem(job); diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java index b101f497748..05b7e899e06 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java @@ -17,8 +17,11 @@ */ package org.apache.drill.exec.store.hive; +import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.Lists; import io.netty.buffer.DrillBuf; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; @@ -51,10 +54,14 @@ import org.apache.drill.exec.work.ExecErrorConstants; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.serde.serdeConstants; @@ -70,6 +77,7 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import javax.annotation.Nullable; import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; @@ -104,8 +112,7 @@ public static Object convertPartitionType(TypeInfo typeInfo, String value, final return Boolean.parseBoolean(value); case DECIMAL: { DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; - return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create(value), - decimalTypeInfo.precision(), decimalTypeInfo.scale()); + return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create(value), decimalTypeInfo); } case DOUBLE: return Double.parseDouble(value); @@ -507,5 +514,59 @@ public static boolean hasHeaderOrFooter(HiveTableWithColumnCache table) { int skipFooter = retrieveIntProperty(tableProperties, serdeConstants.FOOTER_COUNT, -1); return skipHeader > 0 || skipFooter > 0; } + + /** + * This method checks whether the table is transactional and set necessary properties in {@link JobConf}. + * If schema evolution properties aren't set in job conf for the input format, method sets the column names + * and types from table/partition properties or storage descriptor. + * + * @param job the job to update + * @param sd storage descriptor + */ + public static void verifyAndAddTransactionalProperties(JobConf job, StorageDescriptor sd) { + + if (AcidUtils.isTablePropertyTransactional(job)) { + AcidUtils.setTransactionalTableScan(job, true); + + // No work is needed, if schema evolution is used + if (Utilities.isSchemaEvolutionEnabled(job, true) && job.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS) != null && + job.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES) != null) { + return; + } + + String colNames; + String colTypes; + + // Try to get get column names and types from table or partition properties. If they are absent there, get columns + // data from storage descriptor of the table + colNames = job.get(serdeConstants.LIST_COLUMNS); + colTypes = job.get(serdeConstants.LIST_COLUMN_TYPES); + + if (colNames == null || colTypes == null) { + colNames = Joiner.on(",").join(Lists.transform(sd.getCols(), new Function() + { + @Nullable + @Override + public String apply(@Nullable FieldSchema input) + { + return input.getName(); + } + })); + + colTypes = Joiner.on(",").join(Lists.transform(sd.getCols(), new Function() + { + @Nullable + @Override + public String apply(@Nullable FieldSchema input) + { + return input.getType(); + } + })); + } + + job.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS, colNames); + job.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, colTypes); + } + } } diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java index 9df721b2941..b814866109f 100644 --- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java +++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java @@ -50,7 +50,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; @@ -85,10 +85,10 @@ public abstract class HiveAbstractReader extends AbstractRecordReader { protected List selectedPartitionTypes = Lists.newArrayList(); protected List selectedPartitionValues = Lists.newArrayList(); - // SerDe of the reading partition (or table if the table is non-partitioned) - protected SerDe partitionSerDe; + // Deserializer of the reading partition (or table if the table is non-partitioned) + protected Deserializer partitionDeserializer; - // ObjectInspector to read data from partitionSerDe (for a non-partitioned table this is same as the table + // ObjectInspector to read data from partitionDeserializer (for a non-partitioned table this is same as the table // ObjectInspector). protected StructObjectInspector partitionOI; @@ -143,19 +143,20 @@ private void init() throws ExecutionSetupException { HiveUtilities.getPartitionMetadata(partition, table); HiveUtilities.addConfToJob(job, partitionProperties); - final SerDe tableSerDe = createSerDe(job, table.getSd().getSerdeInfo().getSerializationLib(), tableProperties); - final StructObjectInspector tableOI = getStructOI(tableSerDe); + final Deserializer tableDeserializer = createDeserializer(job, table.getSd().getSerdeInfo().getSerializationLib(), tableProperties); + final StructObjectInspector tableOI = getStructOI(tableDeserializer); if (partition != null) { - partitionSerDe = createSerDe(job, partition.getSd().getSerdeInfo().getSerializationLib(), partitionProperties); - partitionOI = getStructOI(partitionSerDe); + partitionDeserializer = createDeserializer(job, partition.getSd().getSerdeInfo().getSerializationLib(), partitionProperties); + partitionOI = getStructOI(partitionDeserializer); finalOI = (StructObjectInspector)ObjectInspectorConverters.getConvertedOI(partitionOI, tableOI); partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(partitionOI, finalOI); job.setInputFormat(HiveUtilities.getInputFormatClass(job, partition.getSd(), table)); + HiveUtilities.verifyAndAddTransactionalProperties(job, table.getSd()); } else { // For non-partitioned tables, there is no need to create converter as there are no schema changes expected. - partitionSerDe = tableSerDe; + partitionDeserializer = tableDeserializer; partitionOI = tableOI; partTblObjectInspectorConverter = null; finalOI = tableOI; @@ -166,7 +167,7 @@ private void init() throws ExecutionSetupException { for (StructField field: finalOI.getAllStructFieldRefs()) { logger.trace("field in finalOI: {}", field.getClass().getName()); } - logger.trace("partitionSerDe class is {} {}", partitionSerDe.getClass().getName()); + logger.trace("partitionDeserializer class is {} {}", partitionDeserializer.getClass().getName()); } // Get list of partition column names final List partitionNames = Lists.newArrayList(); @@ -176,8 +177,8 @@ private void init() throws ExecutionSetupException { // We should always get the columns names from ObjectInspector. For some of the tables (ex. avro) metastore // may not contain the schema, instead it is derived from other sources such as table properties or external file. - // SerDe object knows how to get the schema with all the config and table properties passed in initialization. - // ObjectInspector created from the SerDe object has the schema. + // Deserializer object knows how to get the schema with all the config and table properties passed in initialization. + // ObjectInspector created from the Deserializer object has the schema. final StructTypeInfo sTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(finalOI); final List tableColumnNames = sTypeInfo.getAllStructFieldNames(); @@ -201,7 +202,20 @@ private void init() throws ExecutionSetupException { } } } - ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames); + ColumnProjectionUtils.appendReadColumns(job, columnIds); + + // TODO: Use below overloaded method instead of above simpler version of it, once Hive client dependencies + // (from all profiles) will be updated to 2.3 version or above +// ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames, +// Lists.newArrayList(Iterables.transform(getColumns(), new Function() +// { +// @Nullable +// @Override +// public String apply(@Nullable SchemaPath path) +// { +// return path.getRootSegmentPath(); +// } +// }))); for (String columnName : selectedColumnNames) { StructField fieldRef = finalOI.getStructFieldRef(columnName); @@ -269,18 +283,19 @@ protected boolean initNextReader(JobConf job) throws ExecutionSetupException { } /** - * Utility method which creates a SerDe object for given SerDe class name and properties. + * Utility method which creates a Deserializer object for given Deserializer class name and properties. + * TODO: Replace Deserializer interface with AbstractSerDe, once all Hive clients is upgraded to 2.3 version */ - private static SerDe createSerDe(final JobConf job, final String sLib, final Properties properties) throws Exception { - final Class c = Class.forName(sLib).asSubclass(SerDe.class); - final SerDe serde = c.getConstructor().newInstance(); - serde.initialize(job, properties); + private static Deserializer createDeserializer(final JobConf job, final String sLib, final Properties properties) throws Exception { + final Class c = Class.forName(sLib).asSubclass(Deserializer.class); + final Deserializer deserializer = c.getConstructor().newInstance(); + deserializer.initialize(job, properties); - return serde; + return deserializer; } - private static StructObjectInspector getStructOI(final SerDe serDe) throws Exception { - ObjectInspector oi = serDe.getObjectInspector(); + private static StructObjectInspector getStructOI(final Deserializer deserializer) throws Exception { + ObjectInspector oi = deserializer.getObjectInspector(); if (oi.getCategory() != ObjectInspector.Category.STRUCT) { throw new UnsupportedOperationException(String.format("%s category not supported", oi.getCategory())); } diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java index c2412ad7e7f..9f46e6699ee 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java @@ -392,6 +392,16 @@ public void readFromAlteredPartitionedTable() throws Exception { .go(); } + @Test // DRILL-3938 + public void readFromAlteredPartitionedTableWithEmptyGroupType() throws Exception { + testBuilder() + .sqlQuery("SELECT newcol FROM hive.kv_parquet LIMIT 1") + .unOrdered() + .baselineColumns("newcol") + .baselineValues(new Object[]{null}) + .go(); + } + @Test // DRILL-3938 public void nativeReaderIsDisabledForAlteredPartitionedTable() throws Exception { try { diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java index 3862dc661ca..a289af9ea3c 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/BaseTestHiveImpersonation.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -72,6 +72,9 @@ protected static void prepHiveConfAndData() throws Exception { hiveConf.set(ConfVars.SCRATCHDIR.varname, "file://" + scratchDir.getAbsolutePath()); hiveConf.set(ConfVars.LOCALSCRATCHDIR.varname, localScratchDir.getAbsolutePath()); + hiveConf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false"); + hiveConf.set(ConfVars.METASTORE_AUTO_CREATE_ALL.varname, "true"); + hiveConf.set(ConfVars.HIVE_CBO_ENABLED.varname, "false"); // Set MiniDFS conf in HiveConf hiveConf.set(FS_DEFAULT_NAME_KEY, dfsConf.get(FS_DEFAULT_NAME_KEY)); diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java index e0038655a4c..ef6c547a38b 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestSqlStdBasedAuthorization.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -41,9 +41,12 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_CBO_ENABLED; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION; @Category({SlowTest.class, HiveStorageTest.class}) public class TestSqlStdBasedAuthorization extends BaseTestHiveImpersonation { @@ -101,6 +104,9 @@ private static Map getHivePluginConfig() { hiveConfig.put(HIVE_AUTHORIZATION_ENABLED.varname, hiveConf.get(HIVE_AUTHORIZATION_ENABLED.varname)); hiveConfig.put(HIVE_AUTHENTICATOR_MANAGER.varname, SessionStateUserAuthenticator.class.getName()); hiveConfig.put(HIVE_AUTHORIZATION_MANAGER.varname, SQLStdHiveAuthorizerFactory.class.getName()); + hiveConfig.put(METASTORE_SCHEMA_VERIFICATION.varname, hiveConf.get(METASTORE_SCHEMA_VERIFICATION.varname)); + hiveConfig.put(METASTORE_AUTO_CREATE_ALL.varname, hiveConf.get(METASTORE_AUTO_CREATE_ALL.varname)); + hiveConfig.put(HIVE_CBO_ENABLED.varname, hiveConf.get(HIVE_CBO_ENABLED.varname)); return hiveConfig; } diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java index 685d3bf6320..972c5451073 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/impersonation/hive/TestStorageBasedHiveAuthorization.java @@ -41,13 +41,16 @@ import static org.apache.drill.exec.hive.HiveTestUtilities.executeQuery; import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_CBO_ENABLED; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_SCHEMA_VERIFICATION; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.DYNAMICPARTITIONINGMODE; @Category({SlowTest.class, HiveStorageTest.class}) @@ -136,6 +139,9 @@ private static Map getHivePluginConfig() { hiveConfig.put(FS_DEFAULT_NAME_KEY, dfsConf.get(FS_DEFAULT_NAME_KEY)); hiveConfig.put(HIVE_SERVER2_ENABLE_DOAS.varname, hiveConf.get(HIVE_SERVER2_ENABLE_DOAS.varname)); hiveConfig.put(METASTORE_EXECUTE_SET_UGI.varname, hiveConf.get(METASTORE_EXECUTE_SET_UGI.varname)); + hiveConfig.put(METASTORE_SCHEMA_VERIFICATION.varname, hiveConf.get(METASTORE_SCHEMA_VERIFICATION.varname)); + hiveConfig.put(METASTORE_AUTO_CREATE_ALL.varname, hiveConf.get(METASTORE_AUTO_CREATE_ALL.varname)); + hiveConfig.put(HIVE_CBO_ENABLED.varname, hiveConf.get(HIVE_CBO_ENABLED.varname)); return hiveConfig; } diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java index 924d7cbae9f..93786d08092 100644 --- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java +++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java @@ -149,6 +149,9 @@ private void generateTestData() throws Exception { conf.set(ConfVars.SCRATCHDIR.varname, scratchDir.getAbsolutePath()); conf.set(ConfVars.LOCALSCRATCHDIR.varname, localScratchDir.getAbsolutePath()); conf.set(ConfVars.DYNAMICPARTITIONINGMODE.varname, "nonstrict"); + conf.set(ConfVars.METASTORE_AUTO_CREATE_ALL.varname, "true"); + conf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false"); + conf.set(ConfVars.HIVE_CBO_ENABLED.varname, "false"); SessionState ss = new SessionState(conf); SessionState.start(ss); diff --git a/contrib/storage-hive/hive-exec-shade/pom.xml b/contrib/storage-hive/hive-exec-shade/pom.xml index de9e0551760..b1c1ab470e3 100644 --- a/contrib/storage-hive/hive-exec-shade/pom.xml +++ b/contrib/storage-hive/hive-exec-shade/pom.xml @@ -34,89 +34,92 @@ hive-exec compile + - log4j - log4j + org.apache.calcite + calcite-core - commons-codec - commons-codec + org.apache.calcite + calcite-avatica - calcite-avatica org.apache.calcite + calcite-linq4j + + + org.apache.calcite + calcite-druid + + + org.apache.parquet + parquet-column + - org.apache.maven.plugins maven-shade-plugin - 2.1 - - - package - - shade - - - - - org.apache.hive:hive-exec - com.twitter:parquet-column - com.twitter:parquet-hadoop - commons-codec:commons-codec - com.twitter:parquet-format - com.twitter:parquet-common - com.twitter:parquet-jackson - com.twitter:parquet-encoding - com.twitter:parquet-generator - org.apache.calcite:calcite-core - org.apache.calcite.avatica:avatica-core - - - false - true - - - com.google. - hive.com.google. - - - parquet. - hive.parquet. - - - org.apache.commons.codec. - hive.org.apache.commons.codec. - - - net.hydromatic. - hive.net.hydromatic. - - - org.eigenbase. - hive.org.eigenbase. - - - org.apache.calcite. - hive.org.apache.calcite. - - - - - org.apache.hive:hive-exec - - org/json/* - - - - - - + + + + org.apache.hive:hive-exec + org.apache.parquet:parquet-column + commons-codec:commons-codec + com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-annotations + com.fasterxml.jackson.core:jackson-core + + + false + true + + + com.google. + hive.com.google. + + + org.apache.commons.codec. + hive.org.apache.commons.codec. + + + net.hydromatic. + hive.net.hydromatic. + + + org.eigenbase. + hive.org.eigenbase. + + + com.fasterxml.jackson. + hive.com.fasterxml.jackson. + + + org.apache.parquet. + hive.org.apache.parquet. + + + + + org.apache.hive:hive-exec + + org/apache/parquet/schema/* + + + + org.apache.maven.plugins @@ -153,4 +156,23 @@ + + + mapr + + + 2.4.2 + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.databind.mapr.hive.version} + + + + + diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 8e64224e502..9c121851917 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -236,21 +236,6 @@ org.apache.parquet parquet-column - ${parquet.version} - - - org.apache.hadoop - hadoop-client - - - org.apache.hadoop - hadoop-common - - - commons-codec - commons-codec - - org.apache.parquet diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml index 80d371645a2..5b3486c341f 100644 --- a/exec/jdbc-all/pom.xml +++ b/exec/jdbc-all/pom.xml @@ -269,17 +269,7 @@ - org.apache.maven.plugins maven-shade-plugin - 2.4.1 - - - package - - shade - - - false true @@ -583,17 +573,7 @@ - org.apache.maven.plugins maven-shade-plugin - 2.4.1 - - - package - - shade - - - false true diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml index d7087f402cf..e463c9608dc 100644 --- a/exec/jdbc/pom.xml +++ b/exec/jdbc/pom.xml @@ -62,7 +62,6 @@ com.fasterxml.jackson.core jackson-core - ${jackson.version} com.fasterxml.jackson.core diff --git a/pom.xml b/pom.xml index 200b44a295f..2c1f5fcf8b7 100644 --- a/pom.xml +++ b/pom.xml @@ -52,10 +52,10 @@ 1.3 - 1.2.1 + 2.3.2 2.7.1 1.1.3 1.0 @@ -587,6 +587,19 @@ + + org.apache.maven.plugins + maven-shade-plugin + 3.1.0 + + + package + + shade + + + + @@ -885,6 +898,22 @@ io.netty netty-all + + javax.servlet + servlet-api + + + org.mortbay.jetty + servlet-api-2.5 + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + log4j + log4j + @@ -892,6 +921,10 @@ hive-metastore ${hive.version} + + log4j + log4j + org.apache.hive hive-serde @@ -923,6 +956,10 @@ hive-hbase-handler ${hive.version} + + log4j + log4j + org.slf4j slf4j-log4j12 @@ -996,6 +1033,11 @@ + + io.dropwizard.metrics + metrics-core + 4.0.2 + org.codehaus.janino janino @@ -1011,6 +1053,11 @@ jackson-databind ${jackson.databind.version} + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + com.mapr.db maprdb @@ -1139,6 +1186,25 @@ 1.8 runtime + + org.apache.parquet + parquet-column + ${parquet.version} + + + org.apache.hadoop + hadoop-client + + + org.apache.hadoop + hadoop-common + + + commons-codec + commons-codec + + + @@ -1875,7 +1941,7 @@ mapr true - 1.2.0-mapr-1707 + 2.1.1-mapr-1710 1.1.1-mapr-1602-m7-5.2.0 2.7.0-mapr-1707