diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index cdf8fbee109..184300fe54e 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -63,12 +63,6 @@
org.apache.hive
hive-hbase-handler
-
-
- org.apache.hive
- hive-exec
-
-
org.apache.hbase
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
index 4a75ed336cd..a6e588b84d8 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/HiveRecordReaders.java
@@ -130,7 +130,7 @@ public int next() {
while (!recordsInspector.isBatchFull() && hasNextValue(recordsInspector.getValueHolder())) {
Object value = recordsInspector.getNextValue();
if (value != null) {
- Object deSerializedValue = partitionSerDe.deserialize((Writable) value);
+ Object deSerializedValue = partitionDeserializer.deserialize((Writable) value);
if (partTblObjectInspectorConverter != null) {
deSerializedValue = partTblObjectInspectorConverter.convert(deSerializedValue);
}
@@ -159,7 +159,7 @@ public int next() {
try {
int recordCount = 0;
while (recordCount < TARGET_RECORD_COUNT && hasNextValue(value)) {
- Object deSerializedValue = partitionSerDe.deserialize((Writable) value);
+ Object deSerializedValue = partitionDeserializer.deserialize((Writable) value);
if (partTblObjectInspectorConverter != null) {
deSerializedValue = partTblObjectInspectorConverter.convert(deSerializedValue);
}
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
index d0259ca1413..b11ef3b7a26 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveMetadataProvider.java
@@ -264,6 +264,7 @@ public List run() throws Exception {
final List splits = Lists.newArrayList();
final JobConf job = new JobConf(hiveConf);
HiveUtilities.addConfToJob(job, properties);
+ HiveUtilities.verifyAndAddTransactionalProperties(job, sd);
job.setInputFormat(HiveUtilities.getInputFormatClass(job, sd, hiveReadEntry.getTable()));
final Path path = new Path(sd.getLocation());
final FileSystem fs = path.getFileSystem(job);
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
index b101f497748..05b7e899e06 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveUtilities.java
@@ -17,8 +17,11 @@
*/
package org.apache.drill.exec.store.hive;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import io.netty.buffer.DrillBuf;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -51,10 +54,14 @@
import org.apache.drill.exec.work.ExecErrorConstants;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.io.IOConstants;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -70,6 +77,7 @@
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
+import javax.annotation.Nullable;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
@@ -104,8 +112,7 @@ public static Object convertPartitionType(TypeInfo typeInfo, String value, final
return Boolean.parseBoolean(value);
case DECIMAL: {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
- return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create(value),
- decimalTypeInfo.precision(), decimalTypeInfo.scale());
+ return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create(value), decimalTypeInfo);
}
case DOUBLE:
return Double.parseDouble(value);
@@ -507,5 +514,59 @@ public static boolean hasHeaderOrFooter(HiveTableWithColumnCache table) {
int skipFooter = retrieveIntProperty(tableProperties, serdeConstants.FOOTER_COUNT, -1);
return skipHeader > 0 || skipFooter > 0;
}
+
+ /**
+ * This method checks whether the table is transactional and set necessary properties in {@link JobConf}.
+ * If schema evolution properties aren't set in job conf for the input format, method sets the column names
+ * and types from table/partition properties or storage descriptor.
+ *
+ * @param job the job to update
+ * @param sd storage descriptor
+ */
+ public static void verifyAndAddTransactionalProperties(JobConf job, StorageDescriptor sd) {
+
+ if (AcidUtils.isTablePropertyTransactional(job)) {
+ AcidUtils.setTransactionalTableScan(job, true);
+
+ // No work is needed, if schema evolution is used
+ if (Utilities.isSchemaEvolutionEnabled(job, true) && job.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS) != null &&
+ job.get(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES) != null) {
+ return;
+ }
+
+ String colNames;
+ String colTypes;
+
+ // Try to get get column names and types from table or partition properties. If they are absent there, get columns
+ // data from storage descriptor of the table
+ colNames = job.get(serdeConstants.LIST_COLUMNS);
+ colTypes = job.get(serdeConstants.LIST_COLUMN_TYPES);
+
+ if (colNames == null || colTypes == null) {
+ colNames = Joiner.on(",").join(Lists.transform(sd.getCols(), new Function()
+ {
+ @Nullable
+ @Override
+ public String apply(@Nullable FieldSchema input)
+ {
+ return input.getName();
+ }
+ }));
+
+ colTypes = Joiner.on(",").join(Lists.transform(sd.getCols(), new Function()
+ {
+ @Nullable
+ @Override
+ public String apply(@Nullable FieldSchema input)
+ {
+ return input.getType();
+ }
+ }));
+ }
+
+ job.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS, colNames);
+ job.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, colTypes);
+ }
+ }
}
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
index 9df721b2941..b814866109f 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/readers/HiveAbstractReader.java
@@ -50,7 +50,7 @@
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
-import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
@@ -85,10 +85,10 @@ public abstract class HiveAbstractReader extends AbstractRecordReader {
protected List selectedPartitionTypes = Lists.newArrayList();
protected List
+
+
+ org.apache.parquet
+ parquet-column
+
- org.apache.maven.plugins
maven-shade-plugin
- 2.1
-
-
- package
-
- shade
-
-
-
-
- org.apache.hive:hive-exec
- com.twitter:parquet-column
- com.twitter:parquet-hadoop
- commons-codec:commons-codec
- com.twitter:parquet-format
- com.twitter:parquet-common
- com.twitter:parquet-jackson
- com.twitter:parquet-encoding
- com.twitter:parquet-generator
- org.apache.calcite:calcite-core
- org.apache.calcite.avatica:avatica-core
-
-
- false
- true
-
-
- com.google.
- hive.com.google.
-
-
- parquet.
- hive.parquet.
-
-
- org.apache.commons.codec.
- hive.org.apache.commons.codec.
-
-
- net.hydromatic.
- hive.net.hydromatic.
-
-
- org.eigenbase.
- hive.org.eigenbase.
-
-
- org.apache.calcite.
- hive.org.apache.calcite.
-
-
-
-
- org.apache.hive:hive-exec
-
- org/json/*
-
-
-
-
-
-
+
+
+
+ org.apache.hive:hive-exec
+ org.apache.parquet:parquet-column
+ commons-codec:commons-codec
+ com.fasterxml.jackson.core:jackson-databind
+ com.fasterxml.jackson.core:jackson-annotations
+ com.fasterxml.jackson.core:jackson-core
+
+
+ false
+ true
+
+
+ com.google.
+ hive.com.google.
+
+
+ org.apache.commons.codec.
+ hive.org.apache.commons.codec.
+
+
+ net.hydromatic.
+ hive.net.hydromatic.
+
+
+ org.eigenbase.
+ hive.org.eigenbase.
+
+
+ com.fasterxml.jackson.
+ hive.com.fasterxml.jackson.
+
+
+ org.apache.parquet.
+ hive.org.apache.parquet.
+
+
+
+
+ org.apache.hive:hive-exec
+
+ org/apache/parquet/schema/*
+
+
+
+
org.apache.maven.plugins
@@ -153,4 +156,23 @@
+
+
+ mapr
+
+
+ 2.4.2
+
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${jackson.databind.mapr.hive.version}
+
+
+
+
+
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 8e64224e502..9c121851917 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -236,21 +236,6 @@
org.apache.parquet
parquet-column
- ${parquet.version}
-
-
- org.apache.hadoop
- hadoop-client
-
-
- org.apache.hadoop
- hadoop-common
-
-
- commons-codec
- commons-codec
-
-
org.apache.parquet
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 80d371645a2..5b3486c341f 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -269,17 +269,7 @@
- org.apache.maven.plugins
maven-shade-plugin
- 2.4.1
-
-
- package
-
- shade
-
-
-
false
true
@@ -583,17 +573,7 @@
- org.apache.maven.plugins
maven-shade-plugin
- 2.4.1
-
-
- package
-
- shade
-
-
-
false
true
diff --git a/exec/jdbc/pom.xml b/exec/jdbc/pom.xml
index d7087f402cf..e463c9608dc 100644
--- a/exec/jdbc/pom.xml
+++ b/exec/jdbc/pom.xml
@@ -62,7 +62,6 @@
com.fasterxml.jackson.core
jackson-core
- ${jackson.version}
com.fasterxml.jackson.core
diff --git a/pom.xml b/pom.xml
index 200b44a295f..2c1f5fcf8b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -52,10 +52,10 @@
1.3
- 1.2.1
+ 2.3.2
2.7.1
1.1.3
1.0
@@ -587,6 +587,19 @@
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.1.0
+
+
+ package
+
+ shade
+
+
+
+
@@ -885,6 +898,22 @@
io.netty
netty-all
+
+ javax.servlet
+ servlet-api
+
+
+ org.mortbay.jetty
+ servlet-api-2.5
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+
+
+ log4j
+ log4j
+
@@ -892,6 +921,10 @@
hive-metastore
${hive.version}
+
+ log4j
+ log4j
+
org.apache.hive
hive-serde
@@ -923,6 +956,10 @@
hive-hbase-handler
${hive.version}
+
+ log4j
+ log4j
+
org.slf4j
slf4j-log4j12
@@ -996,6 +1033,11 @@
+
+ io.dropwizard.metrics
+ metrics-core
+ 4.0.2
+
org.codehaus.janino
janino
@@ -1011,6 +1053,11 @@
jackson-databind
${jackson.databind.version}
+
+ com.fasterxml.jackson.core
+ jackson-core
+ ${jackson.version}
+
com.mapr.db
maprdb
@@ -1139,6 +1186,25 @@
1.8
runtime
+
+ org.apache.parquet
+ parquet-column
+ ${parquet.version}
+
+
+ org.apache.hadoop
+ hadoop-client
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+ commons-codec
+ commons-codec
+
+
+
@@ -1875,7 +1941,7 @@
mapr
true
- 1.2.0-mapr-1707
+ 2.1.1-mapr-1710
1.1.1-mapr-1602-m7-5.2.0
2.7.0-mapr-1707