diff --git a/core/pom.xml b/core/pom.xml index 4c7e3a36620a9..cbe575b881689 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -211,10 +211,6 @@ io.netty netty-all - - io.netty - netty - com.clearspring.analytics stream @@ -261,12 +257,6 @@ org.seleniumhq.selenium selenium-java - - - com.google.guava - guava - - test diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index 2c24366cc3a15..0e2b94553bb72 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -27,7 +27,7 @@ commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar -commons-compiler-2.7.6.jar +commons-compiler-2.7.8.jar commons-compress-1.4.1.jar commons-configuration-1.6.jar commons-dbcp-1.4.jar @@ -78,6 +78,12 @@ hadoop-yarn-client-2.2.0.jar hadoop-yarn-common-2.2.0.jar hadoop-yarn-server-common-2.2.0.jar hadoop-yarn-server-web-proxy-2.2.0.jar +hive-beeline-1.2.1.spark.jar +hive-cli-1.2.1.spark.jar +hive-exec-1.2.1.spark.jar +hive-jdbc-1.2.1.spark.jar +hive-metastore-1.2.1.spark.jar +hive-service-1.2.1.spark.jar httpclient-4.3.2.jar httpcore-4.3.2.jar ivy-2.4.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index e9cb0d8f3eac2..71efbdba68d4d 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -29,7 +29,7 @@ commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar -commons-compiler-2.7.6.jar +commons-compiler-2.7.8.jar commons-compress-1.4.1.jar commons-configuration-1.6.jar commons-dbcp-1.4.jar @@ -73,6 +73,12 @@ hadoop-yarn-client-2.3.0.jar hadoop-yarn-common-2.3.0.jar hadoop-yarn-server-common-2.3.0.jar hadoop-yarn-server-web-proxy-2.3.0.jar +hive-beeline-1.2.1.spark.jar +hive-cli-1.2.1.spark.jar +hive-exec-1.2.1.spark.jar +hive-jdbc-1.2.1.spark.jar +hive-metastore-1.2.1.spark.jar +hive-service-1.2.1.spark.jar httpclient-4.3.2.jar httpcore-4.3.2.jar ivy-2.4.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index d8d1840da5531..800d78f1de008 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -29,7 +29,7 @@ commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar -commons-compiler-2.7.6.jar +commons-compiler-2.7.8.jar commons-compress-1.4.1.jar commons-configuration-1.6.jar commons-dbcp-1.4.jar @@ -73,6 +73,12 @@ hadoop-yarn-client-2.4.0.jar hadoop-yarn-common-2.4.0.jar hadoop-yarn-server-common-2.4.0.jar hadoop-yarn-server-web-proxy-2.4.0.jar +hive-beeline-1.2.1.spark.jar +hive-cli-1.2.1.spark.jar +hive-exec-1.2.1.spark.jar +hive-jdbc-1.2.1.spark.jar +hive-metastore-1.2.1.spark.jar +hive-service-1.2.1.spark.jar httpclient-4.3.2.jar httpcore-4.3.2.jar ivy-2.4.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 8beede1e38d28..f41f92aea1ab5 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -33,7 +33,7 @@ commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar -commons-compiler-2.7.6.jar +commons-compiler-2.7.8.jar commons-compress-1.4.1.jar commons-configuration-1.6.jar commons-dbcp-1.4.jar @@ -78,6 +78,12 @@ hadoop-yarn-client-2.6.0.jar hadoop-yarn-common-2.6.0.jar hadoop-yarn-server-common-2.6.0.jar hadoop-yarn-server-web-proxy-2.6.0.jar +hive-beeline-1.2.1.spark.jar +hive-cli-1.2.1.spark.jar +hive-exec-1.2.1.spark.jar +hive-jdbc-1.2.1.spark.jar +hive-metastore-1.2.1.spark.jar +hive-service-1.2.1.spark.jar htrace-core-3.0.4.jar httpclient-4.3.2.jar httpcore-4.3.2.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index a9d814f944872..6b074f67b3a44 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -33,7 +33,7 @@ commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar commons-codec-1.10.jar commons-collections-3.2.2.jar -commons-compiler-2.7.6.jar +commons-compiler-2.7.8.jar commons-compress-1.4.1.jar commons-configuration-1.6.jar commons-dbcp-1.4.jar @@ -78,6 +78,12 @@ hadoop-yarn-client-2.7.0.jar hadoop-yarn-common-2.7.0.jar hadoop-yarn-server-common-2.7.0.jar hadoop-yarn-server-web-proxy-2.7.0.jar +hive-beeline-1.2.1.spark.jar +hive-cli-1.2.1.spark.jar +hive-exec-1.2.1.spark.jar +hive-jdbc-1.2.1.spark.jar +hive-metastore-1.2.1.spark.jar +hive-service-1.2.1.spark.jar htrace-core-3.1.0-incubating.jar httpclient-4.3.2.jar httpcore-4.3.2.jar diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 924b55287c2dc..3a37b72ec44f7 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -81,7 +81,7 @@ for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly \ | grep "Building Spark Project Assembly" -A 5 \ | tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1 | rev | sort \ - | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE + | grep -v ^spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE done if [[ $@ == **replace-manifest** ]]; then diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index ac15b93c048da..e782b164f200f 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -96,7 +96,6 @@ org.apache.avro avro-mapred - ${avro.mapred.classifier} provided diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index 62818f5e8f434..2c27cbff73053 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -88,7 +88,6 @@ org.apache.avro avro-mapred - ${avro.mapred.classifier} provided diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index d1c38c7ca5d69..43772d1641221 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -98,7 +98,6 @@ org.apache.avro avro-mapred - ${avro.mapred.classifier} provided diff --git a/pom.xml b/pom.xml index 984b2859efbec..be0569799dd99 100644 --- a/pom.xml +++ b/pom.xml @@ -144,7 +144,6 @@ 2.0.8 3.1.2 1.7.7 - hadoop2 0.7.1 1.6.1 @@ -164,17 +163,14 @@ 2.5.3 1.1.2.4 1.1.2 - 1.2.0-incubating 1.10 2.6 3.3.2 - 3.2.10 2.7.8 1.9 2.9 - 3.5.2 1.3.9 0.9.2 4.5.2-1 @@ -276,11 +272,7 @@ org.ow2.asm - asm - - - org.ow2.asm - asm-commons + * @@ -291,11 +283,7 @@ org.ow2.asm - asm - - - org.ow2.asm - asm-commons + * @@ -416,16 +404,6 @@ selenium-java 2.45.0 test - - - com.google.guava - guava - - - io.netty - netty - - @@ -529,11 +507,13 @@ netty-all 4.0.29.Final + - io.netty - netty - 3.8.0.Final - + io.netty + netty + 3.8.0.Final + org.apache.derby derby @@ -564,6 +544,11 @@ metrics-graphite ${codahale.metrics.version} + + com.fasterxml.jackson.core + jackson-core + ${fasterxml.jackson.version} + com.fasterxml.jackson.core jackson-databind @@ -574,18 +559,10 @@ jackson-annotations ${fasterxml.jackson.version} - com.fasterxml.jackson.module jackson-module-scala_${scala.binary.version} ${fasterxml.jackson.version} - - - com.google.guava - guava - - com.sun.jersey @@ -685,24 +662,23 @@ test - guava - com.google.guava - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore + commons-logging + commons-logging + - commons-logging - httpclient + org.ow2.asm + * - commons-logging - commons-logging + org.ow2.asm + asm-commons @@ -760,13 +736,9 @@ asm asm - - org.codehaus.jackson - jackson-mapper-asl - org.ow2.asm - asm + * org.jboss.netty @@ -806,10 +778,6 @@ ${avro.version} ${hadoop.deps.scope} - - io.netty - netty - org.mortbay.jetty jetty @@ -840,13 +808,9 @@ org.apache.avro avro-mapred ${avro.version} - ${avro.mapred.classifier} + hadoop2 ${hive.deps.scope} - - io.netty - netty - org.mortbay.jetty jetty @@ -894,7 +858,7 @@ org.ow2.asm - asm + * org.jboss.netty @@ -918,7 +882,7 @@ org.ow2.asm - asm + * org.jboss.netty @@ -947,7 +911,7 @@ org.ow2.asm - asm + * org.jboss.netty @@ -1087,18 +1051,6 @@ org.apache.thrift libthrift - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1143,18 +1095,6 @@ org.apache.thrift libthrift - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1175,22 +1115,6 @@ org.apache.ant ant - - org.apache.zookeeper - zookeeper - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1201,9 +1125,6 @@ ${hive.group} hive-exec - ${hive.version} ${hive.deps.scope} @@ -1233,67 +1154,34 @@ ant ant + - org.apache.ant - ant - - - com.esotericsoftware.kryo - kryo + org.apache.calcite + calcite-core - commons-codec - commons-codec + org.apache.ant + ant - commons-httpclient - commons-httpclient + org.ow2.asm + * org.apache.avro avro-mapred - - - org.apache.calcite - calcite-core - + org.apache.curator apache-curator - - org.apache.curator - curator-client - - - org.apache.curator - curator-framework - org.apache.thrift libthrift - - org.apache.thrift - libfb303 - - - org.apache.zookeeper - zookeeper - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1333,14 +1221,6 @@ ${hive.group} hive-shims - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - org.apache.curator curator-framework @@ -1349,26 +1229,6 @@ org.apache.thrift libthrift - - org.apache.thrift - libfb303 - - - org.apache.zookeeper - zookeeper - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1394,10 +1254,6 @@ ${hive.group} hive-shims - - org.apache.thrift - libfb303 - org.apache.thrift libthrift @@ -1406,18 +1262,6 @@ org.mortbay.jetty servlet-api - - com.google.guava - guava - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - @@ -1435,14 +1279,6 @@ ${hive.group} hive-shims - - commons-codec - commons-codec - - - com.google.code.findbugs - jsr305 - org.apache.avro avro @@ -1451,22 +1287,6 @@ org.apache.thrift libthrift - - org.apache.thrift - libfb303 - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1500,22 +1320,6 @@ ${hive.group} hive-shims - - commons-codec - commons-codec - - - org.apache.curator - curator-framework - - - org.apache.curator - curator-recipes - - - org.apache.thrift - libfb303 - org.apache.thrift libthrift @@ -1540,38 +1344,14 @@ ${hive.version} ${hive.deps.scope} - - com.google.guava - guava - org.apache.hadoop hadoop-yarn-server-resourcemanager - - org.apache.curator - curator-framework - org.apache.thrift libthrift - - org.apache.zookeeper - zookeeper - - - org.slf4j - slf4j-api - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - commons-logging commons-logging @@ -1612,10 +1392,6 @@ ${flume.version} ${flume.deps.scope} - - io.netty - netty - org.apache.flume flume-ng-auth @@ -1636,10 +1412,6 @@ ${flume.version} ${flume.deps.scope} - - io.netty - netty - org.apache.thrift libthrift @@ -1649,35 +1421,9 @@ org.apache.calcite calcite-core - ${calcite.version} + + 1.2.0-incubating - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - org.codehaus.janino - janino - - org.hsqldb hsqldb @@ -1689,27 +1435,13 @@ - org.apache.calcite - calcite-avatica - ${calcite.version} - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - + org.codehaus.janino + janino + ${janino.version} org.codehaus.janino - janino + commons-compiler ${janino.version} @@ -1717,53 +1449,15 @@ joda-time ${joda.version} - - org.jodd - jodd-core - ${jodd.version} - - - org.datanucleus - datanucleus-core - ${datanucleus-core.version} - org.apache.thrift libthrift ${libthrift.version} - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - org.slf4j - slf4j-api - - org.apache.thrift libfb303 ${libthrift.version} - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - org.slf4j - slf4j-api - - org.antlr @@ -1805,6 +1499,18 @@ --> org.jboss.netty org.codehaus.groovy + javax.servlet:servlet-api + + org.hsqldb:hsqldb + + org.ow2.asm + + org.mockito:mockito-all true diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 61504becf1f38..789ec323a167e 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -62,109 +62,41 @@ org.apache.spark spark-test-tags_${scala.binary.version} - ${hive.group} hive-cli - ${hive.group} hive-exec - ${hive.group} hive-metastore - - org.apache.avro avro - - - org.apache.avro - avro-mapred - ${avro.mapred.classifier} - - - commons-httpclient - commons-httpclient - - - org.apache.calcite - calcite-avatica - + org.apache.calcite calcite-core - - org.apache.httpcomponents - httpclient - - - org.codehaus.jackson - jackson-mapper-asl - - - - commons-codec - commons-codec - + joda-time joda-time - - org.jodd - jodd-core - - - com.google.code.findbugs - jsr305 - - - org.datanucleus - datanucleus-core - org.apache.thrift libthrift - - org.apache.thrift - libfb303 - org.scalacheck scalacheck_${scala.binary.version} diff --git a/tools/pom.xml b/tools/pom.xml index 9bb20e1381067..a659fe4e47de2 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -71,6 +71,15 @@ org.apache.maven.plugins maven-source-plugin + + + org.apache.maven.plugins + maven-enforcer-plugin + + true + +