From 122439b1af01ce9e23cbcbcfd95a1c3311561a8c Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 28 Apr 2015 14:39:20 -0700 Subject: [PATCH 1/7] PARQUET-245: Only run tests in Travis CI if build succeeds. Author: Ryan Blue Closes #167 from rdblue/PARQUET-245-fix-travis-ci and squashes the following commits: ccdb0b1 [Ryan Blue] PARQUET-245: Add retry to Travis CI to fix maven downloads. f1bb713 [Ryan Blue] PARQUET-245: Only run tests in Travis CI if build succeeds. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ae33a7ba7d..aa9534935b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,5 +27,5 @@ env: - HADOOP_PROFILE=default - HADOOP_PROFILE=hadoop-2 -install: mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || cat mvn_install.log +install: mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || (cat mvn_install.log && false) script: mvn test -P $HADOOP_PROFILE From d804c73f635489f587cbca8e356ece6bef0ba713 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Thu, 14 May 2015 15:39:39 -0700 Subject: [PATCH 2/7] PARQUET-265: Update POM files for Parquet TLP. Author: Ryan Blue Closes #186 from rdblue/PARQUET-265-update-build-for-graduation and squashes the following commits: 7bd2931 [Ryan Blue] PARQUET-265: Update POM files for Parquet TLP. --- dev/source-release.sh | 10 ++++----- parquet-avro/pom.xml | 4 ++-- parquet-benchmarks/pom.xml | 2 +- parquet-cascading/pom.xml | 4 ++-- parquet-column/pom.xml | 4 ++-- parquet-common/pom.xml | 4 ++-- parquet-encoding/pom.xml | 4 ++-- parquet-generator/pom.xml | 4 ++-- parquet-hadoop-bundle/pom.xml | 4 ++-- parquet-hadoop/pom.xml | 4 ++-- parquet-hive-bundle/pom.xml | 4 ++-- .../parquet-hive-0.10-binding/pom.xml | 2 +- .../parquet-hive-0.12-binding/pom.xml | 2 +- .../parquet-hive-binding-bundle/pom.xml | 2 +- .../parquet-hive-binding-factory/pom.xml | 2 +- .../parquet-hive-binding-interface/pom.xml | 2 +- parquet-hive/parquet-hive-binding/pom.xml | 2 +- .../parquet-hive-storage-handler/pom.xml | 2 +- parquet-hive/pom.xml | 4 ++-- parquet-jackson/pom.xml | 4 ++-- parquet-pig-bundle/pom.xml | 4 ++-- parquet-pig/pom.xml | 4 ++-- parquet-protobuf/pom.xml | 4 ++-- parquet-scala/pom.xml | 4 ++-- parquet-scrooge/pom.xml | 4 ++-- parquet-thrift/pom.xml | 4 ++-- parquet-tools/pom.xml | 4 ++-- pom.xml | 22 +++++++++---------- 28 files changed, 60 insertions(+), 60 deletions(-) diff --git a/dev/source-release.sh b/dev/source-release.sh index a61a6f3237..5a19d8a1bc 100644 --- a/dev/source-release.sh +++ b/dev/source-release.sh @@ -28,7 +28,7 @@ if [ -z "$2" ]; then exit fi -version=$1-incubating +version=$1 rc=$2 if [ -d tmp/ ]; then @@ -36,7 +36,7 @@ if [ -d tmp/ ]; then exit fi -tag=apache-parquet-mr-$version +tag=apache-parquet-$version tagrc=${tag}-rc${rc} echo "Preparing source for $tagrc" @@ -62,19 +62,19 @@ gpg --print-md MD5 $tarball > ${tarball}.md5 shasum $tarball > ${tarball}.sha # check out the parquet RC folder -svn co --depth=empty https://dist.apache.org/repos/dist/dev/incubator/parquet tmp +svn co --depth=empty https://dist.apache.org/repos/dist/dev/parquet tmp # add the release candidate for the tag mkdir -p tmp/$tagrc cp ${tarball}* tmp/$tagrc svn add tmp/$tagrc -echo "svn ci -m 'Apache Parquet MR (Incubating) $version RC${rc}' tmp/$tagrc" +echo "svn ci -m 'Apache Parquet MR $version RC${rc}' tmp/$tagrc" # clean up rm -rf tmp echo "Success! The release candidate is available here:" -echo " https://dist.apache.org/repos/dist/dev/incubator/parquet/$tagrc" +echo " https://dist.apache.org/repos/dist/dev/parquet/$tagrc" echo "" echo "Commit SHA1: $release_hash" diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml index 6d7153105e..9da8733303 100644 --- a/parquet-avro/pom.xml +++ b/parquet-avro/pom.xml @@ -29,8 +29,8 @@ parquet-avro jar - Apache Parquet Avro (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Avro + https://parquet.apache.org 1.7.6 diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml index 25779d4726..c1315d2aec 100644 --- a/parquet-benchmarks/pom.xml +++ b/parquet-benchmarks/pom.xml @@ -29,7 +29,7 @@ parquet-benchmarks jar Apache Parquet Benchmarks - https://github.com/Parquet/parquet-mr + https://parquet.apache.org 1.3.4 diff --git a/parquet-cascading/pom.xml b/parquet-cascading/pom.xml index 99fbc2afbf..aa33f63ce0 100644 --- a/parquet-cascading/pom.xml +++ b/parquet-cascading/pom.xml @@ -29,8 +29,8 @@ parquet-cascading jar - Apache Parquet Cascading (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Cascading + https://parquet.apache.org diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index e50f909a0d..247a6a3f9b 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -29,8 +29,8 @@ parquet-column jar - Apache Parquet Column (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Column + https://parquet.apache.org diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml index cbc3ae61de..dc1d434d55 100644 --- a/parquet-common/pom.xml +++ b/parquet-common/pom.xml @@ -29,8 +29,8 @@ parquet-common jar - Apache Parquet Common (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Common + https://parquet.apache.org diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml index b27c77a052..fd74fac8ea 100644 --- a/parquet-encoding/pom.xml +++ b/parquet-encoding/pom.xml @@ -29,8 +29,8 @@ parquet-encoding jar - Apache Parquet Encodings (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Encodings + https://parquet.apache.org diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml index 52c0cad49d..8a72de2c9e 100644 --- a/parquet-generator/pom.xml +++ b/parquet-generator/pom.xml @@ -29,8 +29,8 @@ parquet-generator jar - Apache Parquet Generator (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Generator + https://parquet.apache.org diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml index a9f4620981..8989bf36c6 100644 --- a/parquet-hadoop-bundle/pom.xml +++ b/parquet-hadoop-bundle/pom.xml @@ -29,8 +29,8 @@ parquet-hadoop-bundle jar - Apache Parquet Hadoop Bundle (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Hadoop Bundle + https://parquet.apache.org diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index f5104014ef..c5e18a40ba 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -29,8 +29,8 @@ parquet-hadoop jar - Apache Parquet Hadoop (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Hadoop + https://parquet.apache.org diff --git a/parquet-hive-bundle/pom.xml b/parquet-hive-bundle/pom.xml index 8cd2407afd..c889ebce2d 100644 --- a/parquet-hive-bundle/pom.xml +++ b/parquet-hive-bundle/pom.xml @@ -29,8 +29,8 @@ parquet-hive-bundle jar - Apache Parquet Hive Bundle (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Hive Bundle + https://parquet.apache.org diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml index 74dcbadb10..42a7dd3227 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml @@ -27,7 +27,7 @@ 4.0.0 parquet-hive-0.10-binding - Apache Parquet Hive 0.10 Binding (Incubating) + Apache Parquet Hive 0.10 Binding jar diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml index f15297287f..17ccd52654 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml @@ -27,7 +27,7 @@ 4.0.0 parquet-hive-0.12-binding - Apache Parquet Hive 0.12 Binding (Incubating) + Apache Parquet Hive 0.12 Binding jar diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml index f77961d57c..39eb353440 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml @@ -29,7 +29,7 @@ 4.0.0 parquet-hive-binding-bundle - Apache Parquet Hive Binding Bundle (Incubating) + Apache Parquet Hive Binding Bundle jar diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml index 9750dbee09..fb464be604 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml @@ -27,7 +27,7 @@ 4.0.0 parquet-hive-binding-factory - Apache Parquet Hive Binding Factory (Incubating) + Apache Parquet Hive Binding Factory jar diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml index 8cf57757bb..fce03d526b 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml @@ -27,7 +27,7 @@ 4.0.0 parquet-hive-binding-interface - Apache Parquet Hive Binding Interface (Incubating) + Apache Parquet Hive Binding Interface jar diff --git a/parquet-hive/parquet-hive-binding/pom.xml b/parquet-hive/parquet-hive-binding/pom.xml index f00c9ac7b3..e3f9785972 100644 --- a/parquet-hive/parquet-hive-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/pom.xml @@ -27,7 +27,7 @@ 4.0.0 parquet-hive-binding - Apache Parquet Hive Binding Parent (Incubating) + Apache Parquet Hive Binding Parent pom diff --git a/parquet-hive/parquet-hive-storage-handler/pom.xml b/parquet-hive/parquet-hive-storage-handler/pom.xml index 8bd874b668..bdad89ad54 100644 --- a/parquet-hive/parquet-hive-storage-handler/pom.xml +++ b/parquet-hive/parquet-hive-storage-handler/pom.xml @@ -27,7 +27,7 @@ 4.0.0 parquet-hive-storage-handler - Parquet Hive Storage Handler + Apache Parquet Hive Storage Handler jar diff --git a/parquet-hive/pom.xml b/parquet-hive/pom.xml index 80ececb501..83c0b3a64b 100644 --- a/parquet-hive/pom.xml +++ b/parquet-hive/pom.xml @@ -29,8 +29,8 @@ parquet-hive pom - Apache Parquet Hive (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Hive + https://parquet.apache.org diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index 9f7d54e88d..056848832e 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -29,8 +29,8 @@ parquet-jackson jar - Apache Parquet Jackson (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Jackson + https://parquet.apache.org diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml index 6c9dc62253..fcb52ebd3c 100644 --- a/parquet-pig-bundle/pom.xml +++ b/parquet-pig-bundle/pom.xml @@ -29,8 +29,8 @@ parquet-pig-bundle jar - Apache Parquet Pig Bundle (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Pig Bundle + https://parquet.apache.org diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml index 4cacb11ff5..69fb4d7d9d 100644 --- a/parquet-pig/pom.xml +++ b/parquet-pig/pom.xml @@ -29,8 +29,8 @@ parquet-pig jar - Apache Parquet Pig (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Pig + https://parquet.apache.org diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml index 00df6759f1..7a26f6c0ab 100644 --- a/parquet-protobuf/pom.xml +++ b/parquet-protobuf/pom.xml @@ -35,8 +35,8 @@ - Apache Parquet Protobuf (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Protobuf + https://parquet.apache.org diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml index 2f1aaf7e39..c22a979102 100644 --- a/parquet-scala/pom.xml +++ b/parquet-scala/pom.xml @@ -29,8 +29,8 @@ parquet-scala_${scala.binary.version} jar - Apache Parquet Scala (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Scala + https://parquet.apache.org diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml index 28268de031..d2a4ccc09d 100644 --- a/parquet-scrooge/pom.xml +++ b/parquet-scrooge/pom.xml @@ -29,8 +29,8 @@ parquet-scrooge_${scala.binary.version} jar - Apache Parquet Scrooge (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Scrooge + https://parquet.apache.org diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml index 153f6fc9b0..00dbb95cd0 100644 --- a/parquet-thrift/pom.xml +++ b/parquet-thrift/pom.xml @@ -29,8 +29,8 @@ parquet-thrift jar - Apache Parquet Thrift (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Thrift + https://parquet.apache.org 4.4 diff --git a/parquet-tools/pom.xml b/parquet-tools/pom.xml index 9f6910cbb5..d2fcdbbe63 100644 --- a/parquet-tools/pom.xml +++ b/parquet-tools/pom.xml @@ -29,8 +29,8 @@ parquet-tools jar - Apache Parquet Tools (Incubating) - https://parquet.incubator.apache.org + Apache Parquet Tools + https://parquet.apache.org provided diff --git a/pom.xml b/pom.xml index 488986eab7..5da6e3a685 100644 --- a/pom.xml +++ b/pom.xml @@ -12,14 +12,14 @@ 1.7.0-incubating-SNAPSHOT pom - Apache Parquet MR (Incubating) - https://parquet.incubator.apache.org + Apache Parquet MR + https://parquet.apache.org Parquet is a columnar storage format that supports nested data. This provides the java implementation. - scm:git:git@github.com:apache/incubator-parquet-mr.git - scm:git:git@github.com:apache/incubator-parquet-mr.git - scm:git:https://git-wip-us.apache.org/repos/asf/incubator-parquet-mr.git + scm:git:git@github.com:apache/parquet-mr.git + scm:git:git@github.com:apache/parquet-mr.git + scm:git:https://git-wip-us.apache.org/repos/asf/parquet-mr.git @@ -37,15 +37,15 @@ Dev Mailing List - dev@parquet.incubator.apache.org - dev-subscribe@parquet.incubator.apache.org - dev-unsubscribe@parquet.incubator.apache.org + dev@parquet.apache.org + dev-subscribe@parquet.apache.org + dev-unsubscribe@parquet.apache.org Commits Mailing List - commits@parquet.incubator.apache.org - commits-subscribe@parquet.incubator.apache.org - commits-unsubscribe@parquet.incubator.apache.org + commits@parquet.apache.org + commits-subscribe@parquet.apache.org + commits-unsubscribe@parquet.apache.org From c1dd8f3b3d51941633bc97abd3b4cb1ba79aee5d Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Thu, 14 May 2015 15:58:22 -0700 Subject: [PATCH 3/7] PARQUET-263: Update changelog. --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 64a19fc682..2b48708d98 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,10 @@ # Parquet # +### Version 1.7.0 ### + +* [PARQUET-23](https://issues.apache.org/jira/browse/PARQUET-23) - Rename to org.apache. + ### Version 1.6.0 ### #### Bug From 57daf13273f7e252da79ad8682494ef82298694d Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Thu, 14 May 2015 16:35:29 -0700 Subject: [PATCH 4/7] PARQUET-263: Update pom to use Apache maven release config. --- pom.xml | 95 +++++++-------------------------------------------------- 1 file changed, 11 insertions(+), 84 deletions(-) diff --git a/pom.xml b/pom.xml index 5da6e3a685..105b45f5e7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.apache apache - 14 + 16 org.apache.parquet @@ -56,32 +56,6 @@ - - - sonatype-nexus-snapshots - Sonatype OSS - https://oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-staging - Nexus Release Repository - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - sonatype-nexus-snapshots - https://oss.sonatype.org/content/repositories/snapshots - - false - - - true - - - - @@ -210,6 +184,16 @@ + + + maven-assembly-plugin + + + source-release-assembly + none + + + maven-enforcer-plugin 1.3.1 @@ -246,16 +230,6 @@ --> - - org.apache.maven.plugins - maven-release-plugin - 2.1 - - forked-path - false - -Psonatype-oss-release - - org.apache.maven.plugins maven-shade-plugin @@ -476,53 +450,6 @@ - - sonatype-oss-release - - - - org.apache.maven.plugins - maven-source-plugin - 2.1.2 - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.7 - - - attach-javadocs - - jar - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.1 - - - sign-artifacts - verify - - sign - - - - - - - hadoop-2 From 32c46643845ea8a705c35d4ec8fc654cc8ff816d Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Fri, 15 May 2015 10:47:26 -0700 Subject: [PATCH 5/7] [maven-release-plugin] prepare release apache-parquet-1.7.0 --- parquet-avro/pom.xml | 2 +- parquet-benchmarks/pom.xml | 2 +- parquet-cascading/pom.xml | 2 +- parquet-column/pom.xml | 2 +- parquet-common/pom.xml | 2 +- parquet-encoding/pom.xml | 2 +- parquet-generator/pom.xml | 2 +- parquet-hadoop-bundle/pom.xml | 2 +- parquet-hadoop/pom.xml | 2 +- parquet-hive-bundle/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-0.10-binding/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-0.12-binding/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-binding-bundle/pom.xml | 2 +- .../parquet-hive-binding/parquet-hive-binding-factory/pom.xml | 2 +- .../parquet-hive-binding-interface/pom.xml | 2 +- parquet-hive/parquet-hive-binding/pom.xml | 2 +- parquet-hive/parquet-hive-storage-handler/pom.xml | 2 +- parquet-hive/pom.xml | 2 +- parquet-jackson/pom.xml | 2 +- parquet-pig-bundle/pom.xml | 2 +- parquet-pig/pom.xml | 2 +- parquet-protobuf/pom.xml | 2 +- parquet-scala/pom.xml | 2 +- parquet-scrooge/pom.xml | 2 +- parquet-thrift/pom.xml | 2 +- parquet-tools/pom.xml | 2 +- pom.xml | 3 ++- 27 files changed, 28 insertions(+), 27 deletions(-) diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml index 9da8733303..4d57425e03 100644 --- a/parquet-avro/pom.xml +++ b/parquet-avro/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml index c1315d2aec..7adfff33f4 100644 --- a/parquet-benchmarks/pom.xml +++ b/parquet-benchmarks/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-cascading/pom.xml b/parquet-cascading/pom.xml index aa33f63ce0..ad31733404 100644 --- a/parquet-cascading/pom.xml +++ b/parquet-cascading/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index 247a6a3f9b..846b8a8846 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml index dc1d434d55..373ec89d3a 100644 --- a/parquet-common/pom.xml +++ b/parquet-common/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml index fd74fac8ea..e579c9827c 100644 --- a/parquet-encoding/pom.xml +++ b/parquet-encoding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml index 8a72de2c9e..c8493308e7 100644 --- a/parquet-generator/pom.xml +++ b/parquet-generator/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml index 8989bf36c6..7b0b011b13 100644 --- a/parquet-hadoop-bundle/pom.xml +++ b/parquet-hadoop-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index c5e18a40ba..bf7819a4f5 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive-bundle/pom.xml b/parquet-hive-bundle/pom.xml index c889ebce2d..ddbc6185a0 100644 --- a/parquet-hive-bundle/pom.xml +++ b/parquet-hive-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml index 42a7dd3227..3c8e807684 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml index 17ccd52654..e54395381f 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml index 39eb353440..774d960e30 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-bundle/pom.xml @@ -23,7 +23,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml index fb464be604..0ae4056c0d 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml index fce03d526b..6cd6152488 100644 --- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml +++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive-binding ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-binding/pom.xml b/parquet-hive/parquet-hive-binding/pom.xml index e3f9785972..42993b43e0 100644 --- a/parquet-hive/parquet-hive-binding/pom.xml +++ b/parquet-hive/parquet-hive-binding/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/parquet-hive-storage-handler/pom.xml b/parquet-hive/parquet-hive-storage-handler/pom.xml index bdad89ad54..7611589fa1 100644 --- a/parquet-hive/parquet-hive-storage-handler/pom.xml +++ b/parquet-hive/parquet-hive-storage-handler/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet-hive ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-hive/pom.xml b/parquet-hive/pom.xml index 83c0b3a64b..47967e21c2 100644 --- a/parquet-hive/pom.xml +++ b/parquet-hive/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml index 056848832e..85619f2618 100644 --- a/parquet-jackson/pom.xml +++ b/parquet-jackson/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml index fcb52ebd3c..e99224fc74 100644 --- a/parquet-pig-bundle/pom.xml +++ b/parquet-pig-bundle/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml index 69fb4d7d9d..d6ef630fcf 100644 --- a/parquet-pig/pom.xml +++ b/parquet-pig/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml index 7a26f6c0ab..3f49f59ee3 100644 --- a/parquet-protobuf/pom.xml +++ b/parquet-protobuf/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml index c22a979102..1e42db72a3 100644 --- a/parquet-scala/pom.xml +++ b/parquet-scala/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml index d2a4ccc09d..4d8bed2738 100644 --- a/parquet-scrooge/pom.xml +++ b/parquet-scrooge/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml index 00dbb95cd0..c59f57be57 100644 --- a/parquet-thrift/pom.xml +++ b/parquet-thrift/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/parquet-tools/pom.xml b/parquet-tools/pom.xml index d2fcdbbe63..e29a4ff7e1 100644 --- a/parquet-tools/pom.xml +++ b/parquet-tools/pom.xml @@ -21,7 +21,7 @@ org.apache.parquet parquet ../pom.xml - 1.7.0-incubating-SNAPSHOT + 1.7.0 4.0.0 diff --git a/pom.xml b/pom.xml index 105b45f5e7..080843b0e9 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.apache.parquet parquet - 1.7.0-incubating-SNAPSHOT + 1.7.0 pom Apache Parquet MR @@ -20,6 +20,7 @@ scm:git:git@github.com:apache/parquet-mr.git scm:git:git@github.com:apache/parquet-mr.git scm:git:https://git-wip-us.apache.org/repos/asf/parquet-mr.git + apache-parquet-1.7.0 From a16bafc485b307509af1599e6f5795f33e84bc80 Mon Sep 17 00:00:00 2001 From: Daniel Weeks Date: Wed, 4 Nov 2015 09:18:28 -0800 Subject: [PATCH 6/7] Initial impl of dictionary filter --- .../column/page/DictionaryPageReadStore.java | 34 +++ .../column/page/DictionaryPageReader.java | 35 +++ parquet-hadoop/pom.xml | 1 - .../filter2/compat/RowGroupFilter.java | 37 ++- .../dictionarylevel/DictionaryFilter.java | 263 ++++++++++++++++++ .../ColumnChunkDictionaryPageReadStore.java | 89 ++++++ .../parquet/hadoop/ParquetFileReader.java | 43 ++- .../parquet/hadoop/ParquetRecordReader.java | 45 ++- .../dictionarylevel/DictionaryFilterTest.java | 171 ++++++++++++ 9 files changed, 704 insertions(+), 14 deletions(-) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReadStore.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReader.java create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkDictionaryPageReadStore.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReadStore.java b/parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReadStore.java new file mode 100644 index 0000000000..e401bff704 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReadStore.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.page; + +import org.apache.parquet.column.ColumnDescriptor; + +/** + * contains all the dictionary readers for all the columns of the corresponding row group + */ +public interface DictionaryPageReadStore { + + /** + * + * @param descriptor the descriptor of the column + * @return the dictionary page reader for that column + */ + DictionaryPageReader getDictionaryPageReader(ColumnDescriptor descriptor); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReader.java b/parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReader.java new file mode 100644 index 0000000000..81cc412194 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/page/DictionaryPageReader.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.page; + +/** + * Reader for a dictionary page from a given column chunk + */ +public interface DictionaryPageReader { + + /** + * @return the dictionary page in that chunk or null if none + */ + DictionaryPage readDictionaryPage(); + + /** + * @return the dictionary size + */ + int getDictionarySize(); +} diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index bf7819a4f5..c872484646 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -77,7 +77,6 @@ com.google.guava guava 11.0 - test org.xerial.snappy diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/RowGroupFilter.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/RowGroupFilter.java index d85a231a31..d55f3cb8f0 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/RowGroupFilter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/compat/RowGroupFilter.java @@ -19,11 +19,16 @@ package org.apache.parquet.filter2.compat; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import com.google.common.base.Supplier; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.io.IOUtils; import org.apache.parquet.filter2.compat.FilterCompat.Filter; import org.apache.parquet.filter2.compat.FilterCompat.NoOpFilter; import org.apache.parquet.filter2.compat.FilterCompat.Visitor; +import org.apache.parquet.filter2.dictionarylevel.DictionaryFilter; import org.apache.parquet.filter2.predicate.FilterPredicate; import org.apache.parquet.filter2.predicate.SchemaCompatibilityValidator; import org.apache.parquet.filter2.statisticslevel.StatisticsFilter; @@ -40,15 +45,34 @@ public class RowGroupFilter implements Visitor> { private final List blocks; private final MessageType schema; + private final List levels; + private final Supplier streamSupplier; + + public enum FilterLevel { + STATISTICS, + DICTIONARY + } public static List filterRowGroups(Filter filter, List blocks, MessageType schema) { checkNotNull(filter, "filter"); return filter.accept(new RowGroupFilter(blocks, schema)); } + public static List filterRowGroups(List levels, Filter filter, List blocks, MessageType schema, Supplier streamSupplier) { + checkNotNull(filter, "filter"); + return filter.accept(new RowGroupFilter(levels, blocks, schema, streamSupplier)); + } + + @Deprecated private RowGroupFilter(List blocks, MessageType schema) { + this(Collections.singletonList(FilterLevel.STATISTICS), blocks, schema, null); + } + + private RowGroupFilter(List levels, List blocks, MessageType schema, Supplier streamSupplier) { this.blocks = checkNotNull(blocks, "blocks"); this.schema = checkNotNull(schema, "schema"); + this.levels = levels; + this.streamSupplier = streamSupplier; } @Override @@ -61,7 +85,18 @@ public List visit(FilterCompat.FilterPredicateCompat filterPredic List filteredBlocks = new ArrayList(); for (BlockMetaData block : blocks) { - if (!StatisticsFilter.canDrop(filterPredicate, block.getColumns())) { + boolean drop = false; + + if(levels.contains(FilterLevel.STATISTICS)) { + drop = StatisticsFilter.canDrop(filterPredicate, block.getColumns()); + } + + if(!drop && levels.contains(FilterLevel.DICTIONARY)) { + drop = DictionaryFilter.canDrop(filterPredicate, block.getColumns(), streamSupplier.get()); + IOUtils.closeStream(streamSupplier.get()); + } + + if(!drop) { filteredBlocks.add(block); } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java new file mode 100644 index 0000000000..51661604a0 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.filter2.dictionarylevel; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.parquet.Log; +import org.apache.parquet.ParquetRuntimeException; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Dictionary; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.filter2.predicate.Operators.*; +import org.apache.parquet.filter2.predicate.UserDefinedPredicate; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ColumnPath; + +import java.io.IOException; +import java.util.*; + +import static org.apache.parquet.Preconditions.checkArgument; +import static org.apache.parquet.Preconditions.checkNotNull; + + +/** + * Applies filters based on the contents of column dictionaries. + */ +public class DictionaryFilter implements FilterPredicate.Visitor { + + private static final Log LOG = Log.getLog(DictionaryFilter.class); + + public static boolean canDrop(FilterPredicate pred, List columns, FSDataInputStream s) { + checkNotNull(pred, "pred"); + checkNotNull(columns, "columns"); + return pred.accept(new DictionaryFilter(columns, s)); + } + + private Configuration conf = null; + private final Map columns = new HashMap(); + private FSDataInputStream f; + + private DictionaryFilter(List columnsList, FSDataInputStream s) { + for (ColumnChunkMetaData chunk : columnsList) { + columns.put(chunk.getPath(), chunk); + } + + this.conf = new Configuration(); + this.f = s; + } + + private ColumnChunkMetaData getColumnChunk(ColumnPath columnPath) { + ColumnChunkMetaData c = columns.get(columnPath); + checkArgument(c != null, "Column " + columnPath.toDotString() + " not found in schema!"); + return c; + } + + private > Set expandDictionary(ColumnChunkMetaData meta) throws IOException, ParquetRuntimeException { + DictionaryPage page = ParquetFileReader.getDictionary(conf, meta, f); + + Dictionary dict = page.getEncoding().initDictionary(new ColumnDescriptor(null, meta.getType(), -1, -1), page); + + Set dictSet = new HashSet(); + + for(int i=0; i> Boolean visit(Eq eq) { + Column filterColumn = eq.getColumn(); + ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath()); + T value = eq.getValue(); + + filterColumn.getColumnPath(); + + try { + Set dictSet = expandDictionary(meta); + return !dictSet.contains(value); + } catch (IOException e) { + LOG.warn("Failed to process dictionary for filter evaluation.", e); + } + + return false; + } + + @Override + public > Boolean visit(NotEq notEq) { + Column filterColumn = notEq.getColumn(); + ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath()); + T value = notEq.getValue(); + + filterColumn.getColumnPath(); + + try { + Set dictSet = expandDictionary(meta); + return dictSet.size() == 1 && dictSet.contains(value); + } catch (IOException e) { + LOG.warn("Failed to process dictionary for filter evaluation.", e); + } + + return false; + } + + @Override + public > Boolean visit(Lt lt) { + Column filterColumn = lt.getColumn(); + ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath()); + T value = lt.getValue(); + + filterColumn.getColumnPath(); + + try { + Set dictSet = expandDictionary(meta); + + for(T entry : dictSet) { + if(value.compareTo(entry) > 0) { + return false; + } + } + + return true; + } catch (IOException e) { + LOG.warn("Failed to process dictionary for filter evaluation.", e); + } + + return false; + } + + @Override + public > Boolean visit(LtEq ltEq) { + Column filterColumn = ltEq.getColumn(); + ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath()); + T value = ltEq.getValue(); + + filterColumn.getColumnPath(); + + try { + Set dictSet = expandDictionary(meta); + + for(T entry : dictSet) { + if(value.compareTo(entry) >= 0) { + return false; + } + } + + return true; + } catch (IOException e) { + LOG.warn("Failed to process dictionary for filter evaluation.", e); + } + + return false; + } + + @Override + public > Boolean visit(Gt gt) { + Column filterColumn = gt.getColumn(); + ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath()); + T value = gt.getValue(); + + filterColumn.getColumnPath(); + + try { + Set dictSet = expandDictionary(meta); + + for(T entry : dictSet) { + if(value.compareTo(entry) < 0) { + return false; + } + } + + return true; + } catch (IOException e) { + LOG.warn("Failed to process dictionary for filter evaluation.", e); + } + + return false; + } + + @Override + public > Boolean visit(GtEq gtEq) { + Column filterColumn = gtEq.getColumn(); + ColumnChunkMetaData meta = getColumnChunk(filterColumn.getColumnPath()); + T value = gtEq.getValue(); + + filterColumn.getColumnPath(); + + try { + Set dictSet = expandDictionary(meta); + + for(T entry : dictSet) { + if(value.compareTo(entry) <= 0) { + return false; + } + } + + return true; + } catch (IOException e) { + LOG.warn("Failed to process dictionary for filter evaluation.", e); + } + + return false; + } + + @Override + public Boolean visit(And and) { + return and.getLeft().accept(this) || and.getRight().accept(this); + } + + @Override + public Boolean visit(Or or) { + return or.getLeft().accept(this) && or.getRight().accept(this); + } + + @Override + public Boolean visit(Not not) { + throw new IllegalArgumentException( + "This predicate contains a not! Did you forget to run this predicate through LogicalInverseRewriter? " + not); + } + + @Override + public , U extends UserDefinedPredicate> Boolean visit(UserDefined udp) { + throw new UnsupportedOperationException("UDP not supported with dictionary evaluation."); + } + + @Override + public , U extends UserDefinedPredicate> Boolean visit(LogicalNotUserDefined udp) { + throw new UnsupportedOperationException("UDP not supported with dictionary evaluation."); + } + +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkDictionaryPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkDictionaryPageReadStore.java new file mode 100644 index 0000000000..4cd34d0688 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkDictionaryPageReadStore.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.parquet.Ints; +import org.apache.parquet.Log; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.DictionaryPageReadStore; +import org.apache.parquet.column.page.DictionaryPageReader; +import org.apache.parquet.hadoop.CodecFactory.BytesDecompressor; + +class ColumnChunkDictionaryPageReadStore implements DictionaryPageReadStore { + /** + * DictionaryPageReader for a single column chunk. A column chunk contains + * several pages, first of which could be a dictionary page. + * + * This implementation is provided with compressed dictionary page + */ + static final class ColumnChunkDictionaryPageReader implements DictionaryPageReader { + + private final BytesDecompressor decompressor; + private final DictionaryPage compressedDictionaryPage; + + ColumnChunkDictionaryPageReader(BytesDecompressor decompressor, DictionaryPage compressedDictionaryPage) { + this.decompressor = decompressor; + this.compressedDictionaryPage = compressedDictionaryPage; + } + + @Override + public int getDictionarySize() { + return compressedDictionaryPage.getDictionarySize(); + } + + @Override + public DictionaryPage readDictionaryPage() { + if (compressedDictionaryPage == null) { + return null; + } + try { + return new DictionaryPage( + decompressor.decompress(compressedDictionaryPage.getBytes(), compressedDictionaryPage.getUncompressedSize()), + compressedDictionaryPage.getDictionarySize(), + compressedDictionaryPage.getEncoding()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private final Map readers = new HashMap(); + + public ColumnChunkDictionaryPageReadStore() { + } + + @Override + public DictionaryPageReader getDictionaryPageReader(ColumnDescriptor column) { + if (!readers.containsKey(column)) { + throw new IllegalArgumentException(column + " is not in the store: " + readers.keySet()); + } + return readers.get(column); + } + + void addColumn(ColumnDescriptor column, ColumnChunkDictionaryPageReader reader) { + if (readers.put(column, reader) != null) { + throw new RuntimeException(column + " was added twice"); + } + } +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 49e0833c96..2a047329c3 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -59,6 +59,8 @@ import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.page.DictionaryPageReadStore; +import org.apache.parquet.column.page.DictionaryPageReader; import org.apache.parquet.column.page.DictionaryPage; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.hadoop.metadata.ColumnPath; @@ -71,6 +73,7 @@ import org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter; import org.apache.parquet.hadoop.CodecFactory.BytesDecompressor; import org.apache.parquet.hadoop.ColumnChunkPageReadStore.ColumnChunkPageReader; +import org.apache.parquet.hadoop.ColumnChunkDictionaryPageReadStore.ColumnChunkDictionaryPageReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; @@ -435,6 +438,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File private final CodecFactory codecFactory; private final List blocks; + private final Configuration configuration; private final FSDataInputStream f; private final Path filePath; private int currentBlock = 0; @@ -448,6 +452,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File * @throws IOException if the file can not be opened */ public ParquetFileReader(Configuration configuration, Path filePath, List blocks, List columns) throws IOException { + this.configuration = configuration; this.filePath = filePath; FileSystem fs = filePath.getFileSystem(configuration); this.f = fs.open(filePath); @@ -500,7 +505,35 @@ public PageReadStore readNextRowGroup() throws IOException { return columnChunkPageReadStore; } + public static DictionaryPage getDictionary(Configuration conf, ColumnChunkMetaData meta, FSDataInputStream fin) throws IOException { + if(fin.getPos() != meta.getStartingPos()) { + fin.seek(meta.getStartingPos()); + } + + PageHeader pageHeader = Util.readPageHeader(fin); + + return getDictionary(conf, pageHeader, meta, fin); + } + + public static DictionaryPage getDictionary(Configuration conf, PageHeader pageHeader, ColumnChunkMetaData meta, FSDataInputStream fin) throws IOException { + DictionaryPageHeader dictHeader = pageHeader.getDictionary_page_header(); + + int uncompressedPageSize = pageHeader.getUncompressed_page_size(); + int compressedPageSize = pageHeader.getCompressed_page_size(); + + byte [] dictPageBytes = new byte[compressedPageSize]; + + fin.read(dictPageBytes); + BytesInput bin = BytesInput.from(dictPageBytes); + + DictionaryPage compressedPage = new DictionaryPage(bin, uncompressedPageSize, dictHeader.getNum_values(), converter.getEncoding(dictHeader.getEncoding())); + + BytesDecompressor decompressor = new CodecFactory(conf).getDecompressor(meta.getCodec()); + DictionaryPageReader dictionaryPageReader = new ColumnChunkDictionaryPageReader(decompressor, compressedPage); + + return dictionaryPageReader.readDictionaryPage(); + } @Override public void close() throws IOException { @@ -552,14 +585,8 @@ public ColumnChunkPageReader readAllPages() throws IOException { if (dictionaryPage != null) { throw new ParquetDecodingException("more than one dictionary page in column " + descriptor.col); } - DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header(); - dictionaryPage = - new DictionaryPage( - this.readAsBytesInput(compressedPageSize), - uncompressedPageSize, - dicHeader.getNum_values(), - converter.getEncoding(dicHeader.getEncoding()) - ); + + dictionaryPage = getDictionary(configuration, pageHeader, descriptor.metadata, f); break; case DATA_PAGE: DataPageHeader dataHeaderV1 = pageHeader.getData_page_header(); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetRecordReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetRecordReader.java index 7b83770a66..744d9abd35 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetRecordReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetRecordReader.java @@ -19,6 +19,7 @@ package org.apache.parquet.hadoop; import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; +import static org.apache.parquet.filter2.compat.RowGroupFilter.FilterLevel.*; import static org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER; import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; @@ -32,8 +33,13 @@ import java.util.Map; import java.util.Set; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; @@ -45,6 +51,7 @@ import org.apache.parquet.filter.UnboundRecordFilter; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.filter2.compat.FilterCompat.Filter; +import org.apache.parquet.filter2.compat.RowGroupFilter.FilterLevel; import org.apache.parquet.hadoop.api.ReadSupport; import org.apache.parquet.hadoop.metadata.BlockMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; @@ -52,6 +59,8 @@ import org.apache.parquet.hadoop.util.counters.BenchmarkCounter; import org.apache.parquet.schema.MessageType; + + /** * Reads the records from a block of a Parquet file * @@ -147,7 +156,8 @@ public void initialize(InputSplit inputSplit, Configuration configuration, Repor } private void initializeInternalReader(ParquetInputSplit split, Configuration configuration) throws IOException { - Path path = split.getPath(); + final Path path = split.getPath(); + final Configuration conf = configuration; long[] rowGroupOffsets = split.getRowGroupOffsets(); List filteredBlocks; ParquetMetadata footer; @@ -157,7 +167,34 @@ private void initializeInternalReader(ParquetInputSplit split, Configuration con footer = readFooter(configuration, path, range(split.getStart(), split.getEnd())); MessageType fileSchema = footer.getFileMetaData().getSchema(); Filter filter = getFilter(configuration); - filteredBlocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); + + List levels = new ArrayList(); + + if(configuration.getBoolean("parquet.filter.statistics.enabled", true)) { + levels.add(STATISTICS); + } + + //This is for lazy evaluation so that if stats level can provide the + //result, we don't need to open a new file stream. + Supplier streamSupplier = null; + if(configuration.getBoolean("parquet.filter.dictionary.enabled", false)) { + levels.add(DICTIONARY); + + streamSupplier = Suppliers.memoize(new Supplier() { + @Override + public FSDataInputStream get() { + try { + FileSystem fs = path.getFileSystem(conf); + return fs.open(path); + } catch (IOException e) { + LOG.warn("Failed to open path for dictionary predicate evaluation: " + path, e); + } + return null; + } + }); + } + + filteredBlocks = filterRowGroups(levels, filter, footer.getBlocks(), fileSchema, streamSupplier); } else { // otherwise we find the row groups that were selected on the client footer = readFooter(configuration, path, NO_FILTER); @@ -189,8 +226,8 @@ private void initializeInternalReader(ParquetInputSplit split, Configuration con } MessageType fileSchema = footer.getFileMetaData().getSchema(); Map fileMetaData = footer.getFileMetaData().getKeyValueMetaData(); - internalReader.initialize( - fileSchema, fileMetaData, path, filteredBlocks, configuration); + internalReader.initialize( + fileSchema, fileMetaData, path, filteredBlocks, configuration); } /** diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java new file mode 100644 index 0000000000..ca7577a104 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.filter2.dictionarylevel; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.filter2.predicate.FilterPredicate; +import org.apache.parquet.filter2.predicate.Operators.BinaryColumn; +import org.apache.parquet.filter2.predicate.Operators.IntColumn; +import org.apache.parquet.format.converter.ParquetMetadataConverter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.GroupWriteSupport; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.junit.*; + +import java.io.IOException; +import java.util.List; + +import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0; +import static org.apache.parquet.filter2.dictionarylevel.DictionaryFilter.canDrop; +import static org.apache.parquet.filter2.predicate.FilterApi.*; +import static org.apache.parquet.hadoop.metadata.CompressionCodecName.UNCOMPRESSED; +import static org.apache.parquet.schema.MessageTypeParser.parseMessageType; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Created by dweeks on 10/19/15. + */ +public class DictionaryFilterTest { + + private static final int nElements = 1000; + private static final Configuration conf = new Configuration(); + private static Path file = new Path("target/test/TestDictionaryFilter/testParquetFile"); + private static final MessageType schema = parseMessageType( + "message test { " + + "required binary binary_field; " + + "required int32 int32_field; " + + "required int64 int64_field; " + + "required double double_field; " + + "required float float_field; " + + "} "); + + private static final int ENGLISH_CHARACTER_NUMBER = 26; + private static final int[] intValues = new int[] {-100, 302, 3333333, 7654321, 1234567, -2000, -77775, 0, 75, 22223, + 77, 22221, -444443, 205, 12, 44444, 889, 66665, -777889, -7, + 52, 33, -257, 1111, 775, 26}; + private static final long[] longValues = new long[] {-100L, 302L, 3333333L, 7654321L, 1234567L, -2000L, -77775L, 0L, + 75L, 22223L, 77L, 22221L, -444443L, 205L, 12L, 44444L, 889L, 66665L, + -777889L, -7L, 52L, 33L, -257L, 1111L, 775L, 26L}; + + private static void writeData(SimpleGroupFactory f, ParquetWriter writer) throws IOException { + for (int i = 0; i < nElements; i++) { + int index = i % ENGLISH_CHARACTER_NUMBER; + char c = (char) ((index) + 'a'); + String b = String.valueOf(c); + + Group group = f.newGroup() + .append("binary_field", b) + .append("int32_field", intValues[index]) + .append("int64_field", longValues[index]) + .append("double_field", intValues[index] * 1.0) + .append("float_field", ((float) (intValues[index] * 2.0))); + + writer.write(group); + } + writer.close(); + } + + @BeforeClass + public static void prepareFile() throws IOException { + cleanup(); + + boolean dictionaryEnabled = true; + boolean validating = false; + GroupWriteSupport.setSchema(schema, conf); + SimpleGroupFactory f = new SimpleGroupFactory(schema); + ParquetWriter writer = new ParquetWriter( + file, + new GroupWriteSupport(), + UNCOMPRESSED, 1024*1024, 1024, 1024*1024, + dictionaryEnabled, validating, PARQUET_1_0, conf); + writeData(f, writer); + } + + @AfterClass + public static void cleanup() throws IOException { + FileSystem fs = file.getFileSystem(conf); + if (fs.exists(file)) { + fs.delete(file, true); + } + } + + + List ccmd; + FSDataInputStream stream; + + @Before + public void setUp() throws Exception { + FileSystem fs = FileSystem.getLocal(conf); + ParquetMetadata meta = ParquetFileReader.readFooter(conf, fs.getFileStatus(file), ParquetMetadataConverter.NO_FILTER); + + ccmd = meta.getBlocks().get(0).getColumns(); + stream = fs.open(file); + } + + @After + public void tearDown() throws Exception { + stream.close(); + } + + @Test + public void testCanDropEq() throws Exception { + BinaryColumn b = binaryColumn("binary_field"); + FilterPredicate pred = eq(b, Binary.fromString("c")); + + assertFalse(canDrop(pred, ccmd, stream)); + } + + @Test + public void testCanDropLt() throws Exception { + IntColumn i32 = intColumn("int32_field"); + FilterPredicate predDrop = lt(i32, -777889); + assertTrue(canDrop(predDrop, ccmd, stream)); + + FilterPredicate predKeep = lt(i32, -1); + assertFalse(canDrop(predKeep, ccmd, stream)); + } + + @Test + public void testCanDropLtEq() throws Exception { + IntColumn i32 = intColumn("int32_field"); + FilterPredicate pred = ltEq(i32, -777890); + + assertTrue(canDrop(pred, ccmd, stream)); + } + + @Test + public void testCanDropAnd() throws Exception { + IntColumn i32 = intColumn("int32_field"); + FilterPredicate pred = and(lt(i32, -777889), gt(i32, 3333332)); + + assertTrue(canDrop(pred, ccmd, stream)); + } + +} \ No newline at end of file From 3c52142f214992dc379e3118992fd14e70c7f054 Mon Sep 17 00:00:00 2001 From: Daniel Weeks Date: Wed, 27 Jan 2016 09:45:21 -0800 Subject: [PATCH 7/7] Fix dictionary read and fully read buffer --- .../filter2/dictionarylevel/DictionaryFilter.java | 4 ++-- .../apache/parquet/hadoop/ParquetFileReader.java | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java index 51661604a0..6cfa42d8b7 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java @@ -72,13 +72,13 @@ private ColumnChunkMetaData getColumnChunk(ColumnPath columnPath) { } private > Set expandDictionary(ColumnChunkMetaData meta) throws IOException, ParquetRuntimeException { - DictionaryPage page = ParquetFileReader.getDictionary(conf, meta, f); + DictionaryPage page = ParquetFileReader.readDictionary(conf, meta, f); Dictionary dict = page.getEncoding().initDictionary(new ColumnDescriptor(null, meta.getType(), -1, -1), page); Set dictSet = new HashSet(); - for(int i=0; i blocks, List columns) throws IOException { + public ParquetFileReader(Configuration configuration, FileMetaData fileMetaData, Path filePath, List blocks, List columns) throws IOException { this.configuration = configuration; this.filePath = filePath; this.fileMetaData = fileMetaData; @@ -529,17 +528,17 @@ public PageReadStore readNextRowGroup() throws IOException { return columnChunkPageReadStore; } - public static DictionaryPage getDictionary(Configuration conf, ColumnChunkMetaData meta, FSDataInputStream fin) throws IOException { + public static DictionaryPage readDictionary(Configuration conf, ColumnChunkMetaData meta, FSDataInputStream fin) throws IOException { if(fin.getPos() != meta.getStartingPos()) { fin.seek(meta.getStartingPos()); } PageHeader pageHeader = Util.readPageHeader(fin); - return getDictionary(conf, pageHeader, meta, fin); + return readDictionary(conf, pageHeader, meta, fin); } - public static DictionaryPage getDictionary(Configuration conf, PageHeader pageHeader, ColumnChunkMetaData meta, FSDataInputStream fin) throws IOException { + public static DictionaryPage readDictionary(Configuration conf, PageHeader pageHeader, ColumnChunkMetaData meta, FSDataInputStream fin) throws IOException { DictionaryPageHeader dictHeader = pageHeader.getDictionary_page_header(); int uncompressedPageSize = pageHeader.getUncompressed_page_size(); @@ -547,7 +546,7 @@ public static DictionaryPage getDictionary(Configuration conf, PageHeader pageHe byte [] dictPageBytes = new byte[compressedPageSize]; - fin.read(dictPageBytes); + fin.readFully(dictPageBytes); BytesInput bin = BytesInput.from(dictPageBytes); @@ -610,7 +609,7 @@ public ColumnChunkPageReader readAllPages() throws IOException { throw new ParquetDecodingException("more than one dictionary page in column " + descriptor.col); } - dictionaryPage = getDictionary(configuration, pageHeader, descriptor.metadata, f); + dictionaryPage = readDictionary(configuration, pageHeader, descriptor.metadata, f); break; case DATA_PAGE: DataPageHeader dataHeaderV1 = pageHeader.getData_page_header();