From 7334be22a74caae77a2eeaa1cfc83d3a3247589e Mon Sep 17 00:00:00 2001 From: Michal Gorecki Date: Fri, 5 Feb 2016 16:42:13 -0500 Subject: [PATCH 1/2] PARQUET-511: Integer overflow when counting values in column. This commit fixes an issue when the number of entries in a column page is larger than the size of an integer. No exception is thrown directly, but the def level is set incorrectly, leading to a null value being returned during read. --- .../java/org/apache/parquet/column/impl/ColumnReaderImpl.java | 4 ++-- .../org/apache/parquet/hadoop/ColumnChunkPageReadStore.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java index 8c2a4bf2eb..7f1f44ac93 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java @@ -150,7 +150,7 @@ public double getDouble() { private int dictionaryId; private long endOfPageValueCount; - private int readValues = 0; + private long readValues = 0; private int pageValueCount = 0; private final PrimitiveConverter converter; @@ -352,7 +352,7 @@ public ColumnReaderImpl(ColumnDescriptor path, PageReader pageReader, PrimitiveC this.dictionary = null; } this.totalValueCount = pageReader.getTotalValueCount(); - if (totalValueCount == 0) { + if (totalValueCount <= 0) { throw new ParquetDecodingException("totalValueCount == 0"); } consume(); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index af067474f8..4787b241fb 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -63,7 +63,7 @@ static final class ColumnChunkPageReader implements PageReader { this.decompressor = decompressor; this.compressedPages = new LinkedList(compressedPages); this.compressedDictionaryPage = compressedDictionaryPage; - int count = 0; + long count = 0; for (DataPage p : compressedPages) { count += p.getValueCount(); } From d224815abdcccd049e9c6acc907aadcaac8c0979 Mon Sep 17 00:00:00 2001 From: Michal Gorecki Date: Thu, 10 Mar 2016 10:51:18 -0500 Subject: [PATCH 2/2] enhancing exception message --- .../java/org/apache/parquet/column/impl/ColumnReaderImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java index 7f1f44ac93..3fc327e3d9 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java @@ -353,7 +353,7 @@ public ColumnReaderImpl(ColumnDescriptor path, PageReader pageReader, PrimitiveC } this.totalValueCount = pageReader.getTotalValueCount(); if (totalValueCount <= 0) { - throw new ParquetDecodingException("totalValueCount == 0"); + throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0"); } consume(); }