From 4b41d0eb6369a63f0f25d27ff517d01aaed936d0 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Fri, 9 Jan 2015 19:01:12 +0530 Subject: [PATCH] PARQUET-161: 1. Statistics should be written for column chunks that are not null 2. Statistics object should be marked non empty in case null values are written 3. Keep a boolean in the object to identify presence of non-null values --- .../column/statistics/BinaryStatistics.java | 40 +++++++++++++++---- .../column/statistics/BooleanStatistics.java | 29 ++++++++++++-- .../column/statistics/DoubleStatistics.java | 33 ++++++++++++--- .../column/statistics/FloatStatistics.java | 29 ++++++++++++-- .../column/statistics/IntStatistics.java | 29 ++++++++++++-- .../column/statistics/LongStatistics.java | 31 ++++++++++++-- .../parquet/column/statistics/Statistics.java | 23 ++++++++++- .../converter/ParquetMetadataConverter.java | 10 +++-- 8 files changed, 193 insertions(+), 31 deletions(-) diff --git a/parquet-column/src/main/java/parquet/column/statistics/BinaryStatistics.java b/parquet-column/src/main/java/parquet/column/statistics/BinaryStatistics.java index f125b2f0ed..d1025b9dd0 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/BinaryStatistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/BinaryStatistics.java @@ -24,7 +24,7 @@ public class BinaryStatistics extends Statistics { @Override public void updateStats(Binary value) { - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(value, value); } else { updateStats(value, value); @@ -34,17 +34,18 @@ public void updateStats(Binary value) { @Override public void mergeStatisticsMinMax(Statistics stats) { BinaryStatistics binaryStats = (BinaryStatistics)stats; - if (this.isEmpty()) { - initializeStats(binaryStats.getMin(), binaryStats.getMax()); - } else { - updateStats(binaryStats.getMin(), binaryStats.getMax()); - } + if (this.isEmpty() || !this.hasValidValue()) { + initializeStats(binaryStats.getMin(), binaryStats.getMax()); + } else { + updateStats(binaryStats.getMin(), binaryStats.getMax()); + } } @Override public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { max = Binary.fromByteArray(maxBytes); min = Binary.fromByteArray(minBytes); + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -60,8 +61,10 @@ public byte[] getMinBytes() { @Override public String toString() { - if(!this.isEmpty()) + if(this.hasValidValue()) return String.format("min: %s, max: %s, num_nulls: %d", min.toStringUsingUTF8(), max.toStringUsingUTF8(), this.getNumNulls()); + else if(!this.isEmpty()) + return String.format("num_nulls: %d, min/max not defined", this.getNumNulls()); else return "no stats for this column"; } @@ -74,6 +77,7 @@ public void updateStats(Binary min_value, Binary max_value) { public void initializeStats(Binary min_value, Binary max_value) { min = min_value; max = max_value; + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -87,6 +91,24 @@ public Binary genericGetMax() { return max; } + @Override + public void incrementNumNulls() { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(); + } + + @Override + public void incrementNumNulls(long increment) { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(increment); + } + public Binary getMax() { return max; } @@ -98,6 +120,8 @@ public Binary getMin() { public void setMinMax(Binary min, Binary max) { this.max = max; this.min = min; + this.markHasValidValue(); this.markAsNotEmpty(); } -} \ No newline at end of file + +} diff --git a/parquet-column/src/main/java/parquet/column/statistics/BooleanStatistics.java b/parquet-column/src/main/java/parquet/column/statistics/BooleanStatistics.java index 6741343496..3707fbb1f6 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/BooleanStatistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/BooleanStatistics.java @@ -24,7 +24,7 @@ public class BooleanStatistics extends Statistics { @Override public void updateStats(boolean value) { - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(value, value); } else { updateStats(value, value); @@ -34,7 +34,7 @@ public void updateStats(boolean value) { @Override public void mergeStatisticsMinMax(Statistics stats) { BooleanStatistics boolStats = (BooleanStatistics)stats; - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(boolStats.getMin(), boolStats.getMax()); } else { updateStats(boolStats.getMin(), boolStats.getMax()); @@ -45,6 +45,7 @@ public void mergeStatisticsMinMax(Statistics stats) { public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { max = BytesUtils.bytesToBool(maxBytes); min = BytesUtils.bytesToBool(minBytes); + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -60,8 +61,10 @@ public byte[] getMinBytes() { @Override public String toString() { - if(!this.isEmpty()) + if(this.hasValidValue()) return String.format("min: %b, max: %b, num_nulls: %d", min, max, this.getNumNulls()); + else if(!this.isEmpty()) + return String.format("num_nulls: %d, min/max not defined", this.getNumNulls()); else return "no stats for this column"; } @@ -74,6 +77,7 @@ public void updateStats(boolean min_value, boolean max_value) { public void initializeStats(boolean min_value, boolean max_value) { min = min_value; max = max_value; + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -86,6 +90,24 @@ public Boolean genericGetMin() { public Boolean genericGetMax() { return max; } + + @Override + public void incrementNumNulls() { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(); + } + + @Override + public void incrementNumNulls(long increment) { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(increment); + } public boolean getMax() { return max; @@ -98,6 +120,7 @@ public boolean getMin() { public void setMinMax(boolean min, boolean max) { this.max = max; this.min = min; + this.markHasValidValue(); this.markAsNotEmpty(); } } diff --git a/parquet-column/src/main/java/parquet/column/statistics/DoubleStatistics.java b/parquet-column/src/main/java/parquet/column/statistics/DoubleStatistics.java index c9695f355e..189dbdeb3e 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/DoubleStatistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/DoubleStatistics.java @@ -24,7 +24,7 @@ public class DoubleStatistics extends Statistics { @Override public void updateStats(double value) { - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(value, value); } else { updateStats(value, value); @@ -34,7 +34,7 @@ public void updateStats(double value) { @Override public void mergeStatisticsMinMax(Statistics stats) { DoubleStatistics doubleStats = (DoubleStatistics)stats; - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(doubleStats.getMin(), doubleStats.getMax()); } else { updateStats(doubleStats.getMin(), doubleStats.getMax()); @@ -45,6 +45,7 @@ public void mergeStatisticsMinMax(Statistics stats) { public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { max = Double.longBitsToDouble(BytesUtils.bytesToLong(maxBytes)); min = Double.longBitsToDouble(BytesUtils.bytesToLong(minBytes)); + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -60,9 +61,11 @@ public byte[] getMinBytes() { @Override public String toString() { - if(!this.isEmpty()) + if(this.hasValidValue()) return String.format("min: %.5f, max: %.5f, num_nulls: %d", min, max, this.getNumNulls()); - else + else if(!this.isEmpty()) + return String.format("num_nulls: %d, min, max not defined", this.getNumNulls()); + else return "no stats for this column"; } @@ -74,6 +77,7 @@ public void updateStats(double min_value, double max_value) { public void initializeStats(double min_value, double max_value) { min = min_value; max = max_value; + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -87,6 +91,24 @@ public Double genericGetMax() { return max; } + @Override + public void incrementNumNulls() { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(); + } + + @Override + public void incrementNumNulls(long increment) { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(increment); + } + public double getMax() { return max; } @@ -98,6 +120,7 @@ public double getMin() { public void setMinMax(double min, double max) { this.max = max; this.min = min; + this.markHasValidValue(); this.markAsNotEmpty(); } -} \ No newline at end of file +} diff --git a/parquet-column/src/main/java/parquet/column/statistics/FloatStatistics.java b/parquet-column/src/main/java/parquet/column/statistics/FloatStatistics.java index b13aafa98f..bc5ab6a21e 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/FloatStatistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/FloatStatistics.java @@ -24,7 +24,7 @@ public class FloatStatistics extends Statistics { @Override public void updateStats(float value) { - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(value, value); } else { updateStats(value, value); @@ -34,7 +34,7 @@ public void updateStats(float value) { @Override public void mergeStatisticsMinMax(Statistics stats) { FloatStatistics floatStats = (FloatStatistics)stats; - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(floatStats.getMin(), floatStats.getMax()); } else { updateStats(floatStats.getMin(), floatStats.getMax()); @@ -45,6 +45,7 @@ public void mergeStatisticsMinMax(Statistics stats) { public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { max = Float.intBitsToFloat(BytesUtils.bytesToInt(maxBytes)); min = Float.intBitsToFloat(BytesUtils.bytesToInt(minBytes)); + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -60,8 +61,10 @@ public byte[] getMinBytes() { @Override public String toString() { - if(!this.isEmpty()) + if(this.hasValidValue()) return String.format("min: %.5f, max: %.5f, num_nulls: %d", min, max, this.getNumNulls()); + else if(!this.isEmpty()) + return String.format("num_nulls: %d, min/max not defined", this.getNumNulls()); else return "no stats for this column"; } @@ -74,6 +77,7 @@ public void updateStats(float min_value, float max_value) { public void initializeStats(float min_value, float max_value) { min = min_value; max = max_value; + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -86,6 +90,24 @@ public Float genericGetMin() { public Float genericGetMax() { return max; } + + @Override + public void incrementNumNulls() { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(); + } + + @Override + public void incrementNumNulls(long increment) { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(increment); + } public float getMax() { return max; @@ -98,6 +120,7 @@ public float getMin() { public void setMinMax(float min, float max) { this.max = max; this.min = min; + this.markHasValidValue(); this.markAsNotEmpty(); } } diff --git a/parquet-column/src/main/java/parquet/column/statistics/IntStatistics.java b/parquet-column/src/main/java/parquet/column/statistics/IntStatistics.java index 7bdd6be297..adcdcafa40 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/IntStatistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/IntStatistics.java @@ -24,7 +24,7 @@ public class IntStatistics extends Statistics { @Override public void updateStats(int value) { - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(value, value); } else { updateStats(value, value); @@ -34,7 +34,7 @@ public void updateStats(int value) { @Override public void mergeStatisticsMinMax(Statistics stats) { IntStatistics intStats = (IntStatistics)stats; - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(intStats.getMin(), intStats.getMax()); } else { updateStats(intStats.getMin(), intStats.getMax()); @@ -45,6 +45,7 @@ public void mergeStatisticsMinMax(Statistics stats) { public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { max = BytesUtils.bytesToInt(maxBytes); min = BytesUtils.bytesToInt(minBytes); + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -60,8 +61,10 @@ public byte[] getMinBytes() { @Override public String toString() { - if(!this.isEmpty()) + if(this.hasValidValue()) return String.format("min: %d, max: %d, num_nulls: %d", min, max, this.getNumNulls()); + else if(!this.isEmpty()) + return String.format("num_nulls: %d, min/max not defined", this.getNumNulls()); else return "no stats for this column"; } @@ -74,6 +77,7 @@ public void updateStats(int min_value, int max_value) { public void initializeStats(int min_value, int max_value) { min = min_value; max = max_value; + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -86,6 +90,24 @@ public Integer genericGetMin() { public Integer genericGetMax() { return max; } + + @Override + public void incrementNumNulls() { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(); + } + + @Override + public void incrementNumNulls(long increment) { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(increment); + } public int getMax() { return max; @@ -98,6 +120,7 @@ public int getMin() { public void setMinMax(int min, int max) { this.max = max; this.min = min; + this.markHasValidValue(); this.markAsNotEmpty(); } } diff --git a/parquet-column/src/main/java/parquet/column/statistics/LongStatistics.java b/parquet-column/src/main/java/parquet/column/statistics/LongStatistics.java index bae63a9973..7d17a20b46 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/LongStatistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/LongStatistics.java @@ -24,7 +24,7 @@ public class LongStatistics extends Statistics { @Override public void updateStats(long value) { - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(value, value); } else { updateStats(value, value); @@ -34,7 +34,7 @@ public void updateStats(long value) { @Override public void mergeStatisticsMinMax(Statistics stats) { LongStatistics longStats = (LongStatistics)stats; - if (this.isEmpty()) { + if (this.isEmpty() || !this.hasValidValue()) { initializeStats(longStats.getMin(), longStats.getMax()); } else { updateStats(longStats.getMin(), longStats.getMax()); @@ -45,6 +45,7 @@ public void mergeStatisticsMinMax(Statistics stats) { public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) { max = BytesUtils.bytesToLong(maxBytes); min = BytesUtils.bytesToLong(minBytes); + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -60,8 +61,10 @@ public byte[] getMinBytes() { @Override public String toString() { - if(!this.isEmpty()) + if(this.hasValidValue()) return String.format("min: %d, max: %d, num_nulls: %d", min, max, this.getNumNulls()); + else if(!this.isEmpty()) + return String.format("num_nulls: %d, min/max not defined", this.getNumNulls()); else return "no stats for this column"; } @@ -74,6 +77,7 @@ public void updateStats(long min_value, long max_value) { public void initializeStats(long min_value, long max_value) { min = min_value; max = max_value; + this.markHasValidValue(); this.markAsNotEmpty(); } @@ -86,6 +90,24 @@ public Long genericGetMin() { public Long genericGetMax() { return max; } + + @Override + public void incrementNumNulls() { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(); + } + + @Override + public void incrementNumNulls(long increment) { + if (this.isEmpty()) { + this.markAsNotEmpty(); + } + + super.incrementNumNulls(increment); + } public long getMax() { return max; @@ -98,6 +120,7 @@ public long getMin() { public void setMinMax(long min, long max) { this.max = max; this.min = min; + this.markHasValidValue(); this.markAsNotEmpty(); } -} \ No newline at end of file +} diff --git a/parquet-column/src/main/java/parquet/column/statistics/Statistics.java b/parquet-column/src/main/java/parquet/column/statistics/Statistics.java index b29b76b3c3..ddcc4360a2 100644 --- a/parquet-column/src/main/java/parquet/column/statistics/Statistics.java +++ b/parquet-column/src/main/java/parquet/column/statistics/Statistics.java @@ -29,6 +29,7 @@ public abstract class Statistics> { private boolean firstValueAccountedFor; + private boolean containsValidValue; private long num_nulls; public Statistics() { @@ -142,7 +143,8 @@ public void mergeStatistics(Statistics stats) { if (this.getClass() == stats.getClass()) { incrementNumNulls(stats.getNumNulls()); - mergeStatisticsMinMax(stats); + if(stats.hasValidValue()) + mergeStatisticsMinMax(stats); } else { throw new StatisticsClassException(this.getClass().toString(), stats.getClass().toString()); } @@ -185,6 +187,7 @@ public void mergeStatistics(Statistics stats) { /** * Increments the null count by one + * Should call markAsNotEmpty, but called in the derived class */ public void incrementNumNulls() { num_nulls++ ; @@ -193,6 +196,7 @@ public void incrementNumNulls() { /** * Increments the null count by the parameter value * @param increment value to increment the null count by + * Should call markAsNotEmpty, but called in the derived class */ public void incrementNumNulls(long increment) { num_nulls += increment ; @@ -216,7 +220,7 @@ public void setNumNulls(long nulls) { /** * Returns a boolean specifying if the Statistics object is empty, - * i.e does not contain valid statistics for the page/column yet + * i.e page/column does not contain any value yet * @return true if object is empty, false otherwise */ public boolean isEmpty() { @@ -226,5 +230,20 @@ public boolean isEmpty() { protected void markAsNotEmpty() { firstValueAccountedFor = true; } + + /** + * Returns a boolean specifying if the min/max of Statistics + * object has a valid value, since it contains non-null + * values + * @return true if page/column has non-null value, false otherwise + */ + public boolean hasValidValue() { + return containsValidValue; + } + + protected void markHasValidValue() { + containsValidValue = true; + } + } diff --git a/parquet-hadoop/src/main/java/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/parquet/format/converter/ParquetMetadataConverter.java index 198b6548ba..a51a168509 100644 --- a/parquet-hadoop/src/main/java/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/parquet/format/converter/ParquetMetadataConverter.java @@ -234,9 +234,11 @@ public Encoding getEncoding(parquet.column.Encoding encoding) { public static Statistics toParquetStatistics(parquet.column.statistics.Statistics statistics) { Statistics stats = new Statistics(); if (!statistics.isEmpty()) { - stats.setMax(statistics.getMaxBytes()); - stats.setMin(statistics.getMinBytes()); stats.setNull_count(statistics.getNumNulls()); + if(statistics.hasValidValue()) { + stats.setMax(statistics.getMaxBytes()); + stats.setMin(statistics.getMinBytes()); + } } return stats; } @@ -246,7 +248,9 @@ public static parquet.column.statistics.Statistics fromParquetStatistics(Statist parquet.column.statistics.Statistics stats = parquet.column.statistics.Statistics.getStatsBasedOnType(type); // If there was no statistics written to the footer, create an empty Statistics object and return if (statistics != null) { - stats.setMinMaxFromBytes(statistics.min.array(), statistics.max.array()); + if (statistics.isSetMax() && statistics.isSetMin()) { + stats.setMinMaxFromBytes(statistics.min.array(), statistics.max.array()); + } stats.setNumNulls(statistics.null_count); } return stats;