From 0610e8f11ef8ed93130c39ccf0e2d4fb8c905be8 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 17:31:26 +0800 Subject: [PATCH 01/52] Preview of replacing Log.java with slf4j --- .../apache/parquet/io/BaseRecordReader.java | 28 ++++++++++--------- .../thrift/AbstractThriftWriteSupport.java | 7 +++-- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java b/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java index f2d88fc11f..6b79df0671 100644 --- a/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java @@ -19,16 +19,18 @@ package org.apache.parquet.io; import static org.apache.parquet.Log.DEBUG; -import org.apache.parquet.Log; import org.apache.parquet.column.ColumnReadStore; import org.apache.parquet.io.RecordReaderImplementation.State; import org.apache.parquet.io.api.Binary; import org.apache.parquet.io.api.RecordConsumer; import org.apache.parquet.io.api.RecordMaterializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + // TODO(julien): this class appears to be unused -- can it be nuked? - todd public abstract class BaseRecordReader extends RecordReader { - private static final Log LOG = Log.getLog(BaseRecordReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BaseRecordReader.class); public RecordConsumer recordConsumer; public RecordMaterializer recordMaterializer; @@ -48,11 +50,11 @@ public T read() { private int endIndex; protected void currentLevel(int currentLevel) { - if (DEBUG) LOG.debug("currentLevel: "+currentLevel); + LOGGER.debug("currentLevel: {}", currentLevel); } protected void log(String message) { - if (DEBUG) LOG.debug("bc: "+message); + LOGGER.debug("bc: {}", message); } final protected int getCaseId(int state, int currentLevel, int d, int nextR) { @@ -62,18 +64,18 @@ final protected int getCaseId(int state, int currentLevel, int d, int nextR) { final protected void startMessage() { // reset state endField = null; - if (DEBUG) LOG.debug("startMessage()"); + LOGGER.debug("startMessage()"); recordConsumer.startMessage(); } final protected void startGroup(String field, int index) { startField(field, index); - if (DEBUG) LOG.debug("startGroup()"); + LOGGER.debug("startGroup()"); recordConsumer.startGroup(); } private void startField(String field, int index) { - if (DEBUG) LOG.debug("startField("+field+","+index+")"); + LOGGER.debug("startField({},{})", field, index); if (endField != null && index == endIndex) { // skip the close/open tag endField = null; @@ -89,13 +91,13 @@ private void startField(String field, int index) { final protected void addPrimitiveINT64(String field, int index, long value) { startField(field, index); - if (DEBUG) LOG.debug("addLong("+value+")"); + LOGGER.debug("addLong({})", value); recordConsumer.addLong(value); endField(field, index); } private void endField(String field, int index) { - if (DEBUG) LOG.debug("endField("+field+","+index+")"); + LOGGER.debug("endField({},{})", field, index); if (endField != null) { recordConsumer.endField(endField, endIndex); } @@ -105,14 +107,14 @@ private void endField(String field, int index) { final protected void addPrimitiveBINARY(String field, int index, Binary value) { startField(field, index); - if (DEBUG) LOG.debug("addBinary("+value+")"); + LOGGER.debug("addBinary({})", value); recordConsumer.addBinary(value); endField(field, index); } final protected void addPrimitiveINT32(String field, int index, int value) { startField(field, index); - if (DEBUG) LOG.debug("addInteger("+value+")"); + LOGGER.debug("addInteger({})", value); recordConsumer.addInteger(value); endField(field, index); } @@ -123,7 +125,7 @@ final protected void endGroup(String field, int index) { recordConsumer.endField(endField, endIndex); endField = null; } - if (DEBUG) LOG.debug("endGroup()"); + LOGGER.debug("endGroup()"); recordConsumer.endGroup(); endField(field, index); } @@ -134,7 +136,7 @@ final protected void endMessage() { recordConsumer.endField(endField, endIndex); endField = null; } - if (DEBUG) LOG.debug("endMessage()"); + LOGGER.debug("endMessage()"); recordConsumer.endMessage(); } diff --git a/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/AbstractThriftWriteSupport.java b/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/AbstractThriftWriteSupport.java index 5f210d3280..1cb4fd46f5 100644 --- a/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/AbstractThriftWriteSupport.java +++ b/parquet-thrift/src/main/java/org/apache/parquet/hadoop/thrift/AbstractThriftWriteSupport.java @@ -22,7 +22,6 @@ import com.twitter.elephantbird.pig.util.ThriftToPig; -import org.apache.parquet.Log; import org.apache.parquet.hadoop.BadConfigurationException; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.io.ColumnIOFactory; @@ -35,10 +34,12 @@ import org.apache.parquet.thrift.ThriftSchemaConverter; import org.apache.parquet.thrift.struct.ThriftType.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class AbstractThriftWriteSupport extends WriteSupport { public static final String PARQUET_THRIFT_CLASS = "parquet.thrift.class"; - private static final Log LOG = Log.getLog(AbstractThriftWriteSupport.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractThriftWriteSupport.class); public static void setGenericThriftClass(Configuration configuration, Class thriftClass) { configuration.set(PARQUET_THRIFT_CLASS, thriftClass.getName()); @@ -101,7 +102,7 @@ protected boolean isPigLoaded() { Class.forName("org.apache.pig.impl.logicalLayer.schema.Schema"); return true; } catch (ClassNotFoundException e) { - LOG.info("Pig is not loaded, pig metadata will not be written"); + LOGGER.info("Pig is not loaded, pig metadata will not be written"); return false; } } From 5c77437dd12abfe685c538ea7cef443c09303b10 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 17:39:56 +0800 Subject: [PATCH 02/52] Remove the static import of BaseRecordReader --- .../src/main/java/org/apache/parquet/io/BaseRecordReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java b/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java index 6b79df0671..45730c80b0 100644 --- a/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/io/BaseRecordReader.java @@ -18,7 +18,6 @@ */ package org.apache.parquet.io; -import static org.apache.parquet.Log.DEBUG; import org.apache.parquet.column.ColumnReadStore; import org.apache.parquet.io.RecordReaderImplementation.State; import org.apache.parquet.io.api.Binary; From a0d2f038576125b420beeb021e91c9404b1e4153 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 18:07:51 +0800 Subject: [PATCH 03/52] 03~10 --- .../parquet/column/impl/ColumnReaderImpl.java | 23 +++++++++------- .../bitpacking/BitPackingValuesReader.java | 7 ++--- .../ByteBitPackingValuesReader.java | 8 +++--- .../column/values/boundedint/BitWriter.java | 27 ++++++++++++------- .../boundedint/BoundedIntValuesReader.java | 18 ++++++++----- .../boundedint/BoundedIntValuesWriter.java | 11 +++++--- .../values/plain/BinaryPlainValuesReader.java | 10 +++---- .../plain/BooleanPlainValuesReader.java | 9 ++++--- 8 files changed, 68 insertions(+), 45 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java index 8c2a4bf2eb..72f6fbf2a9 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnReaderImpl.java @@ -19,7 +19,6 @@ package org.apache.parquet.column.impl; import static java.lang.String.format; -import static org.apache.parquet.Log.DEBUG; import static org.apache.parquet.Preconditions.checkNotNull; import static org.apache.parquet.column.ValuesType.DEFINITION_LEVEL; import static org.apache.parquet.column.ValuesType.REPETITION_LEVEL; @@ -30,7 +29,6 @@ import java.nio.ByteBuffer; import org.apache.parquet.CorruptDeltaByteArrays; -import org.apache.parquet.Log; import org.apache.parquet.VersionParser.ParsedVersion; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.BytesUtils; @@ -52,6 +50,9 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeNameConverter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * ColumnReader implementation * @@ -59,7 +60,7 @@ * */ public class ColumnReaderImpl implements ColumnReader { - private static final Log LOG = Log.getLog(ColumnReaderImpl.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ColumnReaderImpl.class); /** * binds the lower level page decoder to the record converter materializing the records @@ -523,7 +524,7 @@ private void readRepetitionAndDefinitionLevels() { private void checkRead() { if (isPageFullyConsumed()) { if (isFullyConsumed()) { - if (DEBUG) LOG.debug("end reached"); + LOGGER.debug("end reached"); repetitionLevel = 0; // the next repetition level return; } @@ -533,7 +534,7 @@ private void checkRead() { } private void readPage() { - if (DEBUG) LOG.debug("loading page"); + LOGGER.debug("loading page"); DataPage page = pageReader.readPage(); page.accept(new DataPage.Visitor() { @Override @@ -590,14 +591,15 @@ private void readPageV1(DataPageV1 page) { this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader); try { ByteBuffer bytes = page.getBytes().toByteBuffer(); - if (DEBUG) LOG.debug("page size " + bytes.remaining() + " bytes and " + pageValueCount + " records"); - if (DEBUG) LOG.debug("reading repetition levels at 0"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("page size {} bytes and {} records", bytes.remaining(), pageValueCount); + LOGGER.debug("reading repetition levels at 0"); rlReader.initFromPage(pageValueCount, bytes, 0); int next = rlReader.getNextOffset(); - if (DEBUG) LOG.debug("reading definition levels at " + next); + LOGGER.debug("reading definition levels at {}", next); dlReader.initFromPage(pageValueCount, bytes, next); next = dlReader.getNextOffset(); - if (DEBUG) LOG.debug("reading data at " + next); + LOGGER.debug("reading data at {}", next); initDataReader(page.getValueEncoding(), bytes, next, page.getValueCount()); } catch (IOException e) { throw new ParquetDecodingException("could not read page " + page + " in col " + path, e); @@ -608,7 +610,8 @@ private void readPageV2(DataPageV2 page) { this.repetitionLevelColumn = newRLEIterator(path.getMaxRepetitionLevel(), page.getRepetitionLevels()); this.definitionLevelColumn = newRLEIterator(path.getMaxDefinitionLevel(), page.getDefinitionLevels()); try { - if (DEBUG) LOG.debug("page data size " + page.getData().size() + " bytes and " + pageValueCount + " records"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("page data size {} bytes and {} records", page.getData().size(), pageValueCount); initDataReader(page.getDataEncoding(), page.getData().toByteBuffer(), 0, page.getValueCount()); } catch (IOException e) { throw new ParquetDecodingException("could not read page " + page + " in col " + path, e); diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/BitPackingValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/BitPackingValuesReader.java index f540c392b5..682825d888 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/BitPackingValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/BitPackingValuesReader.java @@ -25,12 +25,13 @@ import java.nio.ByteBuffer; import org.apache.parquet.bytes.ByteBufferInputStream; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.column.values.bitpacking.BitPacking.BitPackingReader; import org.apache.parquet.io.ParquetDecodingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * a column reader that packs the ints in the number of bits required based on the maximum size. * @@ -38,7 +39,7 @@ * */ public class BitPackingValuesReader extends ValuesReader { - private static final Log LOG = Log.getLog(BitPackingValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BitPackingValuesReader.class); private ByteBufferInputStream in; private BitPackingReader bitPackingReader; @@ -73,7 +74,7 @@ public int readInteger() { public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { int effectiveBitLength = valueCount * bitsPerValue; int length = BytesUtils.paddedByteCountFromBits(effectiveBitLength); - if (Log.DEBUG) LOG.debug("reading " + length + " bytes for " + valueCount + " values of size " + bitsPerValue + " bits." ); + LOGGER.debug("reading {} bytes for {} values of size {} bits.", length, valueCount, bitsPerValue); this.in = new ByteBufferInputStream(in, offset, length); this.bitPackingReader = createBitPackingReader(bitsPerValue, this.in, valueCount); this.nextOffset = offset + length; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPackingValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPackingValuesReader.java index f4c8c8efc5..829a111dc1 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPackingValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBitPackingValuesReader.java @@ -22,14 +22,16 @@ import java.util.Arrays; import java.nio.ByteBuffer; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.values.ValuesReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class ByteBitPackingValuesReader extends ValuesReader { private static final int VALUES_AT_A_TIME = 8; // because we're using unpack8Values() - private static final Log LOG = Log.getLog(ByteBitPackingValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ByteBitPackingValuesReader.class); private final int bitWidth; private final BytePacker packer; @@ -69,7 +71,7 @@ public void initFromPage(int valueCount, ByteBuffer page, int offset) throws IOException { int effectiveBitLength = valueCount * bitWidth; int length = BytesUtils.paddedByteCountFromBits(effectiveBitLength); // ceil - if (Log.DEBUG) LOG.debug("reading " + length + " bytes for " + valueCount + " values of size " + bitWidth + " bits." ); + LOGGER.debug("reading {} bytes for {} values of size {} bits.", length, valueCount, bitWidth); this.encoded = page; this.encodedPos = offset; this.decodedPosition = VALUES_AT_A_TIME - 1; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BitWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BitWriter.java index 9489714b31..af800bc326 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BitWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BitWriter.java @@ -19,12 +19,14 @@ package org.apache.parquet.column.values.boundedint; import org.apache.parquet.bytes.ByteBufferAllocator; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + class BitWriter { - private static final Log LOG = Log.getLog(BitWriter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BitWriter.class); private static final boolean DEBUG = false;//Log.DEBUG; private CapacityByteArrayOutputStream baos; @@ -47,21 +49,25 @@ public BitWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocato } public void writeBit(boolean bit) { - if (DEBUG) LOG.debug("writing: " + (bit ? "1" : "0")); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("writing: {}", bit ? "1" : "0"); currentByte = setBytePosition(currentByte, currentBytePosition++, bit); if (currentBytePosition == 8) { baos.write(currentByte); - if (DEBUG) LOG.debug("to buffer: " + toBinary(currentByte)); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("to buffer: {}", toBinary(currentByte)); currentByte = 0; currentBytePosition = 0; } } public void writeByte(int val) { - if (DEBUG) LOG.debug("writing: " + toBinary(val) + " (" + val + ")"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("writing: {} ({})", toBinary(val), val); currentByte |= ((val & 0xFF) << currentBytePosition); baos.write(currentByte); - if (DEBUG) LOG.debug("to buffer: " + toBinary(currentByte)); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("to buffer: {}", toBinary(currentByte)); currentByte >>>= 8; } @@ -73,13 +79,15 @@ public void writeByte(int val) { * @param bitsToWrite the number of bits to use */ public void writeNBitInteger(int val, int bitsToWrite) { - if (DEBUG) LOG.debug("writing: " + toBinary(val, bitsToWrite) + " (" + val + ")"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("writing: {} ({})", toBinary(val, bitsToWrite), val); val <<= currentBytePosition; int upperByte = currentBytePosition + bitsToWrite; currentByte |= val; while (upperByte >= 8) { baos.write(currentByte); //this only writes the lowest byte - if (DEBUG) LOG.debug("to buffer: " + toBinary(currentByte)); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("to buffer: {}", toBinary(currentByte)); upperByte -= 8; currentByte >>>= 8; } @@ -102,7 +110,8 @@ public BytesInput finish() { if (!finished) { if (currentBytePosition > 0) { baos.write(currentByte); - if (DEBUG) LOG.debug("to buffer: " + toBinary(currentByte)); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("to buffer: {}", toBinary(currentByte)); } } finished = true; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesReader.java index c322125e7e..074ece3fcb 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesReader.java @@ -18,21 +18,21 @@ */ package org.apache.parquet.column.values.boundedint; -import static org.apache.parquet.Log.DEBUG; - import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.ParquetDecodingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * @see BoundedIntValuesWriter */ class BoundedIntValuesReader extends ValuesReader { - private static final Log LOG = Log.getLog(BoundedIntValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BoundedIntValuesReader.class); private int currentValueCt = 0; private int currentValue = 0; @@ -72,13 +72,17 @@ public int readInteger() { // to BoundedIntColumnWriter.writeData(BytesOutput) @Override public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { - if (DEBUG) LOG.debug("reading size at "+ offset + ": " + in.get(offset) + " " + in.get(offset + 1) + " " + in.get(offset + 2) + " " + in.get(offset + 3) + " "); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("reading size at {}: {} {} {} {} ", + offset, in.get(offset), in.get(offset + 1), in.get(offset + 2), in.get(offset + 3)); + } int totalBytes = BytesUtils.readIntLittleEndian(in, offset); - if (DEBUG) LOG.debug("will read "+ totalBytes + " bytes"); + LOGGER.debug("will read {} bytes", totalBytes); currentValueCt = 0; currentValue = 0; bitReader.prepare(in, offset + 4, totalBytes); - if (DEBUG) LOG.debug("will read next from " + (offset + totalBytes + 4)); + LOGGER.debug("will read next from {}", offset + totalBytes + 4); + this.nextOffset = offset + totalBytes + 4; } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesWriter.java index a90a6e5fcf..deda406739 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/boundedint/BoundedIntValuesWriter.java @@ -21,13 +21,15 @@ import org.apache.parquet.bytes.ByteBufferAllocator; import static org.apache.parquet.bytes.BytesInput.concat; import static org.apache.parquet.column.Encoding.RLE; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.bitpacking.BitPackingValuesWriter; import org.apache.parquet.io.ParquetEncodingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * This is a special ColumnWriter for the case when you need to write * integers in a known range. This is intended primarily for use with @@ -40,7 +42,7 @@ * values are expected. */ class BoundedIntValuesWriter extends ValuesWriter { - private static final Log LOG = Log.getLog(BoundedIntValuesWriter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BoundedIntValuesWriter.class); private int currentValue = -1; private int currentValueCt = -1; @@ -67,7 +69,7 @@ public BoundedIntValuesWriter(int bound, int initialCapacity, int pageSize, Byte this.bitWriter = new BitWriter(initialCapacity, pageSize, allocator); bitsPerValue = (int)Math.ceil(Math.log(bound + 1)/Math.log(2)); shouldRepeatThreshold = (bitsPerValue + 9)/(1 + bitsPerValue); - if (Log.DEBUG) LOG.debug("init column with bit width of " + bitsPerValue + " and repeat threshold of " + shouldRepeatThreshold); + LOGGER.debug("init column with bit width of {} and repeat threshold of {}", bitsPerValue, shouldRepeatThreshold); } @Override @@ -84,7 +86,8 @@ public long getBufferedSize() { public BytesInput getBytes() { serializeCurrentValue(); BytesInput buf = bitWriter.finish(); - if (Log.DEBUG) LOG.debug("writing a buffer of size " + buf.size() + " + 4 bytes"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("writing a buffer of size {} + 4 bytes", buf.size()); // We serialize the length so that on deserialization we can // deserialize as we go, instead of having to load everything // into memory diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BinaryPlainValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BinaryPlainValuesReader.java index 26f5e29796..dffb0158a5 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BinaryPlainValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BinaryPlainValuesReader.java @@ -18,19 +18,19 @@ */ package org.apache.parquet.column.values.plain; -import static org.apache.parquet.Log.DEBUG; - import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.api.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class BinaryPlainValuesReader extends ValuesReader { - private static final Log LOG = Log.getLog(BinaryPlainValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BinaryPlainValuesReader.class); private ByteBuffer in; private int offset; @@ -63,7 +63,7 @@ public void skip() { @Override public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { - if (DEBUG) LOG.debug("init from page at offset "+ offset + " for length " + (in.limit() - offset)); + LOGGER.debug("init from page at offset {} for length {}", offset, (in.limit() - offset)); this.in = in; this.offset = offset; } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BooleanPlainValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BooleanPlainValuesReader.java index a279938c69..71de56a595 100755 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BooleanPlainValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/BooleanPlainValuesReader.java @@ -18,16 +18,17 @@ */ package org.apache.parquet.column.values.plain; -import static org.apache.parquet.Log.DEBUG; import static org.apache.parquet.column.values.bitpacking.Packer.LITTLE_ENDIAN; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.parquet.Log; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.column.values.bitpacking.ByteBitPackingValuesReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * encodes boolean for the plain encoding: one bit at a time (0 = false) * @@ -35,7 +36,7 @@ * */ public class BooleanPlainValuesReader extends ValuesReader { - private static final Log LOG = Log.getLog(BooleanPlainValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BooleanPlainValuesReader.class); private ByteBitPackingValuesReader in = new ByteBitPackingValuesReader(1, LITTLE_ENDIAN); @@ -64,7 +65,7 @@ public void skip() { */ @Override public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { - if (DEBUG) LOG.debug("init from page at offset "+ offset + " for length " + (in.limit() - offset)); + LOGGER.debug("init from page at offset {} for length {}", offset, (in.limit() - offset)); this.in.initFromPage(valueCount, in, offset); } From dfb87aa37012761adbed2b5d4cf10332e3115a49 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 18:15:12 +0800 Subject: [PATCH 04/52] 11~15 --- .../java/org/apache/parquet/Closeables.java | 7 ++-- .../org/apache/parquet/bytes/BytesUtils.java | 8 +++-- .../org/apache/parquet/bytes/BytesInput.java | 34 +++++++++++-------- .../bytes/CapacityByteArrayOutputStream.java | 20 ++++++++--- .../ByteBasedBitPackingEncoder.java | 11 +++--- 5 files changed, 52 insertions(+), 28 deletions(-) diff --git a/parquet-common/src/main/java/org/apache/parquet/Closeables.java b/parquet-common/src/main/java/org/apache/parquet/Closeables.java index 2d8bb77469..af21aafb74 100644 --- a/parquet-common/src/main/java/org/apache/parquet/Closeables.java +++ b/parquet-common/src/main/java/org/apache/parquet/Closeables.java @@ -21,13 +21,16 @@ import java.io.Closeable; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Utility for working with {@link java.io.Closeable}ss */ public final class Closeables { private Closeables() { } - private static final Log LOG = Log.getLog(Closeables.class); + private static final Logger LOGGER = LoggerFactory.getLogger(Closeables.class); /** * Closes a (potentially null) closeable. @@ -49,7 +52,7 @@ public static void closeAndSwallowIOExceptions(Closeable c) { try { c.close(); } catch (IOException e) { - LOG.warn("Encountered exception closing closeable", e); + LOGGER.warn("Encountered exception closing closeable", e); } } } diff --git a/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java b/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java index d40721a8f4..0647cd7e11 100644 --- a/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java +++ b/parquet-common/src/main/java/org/apache/parquet/bytes/BytesUtils.java @@ -25,7 +25,8 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; -import org.apache.parquet.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * utility methods to deal with bytes @@ -34,7 +35,7 @@ * */ public class BytesUtils { - private static final Log LOG = Log.getLog(BytesUtils.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BytesUtils.class); public static final Charset UTF8 = Charset.forName("UTF-8"); @@ -158,7 +159,8 @@ public static void writeIntLittleEndian(OutputStream out, int v) throws IOExcept out.write((v >>> 8) & 0xFF); out.write((v >>> 16) & 0xFF); out.write((v >>> 24) & 0xFF); - if (Log.DEBUG) LOG.debug("write le int: " + v + " => "+ ((v >>> 0) & 0xFF) + " " + ((v >>> 8) & 0xFF) + " " + ((v >>> 16) & 0xFF) + " " + ((v >>> 24) & 0xFF)); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("write le int: " + v + " => "+ ((v >>> 0) & 0xFF) + " " + ((v >>> 8) & 0xFF) + " " + ((v >>> 16) & 0xFF) + " " + ((v >>> 24) & 0xFF)); } /** diff --git a/parquet-encoding/src/main/java/org/apache/parquet/bytes/BytesInput.java b/parquet-encoding/src/main/java/org/apache/parquet/bytes/BytesInput.java index 40190ee3bf..ab0aeb26e1 100644 --- a/parquet-encoding/src/main/java/org/apache/parquet/bytes/BytesInput.java +++ b/parquet-encoding/src/main/java/org/apache/parquet/bytes/BytesInput.java @@ -29,8 +29,8 @@ import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; -import org.apache.parquet.Log; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A source of bytes capable of writing itself to an output. @@ -44,8 +44,7 @@ * */ abstract public class BytesInput { - private static final Log LOG = Log.getLog(BytesInput.class); - private static final boolean DEBUG = false;//Log.DEBUG; + private static final Logger LOGGER = LoggerFactory.getLogger(BytesInput.class); private static final EmptyBytesInput EMPTY_BYTES_INPUT = new EmptyBytesInput(); /** @@ -90,12 +89,13 @@ public static BytesInput from(ByteBuffer buffer, int offset, int length) { * @return a Bytes input that will write the given bytes */ public static BytesInput from(byte[] in) { - if (DEBUG) LOG.debug("BytesInput from array of " + in.length + " bytes"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("BytesInput from array of {} bytes", in.length); return new ByteArrayBytesInput(in, 0 , in.length); } public static BytesInput from(byte[] in, int offset, int length) { - if (DEBUG) LOG.debug("BytesInput from array of " + length + " bytes"); + LOGGER.debug("BytesInput from array of {} bytes", length); return new ByteArrayBytesInput(in, offset, length); } @@ -172,7 +172,8 @@ public static BytesInput copy(BytesInput bytesInput) throws IOException { public byte[] toByteArray() throws IOException { BAOS baos = new BAOS((int)size()); this.writeAllTo(baos); - if (DEBUG) LOG.debug("converted " + size() + " to byteArray of " + baos.size() + " bytes"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("converted {} to byteArray of {} bytes", size(), baos.size()); return baos.getBuf(); } @@ -211,7 +212,7 @@ public byte[] getBuf() { } private static class StreamBytesInput extends BytesInput { - private static final Log LOG = Log.getLog(BytesInput.StreamBytesInput.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BytesInput.StreamBytesInput.class); private final InputStream in; private final int byteCount; @@ -223,13 +224,13 @@ private StreamBytesInput(InputStream in, int byteCount) { @Override public void writeAllTo(OutputStream out) throws IOException { - if (DEBUG) LOG.debug("write All "+ byteCount + " bytes"); + LOGGER.debug("write All {} bytes", byteCount); // TODO: more efficient out.write(this.toByteArray()); } public byte[] toByteArray() throws IOException { - if (DEBUG) LOG.debug("read all "+ byteCount + " bytes"); + LOGGER.debug("read all {} bytes", byteCount); byte[] buf = new byte[byteCount]; new DataInputStream(in).readFully(buf); return buf; @@ -243,7 +244,7 @@ public long size() { } private static class SequenceBytesIn extends BytesInput { - private static final Log LOG = Log.getLog(BytesInput.SequenceBytesIn.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BytesInput.SequenceBytesIn.class); private final List inputs; private final long size; @@ -261,10 +262,15 @@ private SequenceBytesIn(List inputs) { @Override public void writeAllTo(OutputStream out) throws IOException { for (BytesInput input : inputs) { - if (DEBUG) LOG.debug("write " + input.size() + " bytes to out"); - if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("{"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("write {} bytes to out", input.size()); + if (LOGGER.isDebugEnabled() && input instanceof SequenceBytesIn) { + LOGGER.debug("{"); + } input.writeAllTo(out); - if (DEBUG && input instanceof SequenceBytesIn) LOG.debug("}"); + if (LOGGER.isDebugEnabled() && input instanceof SequenceBytesIn) { + LOGGER.debug("}"); + } } } diff --git a/parquet-encoding/src/main/java/org/apache/parquet/bytes/CapacityByteArrayOutputStream.java b/parquet-encoding/src/main/java/org/apache/parquet/bytes/CapacityByteArrayOutputStream.java index 61555653d6..7591cda6bf 100644 --- a/parquet-encoding/src/main/java/org/apache/parquet/bytes/CapacityByteArrayOutputStream.java +++ b/parquet-encoding/src/main/java/org/apache/parquet/bytes/CapacityByteArrayOutputStream.java @@ -30,9 +30,11 @@ import java.util.ArrayList; import java.util.List; -import org.apache.parquet.Log; import org.apache.parquet.OutputStreamCloseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Similar to a {@link ByteArrayOutputStream}, but uses a different strategy for growing that does not involve copying. * Where ByteArrayOutputStream is backed by a single array that "grows" by copying into a new larger array, this output @@ -54,7 +56,7 @@ * */ public class CapacityByteArrayOutputStream extends OutputStream { - private static final Log LOG = Log.getLog(CapacityByteArrayOutputStream.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CapacityByteArrayOutputStream.class); private static final ByteBuffer EMPTY_SLAB = ByteBuffer.wrap(new byte[0]); private int initialSlabSize; @@ -167,11 +169,17 @@ private void addSlab(int minimumSize) { } if (nextSlabSize < minimumSize) { - if (Log.DEBUG) LOG.debug(format("slab size %,d too small for value of size %,d. Bumping up slab size", nextSlabSize, minimumSize)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("slab size %,d too small for value of size %,d. Bumping up slab size", + nextSlabSize, minimumSize)); + } + nextSlabSize = minimumSize; } - if (Log.DEBUG) LOG.debug(format("used %d slabs, adding new slab of size %d", slabs.size(), nextSlabSize)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("used %d slabs, adding new slab of size %d", slabs.size(), nextSlabSize)); + } this.currentSlab = allocator.allocate(nextSlabSize); this.slabs.add(currentSlab); @@ -265,7 +273,9 @@ public void reset() { // readjust slab size. // 7 = 2^3 - 1 so that doubling the initial size 3 times will get to the same size this.initialSlabSize = max(bytesUsed / 7, initialSlabSize); - if (Log.DEBUG) LOG.debug(String.format("initial slab of size %d", initialSlabSize)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("initial slab of size %d", initialSlabSize)); + } for (ByteBuffer slab : slabs) { allocator.release(slab); } diff --git a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java index 448c0bec40..6c27934df0 100644 --- a/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java +++ b/parquet-encoding/src/main/java/org/apache/parquet/column/values/bitpacking/ByteBasedBitPackingEncoder.java @@ -18,17 +18,18 @@ */ package org.apache.parquet.column.values.bitpacking; -import static org.apache.parquet.Log.DEBUG; import static org.apache.parquet.bytes.BytesInput.concat; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.BytesUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Uses the generated Byte based bit packing to write ints into a BytesInput * @@ -36,7 +37,7 @@ * */ public class ByteBasedBitPackingEncoder { - private static final Log LOG = Log.getLog(ByteBasedBitPackingEncoder.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ByteBasedBitPackingEncoder.class); private static final int VALUES_WRITTEN_AT_A_TIME = 8; @@ -99,7 +100,9 @@ private void initPackedSlab() { public BytesInput toBytes() throws IOException { int packedByteLength = packedPosition + BytesUtils.paddedByteCountFromBits(inputSize * bitWidth); - if (DEBUG) LOG.debug("writing " + (slabs.size() * slabSize + packedByteLength) + " bytes"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("writing {} bytes", slabs.size() * slabSize + packedByteLength); + if (inputSize > 0) { for (int i = inputSize; i < input.length; i++) { input[i] = 0; From 5df4d1619fdce7c3a4a96f733cc6c3fc2b567ea1 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 18:23:29 +0800 Subject: [PATCH 05/52] 16~20 --- .../FixedLenByteArrayPlainValuesReader.java | 9 ++++---- .../FixedLenByteArrayPlainValuesWriter.java | 10 ++++++--- .../hadoop/ColumnChunkPageReadStore.java | 6 +++-- .../hadoop/ColumnChunkPageWriteStore.java | 11 +++++----- .../parquet/hadoop/codec/CodecConfig.java | 22 ++++++++++--------- 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java index 8496e7e4c9..5e47aa3bf6 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesReader.java @@ -20,12 +20,12 @@ import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.parquet.Log; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.api.Binary; -import static org.apache.parquet.Log.DEBUG; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * ValuesReader for FIXED_LEN_BYTE_ARRAY. @@ -33,7 +33,7 @@ * @author David Z. Chen */ public class FixedLenByteArrayPlainValuesReader extends ValuesReader { - private static final Log LOG = Log.getLog(FixedLenByteArrayPlainValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(FixedLenByteArrayPlainValuesReader.class); private ByteBuffer in; private int offset; private int length; @@ -61,7 +61,8 @@ public void skip() { @Override public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { - if (DEBUG) LOG.debug("init from page at offset "+ offset + " for length " + (in.limit() - offset)); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("init from page at offset "+ offset + " for length " + (in.limit() - offset)); this.in = in; this.offset = offset; } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesWriter.java index 6ab2dea4b1..fe011c0f72 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/plain/FixedLenByteArrayPlainValuesWriter.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.parquet.bytes.ByteBufferAllocator; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; import org.apache.parquet.bytes.LittleEndianDataOutputStream; @@ -30,13 +29,16 @@ import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * ValuesWriter for FIXED_LEN_BYTE_ARRAY. * * @author David Z. Chen */ public class FixedLenByteArrayPlainValuesWriter extends ValuesWriter { - private static final Log LOG = Log.getLog(PlainValuesWriter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PlainValuesWriter.class); private CapacityByteArrayOutputStream arrayOut; private LittleEndianDataOutputStream out; @@ -76,7 +78,9 @@ public BytesInput getBytes() { } catch (IOException e) { throw new ParquetEncodingException("could not write page", e); } - if (Log.DEBUG) LOG.debug("writing a buffer of size " + arrayOut.size()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("writing a buffer of size " + arrayOut.size()); + } return BytesInput.from(arrayOut); } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index af067474f8..00589e7a95 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -25,7 +25,6 @@ import java.util.Map; import org.apache.parquet.Ints; -import org.apache.parquet.Log; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.DataPage; import org.apache.parquet.column.page.DataPageV1; @@ -36,6 +35,9 @@ import org.apache.parquet.hadoop.CodecFactory.BytesDecompressor; import org.apache.parquet.io.ParquetDecodingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * TODO: should this actually be called RowGroupImpl or something? * The name is kind of confusing since it references three different "entities" @@ -43,7 +45,7 @@ * */ class ColumnChunkPageReadStore implements PageReadStore { - private static final Log LOG = Log.getLog(ColumnChunkPageReadStore.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ColumnChunkPageReadStore.class); /** * PageReader for a single column chunk. A column chunk contains diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java index 2eab54a1a0..bb07739f02 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java @@ -18,7 +18,6 @@ */ package org.apache.parquet.hadoop; -import static org.apache.parquet.Log.INFO; import static org.apache.parquet.column.statistics.Statistics.getStatsBasedOnType; import java.io.ByteArrayOutputStream; @@ -29,7 +28,6 @@ import java.util.Map; import java.util.Set; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.ConcatenatingByteArrayCollector; import org.apache.parquet.column.ColumnDescriptor; @@ -44,8 +42,11 @@ import org.apache.parquet.schema.MessageType; import org.apache.parquet.bytes.ByteBufferAllocator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + class ColumnChunkPageWriteStore implements PageWriteStore { - private static final Log LOG = Log.getLog(ColumnChunkPageWriteStore.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ColumnChunkPageWriteStore.class); private static ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); @@ -186,8 +187,8 @@ public void writeToFileWriter(ParquetFileWriter writer) throws IOException { } writer.writeDataPages(buf, uncompressedLength, compressedLength, totalStatistics, new ArrayList(encodings)); writer.endColumn(); - if (INFO) { - LOG.info( + if (LOGGER.isInfoEnabled()) { + LOGGER.info( String.format( "written %,dB for %s: %,d values, %,dB raw, %,dB comp, %d pages, encodings: %s", buf.size(), path, totalValueCount, uncompressedLength, compressedLength, pageCount, encodings) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CodecConfig.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CodecConfig.java index 9657865ce0..23ba2b6b6c 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CodecConfig.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CodecConfig.java @@ -22,15 +22,15 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.parquet.Log; import org.apache.parquet.hadoop.ParquetOutputFormat; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.hadoop.util.ContextUtil; -import static org.apache.parquet.Log.INFO; -import static org.apache.parquet.Log.WARN; import static org.apache.parquet.hadoop.metadata.CompressionCodecName.UNCOMPRESSED; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Template class and factory for accessing codec related configurations in different APIs(mapreduce or mapred), * use {@link #from(org.apache.hadoop.mapred.JobConf)} for mapred API, @@ -39,7 +39,7 @@ * @author Tianshuo Deng */ public abstract class CodecConfig { - private static final Log LOG = Log.getLog(CodecConfig.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CodecConfig.class); /** * @return if a compress flag is set from hadoop @@ -89,11 +89,11 @@ public CompressionCodecName getCodec() { } else if (isHadoopCompressionSet()) { // from hadoop config codec = getHadoopCompressionCodec(); } else { - if (INFO) LOG.info("Compression set to false"); + LOGGER.info("Compression set to false"); codec = CompressionCodecName.UNCOMPRESSED; } - if (INFO) LOG.info("Compression: " + codec.name()); + LOGGER.info("Compression: {}", codec.name()); return codec; } @@ -102,14 +102,16 @@ private CompressionCodecName getHadoopCompressionCodec() { try { // find the right codec Class codecClass = getHadoopOutputCompressorClass(CompressionCodecName.UNCOMPRESSED.getHadoopCompressionCodecClass()); - if (INFO) LOG.info("Compression set through hadoop codec: " + codecClass.getName()); + LOGGER.info("Compression set through hadoop codec: {}", codecClass.getName()); codec = CompressionCodecName.fromCompressionCodec(codecClass); } catch (CompressionCodecNotSupportedException e) { - if (WARN) - LOG.warn("codec defined in hadoop config is not supported by parquet [" + e.getCodecClass().getName() + "] and will use UNCOMPRESSED", e); + if (LOGGER.isWarnEnabled()) { + LOGGER.warn("codec defined in hadoop config is not supported by parquet [" + e.getCodecClass().getName() + "] and will use UNCOMPRESSED", e); + } codec = CompressionCodecName.UNCOMPRESSED; } catch (IllegalArgumentException e) { - if (WARN) LOG.warn("codec class not found: " + e.getMessage(), e); + if (LOGGER.isWarnEnabled()) + LOGGER.warn("codec class not found: " + e.getMessage(), e); codec = CompressionCodecName.UNCOMPRESSED; } return codec; From 1430dae36178e54bfcd687bb84c0c629ad163ed6 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 18:31:17 +0800 Subject: [PATCH 06/52] 21~25 --- .../parquet/CorruptDeltaByteArrays.java | 13 +++--- .../org/apache/parquet/CorruptStatistics.java | 9 ++-- .../parquet/column/impl/ColumnWriterV1.java | 41 +++++++++++++------ .../parquet/column/impl/ColumnWriterV2.java | 41 +++++++++++++------ .../DeltaLengthByteArrayValuesReader.java | 11 ++--- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/CorruptDeltaByteArrays.java b/parquet-column/src/main/java/org/apache/parquet/CorruptDeltaByteArrays.java index 258c9ee1fd..c17454f8df 100644 --- a/parquet-column/src/main/java/org/apache/parquet/CorruptDeltaByteArrays.java +++ b/parquet-column/src/main/java/org/apache/parquet/CorruptDeltaByteArrays.java @@ -21,8 +21,11 @@ import org.apache.parquet.VersionParser.ParsedVersion; import org.apache.parquet.column.Encoding; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class CorruptDeltaByteArrays { - private static final Log LOG = Log.getLog(CorruptStatistics.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CorruptStatistics.class); private static final SemanticVersion PARQUET_246_FIXED_VERSION = new SemanticVersion(1, 8, 0); @@ -42,7 +45,7 @@ public static boolean requiresSequentialReads(ParsedVersion version, Encoding en } if (!version.hasSemanticVersion()) { - LOG.warn("Requiring sequential reads because created_by did not " + + LOGGER.warn("Requiring sequential reads because created_by did not " + "contain a valid version (see PARQUET-246): " + version.version); return true; } @@ -60,7 +63,7 @@ public static boolean requiresSequentialReads(SemanticVersion semver, Encoding e } if (semver.compareTo(PARQUET_246_FIXED_VERSION) < 0) { - LOG.info("Requiring sequential reads because this file was created " + + LOGGER.info("Requiring sequential reads because this file was created " + "prior to " + PARQUET_246_FIXED_VERSION + ". See PARQUET-246" ); return true; } @@ -75,7 +78,7 @@ public static boolean requiresSequentialReads(String createdBy, Encoding encodin } if (Strings.isNullOrEmpty(createdBy)) { - LOG.info("Requiring sequential reads because file version is empty. " + + LOGGER.info("Requiring sequential reads because file version is empty. " + "See PARQUET-246"); return true; } @@ -93,7 +96,7 @@ public static boolean requiresSequentialReads(String createdBy, Encoding encodin } private static void warnParseError(String createdBy, Throwable e) { - LOG.warn("Requiring sequential reads because created_by could not be " + + LOGGER.warn("Requiring sequential reads because created_by could not be " + "parsed (see PARQUET-246): " + createdBy, e); } } diff --git a/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java b/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java index 3b9033850e..6011f01820 100644 --- a/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java +++ b/parquet-column/src/main/java/org/apache/parquet/CorruptStatistics.java @@ -25,6 +25,9 @@ import org.apache.parquet.VersionParser.VersionParseException; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * There was a bug (PARQUET-251) that caused the statistics metadata * for binary columns to be corrupted in the write path. @@ -35,7 +38,7 @@ public class CorruptStatistics { private static final AtomicBoolean alreadyLogged = new AtomicBoolean(false); - private static final Log LOG = Log.getLog(CorruptStatistics.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CorruptStatistics.class); // the version in which the bug described by jira: PARQUET-251 was fixed // the bug involved writing invalid binary statistics, so stats written prior to this @@ -108,13 +111,13 @@ public static boolean shouldIgnoreStatistics(String createdBy, PrimitiveTypeName private static void warnParseErrorOnce(String createdBy, Throwable e) { if(!alreadyLogged.getAndSet(true)) { - LOG.warn("Ignoring statistics because created_by could not be parsed (see PARQUET-251): " + createdBy, e); + LOGGER.warn("Ignoring statistics because created_by could not be parsed (see PARQUET-251): " + createdBy, e); } } private static void warnOnce(String message) { if(!alreadyLogged.getAndSet(true)) { - LOG.warn(message); + LOGGER.warn(message); } } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java index dc6ebecb5a..96fcd5eeea 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV1.java @@ -22,7 +22,6 @@ import java.io.IOException; -import org.apache.parquet.Log; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ColumnWriter; import org.apache.parquet.column.ParquetProperties; @@ -35,6 +34,9 @@ import static java.lang.Math.max; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Writes (repetition level, definition level, value) triplets and deals with writing pages to the underlying layer. * @@ -42,8 +44,7 @@ * */ final class ColumnWriterV1 implements ColumnWriter { - private static final Log LOG = Log.getLog(ColumnWriterV1.class); - private static final boolean DEBUG = Log.DEBUG; + private static final Logger LOGGER = LoggerFactory.getLogger(ColumnWriterV1.class); private final ColumnDescriptor path; private final PageWriter pageWriter; @@ -74,7 +75,7 @@ public ColumnWriterV1(ColumnDescriptor path, PageWriter pageWriter, } private void log(Object value, int r, int d) { - LOG.debug(path + " " + value + " r:" + r + " d:" + d); + LOGGER.debug(path + " " + value + " r:" + r + " d:" + d); } private void resetStatistics() { @@ -143,7 +144,7 @@ private void updateStatistics(boolean value) { } private void writePage() { - if (DEBUG) LOG.debug("write page"); + LOGGER.debug("write page"); try { pageWriter.writePage( concat(repetitionLevelColumn.getBytes(), definitionLevelColumn.getBytes(), dataColumn.getBytes()), @@ -164,7 +165,9 @@ private void writePage() { @Override public void writeNull(int repetitionLevel, int definitionLevel) { - if (DEBUG) log(null, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(null, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); updateStatisticsNumNulls(); @@ -173,7 +176,9 @@ public void writeNull(int repetitionLevel, int definitionLevel) { @Override public void write(double value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeDouble(value); @@ -183,7 +188,9 @@ public void write(double value, int repetitionLevel, int definitionLevel) { @Override public void write(float value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeFloat(value); @@ -193,7 +200,9 @@ public void write(float value, int repetitionLevel, int definitionLevel) { @Override public void write(Binary value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeBytes(value); @@ -203,7 +212,9 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) { @Override public void write(boolean value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeBoolean(value); @@ -213,7 +224,9 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) { @Override public void write(int value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeInteger(value); @@ -223,7 +236,9 @@ public void write(int value, int repetitionLevel, int definitionLevel) { @Override public void write(long value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevelColumn.writeInteger(repetitionLevel); definitionLevelColumn.writeInteger(definitionLevel); dataColumn.writeLong(value); @@ -237,7 +252,7 @@ public void flush() { } final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose(); if (dictionaryPage != null) { - if (DEBUG) LOG.debug("write dictionary"); + LOGGER.debug("write dictionary"); try { pageWriter.writeDictionaryPage(dictionaryPage); } catch (IOException e) { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java index 396d53a1a5..10259f17c6 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/impl/ColumnWriterV2.java @@ -24,7 +24,6 @@ import java.io.IOException; import org.apache.parquet.Ints; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; import org.apache.parquet.column.ColumnDescriptor; @@ -39,6 +38,9 @@ import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Writes (repetition level, definition level, value) triplets and deals with writing pages to the underlying layer. * @@ -46,8 +48,7 @@ * */ final class ColumnWriterV2 implements ColumnWriter { - private static final Log LOG = Log.getLog(ColumnWriterV2.class); - private static final boolean DEBUG = Log.DEBUG; + private static final Logger LOGGER = LoggerFactory.getLogger(ColumnWriterV2.class); private final ColumnDescriptor path; private final PageWriter pageWriter; @@ -73,7 +74,7 @@ public ColumnWriterV2( } private void log(Object value, int r, int d) { - LOG.debug(path + " " + value + " r:" + r + " d:" + d); + LOGGER.debug(path + " " + value + " r:" + r + " d:" + d); } private void resetStatistics() { @@ -102,7 +103,9 @@ private void repetitionLevel(int repetitionLevel) { * @param definitionLevel */ public void writeNull(int repetitionLevel, int definitionLevel) { - if (DEBUG) log(null, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(null, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); statistics.incrementNumNulls(); @@ -132,7 +135,9 @@ public long getBufferedSizeInMemory() { * @param definitionLevel */ public void write(double value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeDouble(value); @@ -147,7 +152,9 @@ public void write(double value, int repetitionLevel, int definitionLevel) { * @param definitionLevel */ public void write(float value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeFloat(value); @@ -162,7 +169,9 @@ public void write(float value, int repetitionLevel, int definitionLevel) { * @param definitionLevel */ public void write(Binary value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeBytes(value); @@ -177,7 +186,9 @@ public void write(Binary value, int repetitionLevel, int definitionLevel) { * @param definitionLevel */ public void write(boolean value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeBoolean(value); @@ -192,7 +203,9 @@ public void write(boolean value, int repetitionLevel, int definitionLevel) { * @param definitionLevel */ public void write(int value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeInteger(value); @@ -207,7 +220,9 @@ public void write(int value, int repetitionLevel, int definitionLevel) { * @param definitionLevel */ public void write(long value, int repetitionLevel, int definitionLevel) { - if (DEBUG) log(value, repetitionLevel, definitionLevel); + if (LOGGER.isDebugEnabled()) { + log(value, repetitionLevel, definitionLevel); + } repetitionLevel(repetitionLevel); definitionLevel(definitionLevel); dataColumn.writeLong(value); @@ -222,7 +237,7 @@ public void write(long value, int repetitionLevel, int definitionLevel) { public void finalizeColumnChunk() { final DictionaryPage dictionaryPage = dataColumn.toDictPageAndClose(); if (dictionaryPage != null) { - if (DEBUG) LOG.debug("write dictionary"); + LOGGER.debug("write dictionary"); try { pageWriter.writeDictionaryPage(dictionaryPage); } catch (IOException e) { @@ -289,7 +304,7 @@ public long getRowsWrittenSoFar() { public void writePage(long rowCount) { int pageRowCount = Ints.checkedCast(rowCount - rowsWrittenSoFar); this.rowsWrittenSoFar = rowCount; - if (DEBUG) LOG.debug("write page"); + LOGGER.debug("write page"); try { // TODO: rework this API. Those must be called *in that order* BytesInput bytes = dataColumn.getBytes(); diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java index 41f221d85a..9f7a56163b 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesReader.java @@ -18,16 +18,15 @@ */ package org.apache.parquet.column.values.deltalengthbytearray; -import static org.apache.parquet.Log.DEBUG; - import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.parquet.Log; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesReader; import org.apache.parquet.io.api.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Reads binary data written by {@link DeltaLengthByteArrayValuesWriter} * @@ -36,7 +35,7 @@ */ public class DeltaLengthByteArrayValuesReader extends ValuesReader { - private static final Log LOG = Log.getLog(DeltaLengthByteArrayValuesReader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DeltaLengthByteArrayValuesReader.class); private ValuesReader lengthReader; private ByteBuffer in; private int offset; @@ -48,7 +47,9 @@ public DeltaLengthByteArrayValuesReader() { @Override public void initFromPage(int valueCount, ByteBuffer in, int offset) throws IOException { - if (DEBUG) LOG.debug("init from page at offset "+ offset + " for length " + (in.limit() - offset)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("init from page at offset {} for length {}", offset, (in.limit() - offset)); + } lengthReader.initFromPage(valueCount, in, offset); offset = lengthReader.getNextOffset(); this.in = in; From d14b3f8f11642174914da018e6e2e5d1cbe09ee3 Mon Sep 17 00:00:00 2001 From: Liwei Lin Date: Tue, 2 Feb 2016 18:39:26 +0800 Subject: [PATCH 07/52] 26~30 --- .../DeltaLengthByteArrayValuesWriter.java | 10 +++++++--- .../dictionary/DictionaryValuesReader.java | 15 ++++++++------- .../dictionary/DictionaryValuesWriter.java | 13 ++++++++----- .../parquet/filter2/compat/FilterCompat.java | 10 ++++++---- .../parquet/hadoop/DirectCodecFactory.java | 18 ++++++++++-------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesWriter.java index 2d6b21389c..836a01b749 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesWriter.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/deltalengthbytearray/DeltaLengthByteArrayValuesWriter.java @@ -21,7 +21,6 @@ import java.io.IOException; import org.apache.parquet.bytes.ByteBufferAllocator; -import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; import org.apache.parquet.bytes.LittleEndianDataOutputStream; @@ -31,6 +30,9 @@ import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.io.api.Binary; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Write lengths of byte-arrays using delta encoding, followed by concatenated byte-arrays * 
@@ -43,7 +45,7 @@
  */
 public class DeltaLengthByteArrayValuesWriter extends ValuesWriter {
 
-  private static final Log LOG = Log.getLog(DeltaLengthByteArrayValuesWriter.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(DeltaLengthByteArrayValuesWriter.class);
 
   private ValuesWriter lengthWriter;
   private CapacityByteArrayOutputStream arrayOut;
@@ -80,7 +82,9 @@ public BytesInput getBytes() {
     } catch (IOException e) {
       throw new ParquetEncodingException("could not write page", e);
     }
-    if (Log.DEBUG) LOG.debug("writing a buffer of size " + arrayOut.size());
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug("writing a buffer of size {}", arrayOut.size());
+    }
     return BytesInput.concat(lengthWriter.getBytes(), BytesInput.from(arrayOut));
   }
 
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesReader.java b/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesReader.java
index e421da99ff..98511a14e0 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesReader.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesReader.java
@@ -18,13 +18,10 @@
  */
 package org.apache.parquet.column.values.dictionary;
 
-import static org.apache.parquet.Log.DEBUG;
-
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.parquet.bytes.ByteBufferInputStream;
-import org.apache.parquet.Log;
 import org.apache.parquet.bytes.BytesUtils;
 import org.apache.parquet.column.Dictionary;
 import org.apache.parquet.column.values.ValuesReader;
@@ -32,6 +29,9 @@
 import org.apache.parquet.io.ParquetDecodingException;
 import org.apache.parquet.io.api.Binary;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Reads values that have been dictionary encoded
  *
@@ -39,7 +39,7 @@
  *
  */
 public class DictionaryValuesReader extends ValuesReader {
-  private static final Log LOG = Log.getLog(DictionaryValuesReader.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(DictionaryValuesReader.class);
 
   private ByteBufferInputStream in;
 
@@ -56,10 +56,11 @@ public void initFromPage(int valueCount, ByteBuffer page, int offset)
       throws IOException {
     this.in = new ByteBufferInputStream(page, offset, page.limit() - offset);
     if (page.limit() - offset > 0) {
-      if (DEBUG)
-        LOG.debug("init from page at offset " + offset + " for length " + (page.limit() - offset));
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("init from page at offset {} for length {}", offset, (page.limit() - offset));
+      }
       int bitWidth = BytesUtils.readIntLittleEndianOnOneByte(in);
-      if (DEBUG) LOG.debug("bit width " + bitWidth);
+      LOGGER.debug("bit width {}", bitWidth);
       decoder = new RunLengthBitPackingHybridDecoder(bitWidth, in);
     } else {
       decoder = new RunLengthBitPackingHybridDecoder(1, in) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java
index 86edd79b13..d5cfc5b088 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java
@@ -18,7 +18,6 @@
  */
 package org.apache.parquet.column.values.dictionary;
 
-import static org.apache.parquet.Log.DEBUG;
 import static org.apache.parquet.bytes.BytesInput.concat;
 import it.unimi.dsi.fastutil.doubles.Double2IntLinkedOpenHashMap;
 import it.unimi.dsi.fastutil.doubles.Double2IntMap;
@@ -41,7 +40,6 @@
 import java.util.List;
 
 import org.apache.parquet.bytes.ByteBufferAllocator;
-import org.apache.parquet.Log;
 import org.apache.parquet.bytes.BytesInput;
 import org.apache.parquet.bytes.BytesUtils;
 import org.apache.parquet.bytes.CapacityByteArrayOutputStream;
@@ -56,6 +54,9 @@
 import org.apache.parquet.io.ParquetEncodingException;
 import org.apache.parquet.io.api.Binary;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Will attempt to encode values using a dictionary and fall back to plain encoding
  *  if the dictionary gets too big
@@ -64,7 +65,7 @@
  *
  */
 public abstract class DictionaryValuesWriter extends ValuesWriter implements RequiresFallback {
-  private static final Log LOG = Log.getLog(DictionaryValuesWriter.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(DictionaryValuesWriter.class);
 
   /* max entries allowed for the dictionary will fail over to plain encoding if reached */
   private static final int MAX_DICTIONARY_ENTRIES = Integer.MAX_VALUE - 1;
@@ -158,7 +159,7 @@ public long getAllocatedSize() {
   @Override
   public BytesInput getBytes() {
     int maxDicId = getDictionarySize() - 1;
-    if (DEBUG) LOG.debug("max dic id " + maxDicId);
+    LOGGER.debug("max dic id {}", maxDicId);
     int bitWidth = BytesUtils.getWidthFromMaxInt(maxDicId);
     int initialSlabSize =
         CapacityByteArrayOutputStream.initialSlabSizeHeuristic(MIN_INITIAL_SLAB_SIZE, maxDictionaryByteSize, 10);
@@ -174,7 +175,9 @@ public BytesInput getBytes() {
       // encodes the bit width
       byte[] bytesHeader = new byte[] { (byte) bitWidth };
       BytesInput rleEncodedBytes = encoder.toBytes();
-      if (DEBUG) LOG.debug("rle encoded bytes " + rleEncodedBytes.size());
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("rle encoded bytes {}", rleEncodedBytes.size());
+      }
       BytesInput bytes = concat(BytesInput.from(bytesHeader), rleEncodedBytes);
       // remember size of dictionary when we last wrote a page
       lastUsedDictionarySize = getDictionarySize();
diff --git a/parquet-column/src/main/java/org/apache/parquet/filter2/compat/FilterCompat.java b/parquet-column/src/main/java/org/apache/parquet/filter2/compat/FilterCompat.java
index 2efcc390af..8fd85af49f 100644
--- a/parquet-column/src/main/java/org/apache/parquet/filter2/compat/FilterCompat.java
+++ b/parquet-column/src/main/java/org/apache/parquet/filter2/compat/FilterCompat.java
@@ -18,7 +18,6 @@
  */
 package org.apache.parquet.filter2.compat;
 
-import org.apache.parquet.Log;
 import org.apache.parquet.filter.UnboundRecordFilter;
 import org.apache.parquet.filter2.predicate.FilterPredicate;
 import org.apache.parquet.filter2.predicate.LogicalInverseRewriter;
@@ -26,6 +25,9 @@
 import static org.apache.parquet.Preconditions.checkArgument;
 import static org.apache.parquet.Preconditions.checkNotNull;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Parquet currently has two ways to specify a filter for dropping records at read time.
  * The first way, that only supports filtering records during record assembly, is found
@@ -40,7 +42,7 @@
  * codebase.
  */
 public class FilterCompat {
-  private static final Log LOG = Log.getLog(FilterCompat.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(FilterCompat.class);
 
   /**
    * Anyone wanting to use a {@link Filter} need only implement this interface,
@@ -67,13 +69,13 @@ public static interface Filter {
   public static Filter get(FilterPredicate filterPredicate) {
     checkNotNull(filterPredicate, "filterPredicate");
 
-    LOG.info("Filtering using predicate: " + filterPredicate);
+    LOGGER.info("Filtering using predicate: {}", filterPredicate);
 
     // rewrite the predicate to not include the not() operator
     FilterPredicate collapsedPredicate = LogicalInverseRewriter.rewrite(filterPredicate);
 
     if (!filterPredicate.equals(collapsedPredicate)) {
-      LOG.info("Predicate has been collapsed to: " + collapsedPredicate);
+      LOGGER.info("Predicate has been collapsed to: {}", collapsedPredicate);
     }
 
     return new FilterPredicateCompat(collapsedPredicate);
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/DirectCodecFactory.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/DirectCodecFactory.java
index bb711daa29..4dd0c28d25 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/DirectCodecFactory.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/DirectCodecFactory.java
@@ -38,16 +38,18 @@
 import org.apache.parquet.bytes.ByteBufferAllocator;
 import org.apache.parquet.bytes.BytesInput;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.apache.parquet.Log;
 import org.apache.parquet.ParquetRuntimeException;
 import org.apache.parquet.Preconditions;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Factory to produce compressors and decompressors that operate on java
  * direct memory, without requiring a copy into heap memory (where possible).
  */
 class DirectCodecFactory extends CodecFactory implements AutoCloseable {
-  private static final Log LOG = Log.getLog(DirectCodecFactory.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(DirectCodecFactory.class);
 
   private final ByteBufferAllocator allocator;
 
@@ -372,8 +374,8 @@ public Object makeObject() throws Exception {
             cPools.put(com.getClass(), compressorPool);
             compressorPool.returnObject(com);
           } else {
-            if (Log.DEBUG) {
-              LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor", codec.getClass().getName()));
+            if (LOGGER.isDebugEnabled()) {
+              LOGGER.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor", codec.getClass().getName()));
             }
           }
 
@@ -388,8 +390,8 @@ public Object makeObject() throws Exception {
             dePools.put(decom.getClass(), decompressorPool);
             decompressorPool.returnObject(decom);
           } else {
-            if (Log.DEBUG) {
-              LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "decompressor" + codec.getClass().getName()));
+            if (LOGGER.isDebugEnabled()) {
+              LOGGER.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "decompressor" + codec.getClass().getName()));
             }
           }
 
@@ -408,8 +410,8 @@ public Object makeObject() throws Exception {
 
             } else {
               supportDirectDecompressor = false;
-              if (Log.DEBUG) {
-                LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor" + codec.getClass().getName()));
+              if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor" + codec.getClass().getName()));
               }
             }
 

From 0116fc293403c67b1fdcec156c8155a87428a9cf Mon Sep 17 00:00:00 2001
From: Liwei Lin 
Date: Tue, 2 Feb 2016 18:49:43 +0800
Subject: [PATCH 08/52] 31~35

---
 .../apache/parquet/example/data/Group.java    | 10 ++-
 .../org/apache/parquet/io/GroupColumnIO.java  |  6 +-
 .../parquet/hadoop/ParquetFileWriter.java     | 76 +++++++++++++------
 .../parquet/hadoop/ParquetInputFormat.java    | 30 ++++----
 .../parquet/hive/internal/Hive010Binding.java | 13 ++--
 5 files changed, 88 insertions(+), 47 deletions(-)

diff --git a/parquet-column/src/main/java/org/apache/parquet/example/data/Group.java b/parquet-column/src/main/java/org/apache/parquet/example/data/Group.java
index 3fb7d4d152..4b1a056180 100644
--- a/parquet-column/src/main/java/org/apache/parquet/example/data/Group.java
+++ b/parquet-column/src/main/java/org/apache/parquet/example/data/Group.java
@@ -23,9 +23,11 @@
 import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.io.api.RecordConsumer;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 abstract public class Group extends GroupValueSource {
-  private static final Log logger = Log.getLog(Group.class);
-  private static final boolean DEBUG = Log.DEBUG;
+  private static final Logger LOGGER = LoggerFactory.getLogger(Group.class);
 
   public void add(String field, int value) {
     add(getType().getFieldIndex(field), value);
@@ -64,7 +66,9 @@ public void add(String field, Group value) {
   }
 
   public Group addGroup(String field) {
-    if (DEBUG) logger.debug("add group "+field+" to "+getType().getName());
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug("add group " + field + " to " + getType().getName());
+    }
     return addGroup(getType().getFieldIndex(field));
   }
 
diff --git a/parquet-column/src/main/java/org/apache/parquet/io/GroupColumnIO.java b/parquet-column/src/main/java/org/apache/parquet/io/GroupColumnIO.java
index 1efe0d1414..70c2b0647b 100644
--- a/parquet-column/src/main/java/org/apache/parquet/io/GroupColumnIO.java
+++ b/parquet-column/src/main/java/org/apache/parquet/io/GroupColumnIO.java
@@ -27,9 +27,11 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.parquet.Log;
 import org.apache.parquet.schema.GroupType;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Group level of the IO structure
  *
@@ -38,7 +40,7 @@
  *
  */
 public class GroupColumnIO extends ColumnIO {
-  private static final Log LOG = Log.getLog(GroupColumnIO.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(GroupColumnIO.class);
 
   private final Map childrenByName = new HashMap();
   private final List children = new ArrayList();
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
index 442d3f235b..56e8ccb0c6 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
@@ -18,7 +18,6 @@
  */
 package org.apache.parquet.hadoop;
 
-import static org.apache.parquet.Log.DEBUG;
 import static org.apache.parquet.format.Util.writeFileMetaData;
 import static org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE;
 import static org.apache.parquet.hadoop.ParquetWriter.MAX_PADDING_SIZE_DEFAULT;
@@ -41,7 +40,6 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import org.apache.parquet.Log;
 import org.apache.parquet.Preconditions;
 import org.apache.parquet.Strings;
 import org.apache.parquet.Version;
@@ -65,6 +63,9 @@
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
 import org.apache.parquet.schema.TypeUtil;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Internal implementation of the Parquet file writer as a block container
  *
@@ -72,7 +73,7 @@
  *
  */
 public class ParquetFileWriter {
-  private static final Log LOG = Log.getLog(ParquetFileWriter.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(ParquetFileWriter.class);
 
   private static ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter();
 
@@ -267,7 +268,9 @@ public ParquetFileWriter(Configuration configuration, MessageType schema,
    */
   public void start() throws IOException {
     state = state.start();
-    if (DEBUG) LOG.debug(out.getPos() + ": start");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": start");
+    }
     out.write(MAGIC);
   }
 
@@ -278,7 +281,9 @@ public void start() throws IOException {
    */
   public void startBlock(long recordCount) throws IOException {
     state = state.startBlock();
-    if (DEBUG) LOG.debug(out.getPos() + ": start block");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": start block");
+    }
 //    out.write(MAGIC); // TODO: add a magic delimiter
 
     alignment.alignForRowGroup(out);
@@ -317,7 +322,9 @@ public void startColumn(ColumnDescriptor descriptor,
    */
   public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOException {
     state = state.write();
-    if (DEBUG) LOG.debug(out.getPos() + ": write dictionary page: " + dictionaryPage.getDictionarySize() + " values");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": write dictionary page: " + dictionaryPage.getDictionarySize() + " values");
+    }
     currentChunkDictionaryPageOffset = out.getPos();
     int uncompressedSize = dictionaryPage.getUncompressedSize();
     int compressedPageSize = (int)dictionaryPage.getBytes().size(); // TODO: fix casts
@@ -330,7 +337,9 @@ public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOExceptio
     long headerSize = out.getPos() - currentChunkDictionaryPageOffset;
     this.uncompressedLength += uncompressedSize + headerSize;
     this.compressedLength += compressedPageSize + headerSize;
-    if (DEBUG) LOG.debug(out.getPos() + ": write dictionary page content " + compressedPageSize);
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": write dictionary page content " + compressedPageSize);
+    }
     dictionaryPage.getBytes().writeAllTo(out);
     currentEncodings.add(dictionaryPage.getEncoding());
   }
@@ -354,7 +363,9 @@ public void writeDataPage(
       Encoding valuesEncoding) throws IOException {
     state = state.write();
     long beforeHeader = out.getPos();
-    if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(beforeHeader + ": write data page: " + valueCount + " values");
+    }
     int compressedPageSize = (int)bytes.size();
     metadataConverter.writeDataPageHeader(
         uncompressedPageSize, compressedPageSize,
@@ -366,7 +377,9 @@ public void writeDataPage(
     long headerSize = out.getPos() - beforeHeader;
     this.uncompressedLength += uncompressedPageSize + headerSize;
     this.compressedLength += compressedPageSize + headerSize;
-    if (DEBUG) LOG.debug(out.getPos() + ": write data page content " + compressedPageSize);
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": write data page content " + compressedPageSize);
+    }
     bytes.writeAllTo(out);
     currentEncodings.add(rlEncoding);
     currentEncodings.add(dlEncoding);
@@ -391,7 +404,9 @@ public void writeDataPage(
       Encoding valuesEncoding) throws IOException {
     state = state.write();
     long beforeHeader = out.getPos();
-    if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(beforeHeader + ": write data page: " + valueCount + " values");
+    }
     int compressedPageSize = (int)bytes.size();
     metadataConverter.writeDataPageHeader(
         uncompressedPageSize, compressedPageSize,
@@ -404,7 +419,9 @@ public void writeDataPage(
     long headerSize = out.getPos() - beforeHeader;
     this.uncompressedLength += uncompressedPageSize + headerSize;
     this.compressedLength += compressedPageSize + headerSize;
-    if (DEBUG) LOG.debug(out.getPos() + ": write data page content " + compressedPageSize);
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": write data page content " + compressedPageSize);
+    }
     bytes.writeAllTo(out);
     currentStatistics.mergeStatistics(statistics);
     currentEncodings.add(rlEncoding);
@@ -425,11 +442,15 @@ void writeDataPages(BytesInput bytes,
                        Statistics totalStats,
                        List encodings) throws IOException {
     state = state.write();
-    if (DEBUG) LOG.debug(out.getPos() + ": write data pages");
+     if (LOGGER.isDebugEnabled()) {
+       LOGGER.debug(out.getPos() + ": write data pages");
+     }
     long headersSize = bytes.size() - compressedTotalPageSize;
     this.uncompressedLength += uncompressedTotalPageSize + headersSize;
     this.compressedLength += compressedTotalPageSize + headersSize;
-    if (DEBUG) LOG.debug(out.getPos() + ": write data pages content");
+     if (LOGGER.isDebugEnabled()) {
+       LOGGER.debug(out.getPos() + ": write data pages content");
+     }
     bytes.writeAllTo(out);
     currentEncodings.addAll(encodings);
     currentStatistics = totalStats;
@@ -441,7 +462,9 @@ void writeDataPages(BytesInput bytes,
    */
   public void endColumn() throws IOException {
     state = state.endColumn();
-    if (DEBUG) LOG.debug(out.getPos() + ": end column");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": end column");
+    }
     currentBlock.addColumn(ColumnChunkMetaData.get(
         currentChunkPath,
         currentChunkType,
@@ -464,7 +487,9 @@ public void endColumn() throws IOException {
    */
   public void endBlock() throws IOException {
     state = state.endBlock();
-    if (DEBUG) LOG.debug(out.getPos() + ": end block");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": end block");
+    }
     currentBlock.setRowCount(currentRecordCount);
     blocks.add(currentBlock);
     currentBlock = null;
@@ -579,8 +604,9 @@ protected byte[] initialValue() {
    */
   private static void copy(FSDataInputStream from, FSDataOutputStream to,
                           long start, long length) throws IOException{
-    if (DEBUG) LOG.debug(
-        "Copying " + length + " bytes at " + start + " to " + to.getPos());
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug("Copying " + length + " bytes at " + start + " to " + to.getPos());
+    }
     from.seek(start);
     long bytesCopied = 0;
     byte[] buffer = COPY_BUFFER.get();
@@ -605,7 +631,9 @@ private static void copy(FSDataInputStream from, FSDataOutputStream to,
    */
   public void end(Map extraMetaData) throws IOException {
     state = state.end();
-    if (DEBUG) LOG.debug(out.getPos() + ": end");
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": end");
+    }
     ParquetMetadata footer = new ParquetMetadata(new FileMetaData(schema, extraMetaData, Version.FULL_VERSION), blocks);
     serializeFooter(footer, out);
     out.close();
@@ -615,7 +643,9 @@ private static void serializeFooter(ParquetMetadata footer, FSDataOutputStream o
     long footerIndex = out.getPos();
     org.apache.parquet.format.FileMetaData parquetMetadata = metadataConverter.toParquetMetadata(CURRENT_VERSION, footer);
     writeFileMetaData(parquetMetadata, out);
-    if (DEBUG) LOG.debug(out.getPos() + ": footer length = " + (out.getPos() - footerIndex));
+    if (LOGGER.isDebugEnabled()) {
+      LOGGER.debug(out.getPos() + ": footer length = " + (out.getPos() - footerIndex));
+    }
     BytesUtils.writeIntLittleEndian(out, (int) (out.getPos() - footerIndex));
     out.write(MAGIC);
   }
@@ -873,9 +903,11 @@ public void alignForRowGroup(FSDataOutputStream out) throws IOException {
       long remaining = dfsBlockSize - (out.getPos() % dfsBlockSize);
 
       if (isPaddingNeeded(remaining)) {
-        if (DEBUG) LOG.debug("Adding " + remaining + " bytes of padding (" +
-            "row group size=" + rowGroupSize + "B, " +
-            "block size=" + dfsBlockSize + "B)");
+        if (LOGGER.isDebugEnabled()) {
+          LOGGER.debug("Adding " + remaining + " bytes of padding (" +
+                       "row group size=" + rowGroupSize + "B, " +
+                       "block size=" + dfsBlockSize + "B)");
+        }
         for (; remaining > 0; remaining -= zeros.length) {
           out.write(zeros, 0, (int) Math.min((long) zeros.length, remaining));
         }
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
index e3536d7c56..b29b603a5c 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java
@@ -51,7 +51,6 @@
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.parquet.Log;
 import org.apache.parquet.Preconditions;
 import org.apache.parquet.filter.UnboundRecordFilter;
 import org.apache.parquet.filter2.compat.FilterCompat;
@@ -73,6 +72,9 @@
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.MessageTypeParser;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * The input format to read a Parquet file.
  *
@@ -93,7 +95,7 @@
  */
 public class ParquetInputFormat extends FileInputFormat {
 
-  private static final Log LOG = Log.getLog(ParquetInputFormat.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(ParquetInputFormat.class);
 
   /**
    * key to configure the ReadSupport implementation
@@ -364,7 +366,7 @@ private static List getAllFileRecursively(
         result.add(file);
       }
     }
-    LOG.info("Total input paths to process : " + result.size());
+    LOGGER.info("Total input paths to process : {}", result.size());
     return result;
   }
 
@@ -406,8 +408,8 @@ public List