From ff4c90d14fcac9f93cfc62d5cdcde19010ac5f8b Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Fri, 29 Apr 2016 14:16:08 -0700 Subject: [PATCH 01/15] Pull out value writer creation to ValuesWriterFactory and add unit tests --- parquet-column/pom.xml | 6 + .../parquet/column/ParquetProperties.java | 143 +------- .../column/values/ValuesWriterFactory.java | 177 +++++++++ .../values/ValuesWriterFactoryTest.java | 342 ++++++++++++++++++ 4 files changed, 533 insertions(+), 135 deletions(-) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index ccceafae2a..3653a87d71 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -83,6 +83,12 @@ ${slf4j.version} test + + org.mockito + mockito-all + 1.9.5 + test + diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index e3881f8110..257f6f7b39 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -24,28 +24,12 @@ import org.apache.parquet.bytes.HeapByteBufferAllocator; import static org.apache.parquet.bytes.BytesUtils.getWidthFromMaxInt; -import static org.apache.parquet.column.Encoding.PLAIN; -import static org.apache.parquet.column.Encoding.PLAIN_DICTIONARY; -import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; import org.apache.parquet.column.impl.ColumnWriteStoreV1; import org.apache.parquet.column.impl.ColumnWriteStoreV2; import org.apache.parquet.column.page.PageWriteStore; import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.ValuesWriterFactory; import org.apache.parquet.column.values.bitpacking.DevNullValuesWriter; -import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; -import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; -import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainFloatDictionaryValuesWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter; -import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.PlainLongDictionaryValuesWriter; -import org.apache.parquet.column.values.fallback.FallbackValuesWriter; -import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter; -import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; -import org.apache.parquet.column.values.plain.PlainValuesWriter; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; import org.apache.parquet.schema.MessageType; @@ -98,13 +82,13 @@ public static WriterVersion fromString(String name) { private final boolean estimateNextSizeCheck; private final ByteBufferAllocator allocator; - private final int initialSlabSize; + private final ValuesWriterFactory valuesWriterFactory; private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPageSize, boolean enableDict, int minRowCountForPageSizeCheck, int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator) { this.pageSizeThreshold = pageSize; - this.initialSlabSize = CapacityByteArrayOutputStream - .initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10); + int initialSlabSize = CapacityByteArrayOutputStream + .initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10); this.dictionaryPageSizeThreshold = dictPageSize; this.writerVersion = writerVersion; this.enableDictionary = enableDict; @@ -112,6 +96,9 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag this.maxRowCountForPageSizeCheck = maxRowCountForPageSizeCheck; this.estimateNextSizeCheck = estimateNextSizeCheck; this.allocator = allocator; + this.valuesWriterFactory = + new ValuesWriterFactory(writerVersion, initialSlabSize, pageSizeThreshold, allocator, + dictionaryPageSizeThreshold, enableDictionary); } public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) { @@ -144,122 +131,8 @@ private RunLengthBitPackingHybridEncoder newLevelEncoder(int maxLevel) { getWidthFromMaxInt(maxLevel), MIN_SLAB_SIZE, pageSizeThreshold, allocator); } - private ValuesWriter plainWriter(ColumnDescriptor path) { - switch (path.getType()) { - case BOOLEAN: - return new BooleanPlainValuesWriter(); - case INT96: - return new FixedLenByteArrayPlainValuesWriter(12, initialSlabSize, pageSizeThreshold, allocator); - case FIXED_LEN_BYTE_ARRAY: - return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), initialSlabSize, pageSizeThreshold, allocator); - case BINARY: - case INT32: - case INT64: - case DOUBLE: - case FLOAT: - return new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); - default: - throw new IllegalArgumentException("Unknown type " + path.getType()); - } - } - - @SuppressWarnings("deprecation") - private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { - Encoding encodingForDataPage; - Encoding encodingForDictionaryPage; - switch(writerVersion) { - case PARQUET_1_0: - encodingForDataPage = PLAIN_DICTIONARY; - encodingForDictionaryPage = PLAIN_DICTIONARY; - break; - case PARQUET_2_0: - encodingForDataPage = RLE_DICTIONARY; - encodingForDictionaryPage = PLAIN; - break; - default: - throw new IllegalArgumentException("Unknown version: " + writerVersion); - } - switch (path.getType()) { - case BOOLEAN: - throw new IllegalArgumentException("no dictionary encoding for BOOLEAN"); - case BINARY: - return new PlainBinaryDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); - case INT32: - return new PlainIntegerDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); - case INT64: - return new PlainLongDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); - case INT96: - return new PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, 12, encodingForDataPage, encodingForDictionaryPage, this.allocator); - case DOUBLE: - return new PlainDoubleDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); - case FLOAT: - return new PlainFloatDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); - case FIXED_LEN_BYTE_ARRAY: - return new PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, this.allocator); - default: - throw new IllegalArgumentException("Unknown type " + path.getType()); - } - } - - private ValuesWriter writerToFallbackTo(ColumnDescriptor path) { - switch(writerVersion) { - case PARQUET_1_0: - return plainWriter(path); - case PARQUET_2_0: - switch (path.getType()) { - case BOOLEAN: - return new RunLengthBitPackingHybridValuesWriter(1, initialSlabSize, pageSizeThreshold, allocator); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); - case INT32: - return new DeltaBinaryPackingValuesWriterForInteger(initialSlabSize, pageSizeThreshold, allocator); - case INT64: - return new DeltaBinaryPackingValuesWriterForLong(initialSlabSize, pageSizeThreshold, allocator); - case INT96: - case DOUBLE: - case FLOAT: - return plainWriter(path); - default: - throw new IllegalArgumentException("Unknown type " + path.getType()); - } - default: - throw new IllegalArgumentException("Unknown version: " + writerVersion); - } - } - - private ValuesWriter dictWriterWithFallBack(ColumnDescriptor path) { - ValuesWriter writerToFallBackTo = writerToFallbackTo(path); - if (enableDictionary) { - return FallbackValuesWriter.of( - dictionaryWriter(path), - writerToFallBackTo); - } else { - return writerToFallBackTo; - } - } - public ValuesWriter newValuesWriter(ColumnDescriptor path) { - switch (path.getType()) { - case BOOLEAN: // no dictionary encoding for boolean - return writerToFallbackTo(path); - case FIXED_LEN_BYTE_ARRAY: - // dictionary encoding for that type was not enabled in PARQUET 1.0 - if (writerVersion == WriterVersion.PARQUET_2_0) { - return dictWriterWithFallBack(path); - } else { - return writerToFallbackTo(path); - } - case BINARY: - case INT32: - case INT64: - case INT96: - case DOUBLE: - case FLOAT: - return dictWriterWithFallBack(path); - default: - throw new IllegalArgumentException("Unknown type " + path.getType()); - } + return valuesWriterFactory.newValuesWriter(path); } public int getPageSizeThreshold() { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java new file mode 100644 index 0000000000..ca47e92bf9 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values; + +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; +import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; +import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; +import org.apache.parquet.column.values.fallback.FallbackValuesWriter; +import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter; +import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; +import org.apache.parquet.column.values.plain.PlainValuesWriter; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; + +import static org.apache.parquet.column.Encoding.PLAIN; +import static org.apache.parquet.column.Encoding.PLAIN_DICTIONARY; +import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; + +public class ValuesWriterFactory { + + private final WriterVersion writerVersion; + + private final boolean enableDictionary; + private final int initialSlabSize; + private final int pageSizeThreshold; + private final ByteBufferAllocator allocator; + private final int dictionaryPageSizeThreshold; + + public ValuesWriterFactory(WriterVersion writerVersion, int initialSlabSize, int pageSizeThreshold, + ByteBufferAllocator allocator, int dictionaryPageSizeThreshold, + boolean enableDictionary) { + this.writerVersion = writerVersion; + this.initialSlabSize = initialSlabSize; + this.pageSizeThreshold = pageSizeThreshold; + this.allocator = allocator; + this.dictionaryPageSizeThreshold = dictionaryPageSizeThreshold; + this.enableDictionary = enableDictionary; + } + + public ValuesWriter newValuesWriter(ColumnDescriptor path) { + switch (path.getType()) { + case BOOLEAN: // no dictionary encoding for boolean + return writerToFallbackTo(path); + case FIXED_LEN_BYTE_ARRAY: + // dictionary encoding for that type was not enabled in PARQUET 1.0 + if (writerVersion == WriterVersion.PARQUET_2_0) { + return dictWriterWithFallBack(path); + } else { + return writerToFallbackTo(path); + } + case BINARY: + case INT32: + case INT64: + case INT96: + case DOUBLE: + case FLOAT: + return dictWriterWithFallBack(path); + default: + throw new IllegalArgumentException("Unknown type " + path.getType()); + } + } + + private ValuesWriter plainWriter(ColumnDescriptor path) { + switch (path.getType()) { + case BOOLEAN: + return new BooleanPlainValuesWriter(); + case INT96: + return new FixedLenByteArrayPlainValuesWriter(12, initialSlabSize, pageSizeThreshold, allocator); + case FIXED_LEN_BYTE_ARRAY: + return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), initialSlabSize, pageSizeThreshold, allocator); + case BINARY: + case INT32: + case INT64: + case DOUBLE: + case FLOAT: + return new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + default: + throw new IllegalArgumentException("Unknown type " + path.getType()); + } + } + + @SuppressWarnings("deprecation") + private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { + Encoding encodingForDataPage; + Encoding encodingForDictionaryPage; + switch(writerVersion) { + case PARQUET_1_0: + encodingForDataPage = PLAIN_DICTIONARY; + encodingForDictionaryPage = PLAIN_DICTIONARY; + break; + case PARQUET_2_0: + encodingForDataPage = RLE_DICTIONARY; + encodingForDictionaryPage = PLAIN; + break; + default: + throw new IllegalArgumentException("Unknown version: " + writerVersion); + } + switch (path.getType()) { + case BOOLEAN: + throw new IllegalArgumentException("no dictionary encoding for BOOLEAN"); + case BINARY: + return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + case INT32: + return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + case INT64: + return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + case INT96: + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, 12, encodingForDataPage, encodingForDictionaryPage, this.allocator); + case DOUBLE: + return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + case FLOAT: + return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + case FIXED_LEN_BYTE_ARRAY: + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, this.allocator); + default: + throw new IllegalArgumentException("Unknown type " + path.getType()); + } + } + + private ValuesWriter writerToFallbackTo(ColumnDescriptor path) { + switch(writerVersion) { + case PARQUET_1_0: + return plainWriter(path); + case PARQUET_2_0: + switch (path.getType()) { + case BOOLEAN: + return new RunLengthBitPackingHybridValuesWriter(1, initialSlabSize, pageSizeThreshold, allocator); + case BINARY: + case FIXED_LEN_BYTE_ARRAY: + return new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); + case INT32: + return new DeltaBinaryPackingValuesWriterForInteger(initialSlabSize, pageSizeThreshold, allocator); + case INT64: + return new DeltaBinaryPackingValuesWriterForLong(initialSlabSize, pageSizeThreshold, allocator); + case INT96: + case DOUBLE: + case FLOAT: + return plainWriter(path); + default: + throw new IllegalArgumentException("Unknown type " + path.getType()); + } + default: + throw new IllegalArgumentException("Unknown version: " + writerVersion); + } + } + + private ValuesWriter dictWriterWithFallBack(ColumnDescriptor path) { + ValuesWriter writerToFallBackTo = writerToFallbackTo(path); + if (enableDictionary) { + return FallbackValuesWriter.of( + dictionaryWriter(path), + writerToFallBackTo); + } else { + return writerToFallBackTo; + } + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java new file mode 100644 index 0000000000..b689d5ed04 --- /dev/null +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; +import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; +import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; +import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter.*; +import org.apache.parquet.column.values.fallback.FallbackValuesWriter; +import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter; +import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; +import org.apache.parquet.column.values.plain.PlainValuesWriter; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; + +import org.junit.Test; + +import static junit.framework.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ValuesWriterFactoryTest { + + @Test + public void testBoolean() { + doTestValueWriter( + PrimitiveTypeName.BOOLEAN, + WriterVersion.PARQUET_1_0, + true, + BooleanPlainValuesWriter.class); + } + + @Test + public void testBoolean_V2() { + doTestValueWriter( + PrimitiveTypeName.BOOLEAN, + WriterVersion.PARQUET_2_0, + true, + RunLengthBitPackingHybridValuesWriter.class); + } + + @Test + public void testFixedLenByteArray() { + doTestValueWriter( + PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, + WriterVersion.PARQUET_1_0, + true, + FixedLenByteArrayPlainValuesWriter.class); + } + + @Test + public void testFixedLenByteArray_V2() { + doTestValueWriter( + PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, + WriterVersion.PARQUET_2_0, + true, + DictionaryValuesWriter.class, DeltaByteArrayWriter.class); + } + + @Test + public void testFixedLenByteArray_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, + WriterVersion.PARQUET_2_0, + false, + DeltaByteArrayWriter.class); + } + + @Test + public void testBinary() { + doTestValueWriter( + PrimitiveTypeName.BINARY, + WriterVersion.PARQUET_1_0, + true, + PlainBinaryDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testBinary_NoDict() { + doTestValueWriter( + PrimitiveTypeName.BINARY, + WriterVersion.PARQUET_1_0, + false, + PlainValuesWriter.class); + } + + @Test + public void testBinary_V2() { + doTestValueWriter( + PrimitiveTypeName.BINARY, + WriterVersion.PARQUET_2_0, + true, + PlainBinaryDictionaryValuesWriter.class, DeltaByteArrayWriter.class); + } + + @Test + public void testBinary_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.BINARY, + WriterVersion.PARQUET_2_0, + false, + DeltaByteArrayWriter.class); + } + + @Test + public void testInt32() { + doTestValueWriter( + PrimitiveTypeName.INT32, + WriterVersion.PARQUET_1_0, + true, + PlainIntegerDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testInt32_NoDict() { + doTestValueWriter( + PrimitiveTypeName.INT32, + WriterVersion.PARQUET_1_0, + false, + PlainValuesWriter.class); + } + + @Test + public void testInt32_V2() { + doTestValueWriter( + PrimitiveTypeName.INT32, + WriterVersion.PARQUET_2_0, + true, + PlainIntegerDictionaryValuesWriter.class, DeltaBinaryPackingValuesWriter.class); + } + + @Test + public void testInt32_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.INT32, + WriterVersion.PARQUET_2_0, + false, + DeltaBinaryPackingValuesWriter.class); + } + + @Test + public void testInt64() { + doTestValueWriter( + PrimitiveTypeName.INT64, + WriterVersion.PARQUET_1_0, + true, + PlainLongDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testInt64_NoDict() { + doTestValueWriter( + PrimitiveTypeName.INT64, + WriterVersion.PARQUET_1_0, + false, + PlainValuesWriter.class); + } + + @Test + public void testInt64_V2() { + doTestValueWriter( + PrimitiveTypeName.INT64, + WriterVersion.PARQUET_2_0, + true, + PlainLongDictionaryValuesWriter.class, DeltaBinaryPackingValuesWriterForLong.class); + } + + @Test + public void testInt64_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.INT64, + WriterVersion.PARQUET_2_0, + false, + DeltaBinaryPackingValuesWriterForLong.class); + } + + @Test + public void testInt96() { + doTestValueWriter( + PrimitiveTypeName.INT96, + WriterVersion.PARQUET_1_0, + true, + PlainFixedLenArrayDictionaryValuesWriter.class, FixedLenByteArrayPlainValuesWriter.class); + } + + @Test + public void testInt96_NoDict() { + doTestValueWriter( + PrimitiveTypeName.INT96, + WriterVersion.PARQUET_1_0, + false, + FixedLenByteArrayPlainValuesWriter.class); + } + + @Test + public void testInt96_V2() { + doTestValueWriter( + PrimitiveTypeName.INT96, + WriterVersion.PARQUET_2_0, + true, + PlainFixedLenArrayDictionaryValuesWriter.class, FixedLenByteArrayPlainValuesWriter.class); + } + + @Test + public void testInt96_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.INT96, + WriterVersion.PARQUET_2_0, + false, + FixedLenByteArrayPlainValuesWriter.class); + } + + @Test + public void testDouble() { + doTestValueWriter( + PrimitiveTypeName.DOUBLE, + WriterVersion.PARQUET_1_0, + true, + PlainDoubleDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testDouble_NoDict() { + doTestValueWriter( + PrimitiveTypeName.DOUBLE, + WriterVersion.PARQUET_1_0, + false, + PlainValuesWriter.class); + } + + @Test + public void testDouble_V2() { + doTestValueWriter( + PrimitiveTypeName.DOUBLE, + WriterVersion.PARQUET_2_0, + true, + PlainDoubleDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testDouble_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.DOUBLE, + WriterVersion.PARQUET_2_0, + false, + PlainValuesWriter.class); + } + + @Test + public void testFloat() { + doTestValueWriter( + PrimitiveTypeName.FLOAT, + WriterVersion.PARQUET_1_0, + true, + PlainFloatDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testFloat_NoDict() { + doTestValueWriter( + PrimitiveTypeName.FLOAT, + WriterVersion.PARQUET_1_0, + false, + PlainValuesWriter.class); + } + + @Test + public void testFloat_V2() { + doTestValueWriter( + PrimitiveTypeName.FLOAT, + WriterVersion.PARQUET_2_0, + true, + PlainFloatDictionaryValuesWriter.class, PlainValuesWriter.class); + } + + @Test + public void testFloat_V2_NoDict() { + doTestValueWriter( + PrimitiveTypeName.FLOAT, + WriterVersion.PARQUET_2_0, + false, + PlainValuesWriter.class); + } + + private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, Class expectedValueWriterClass) { + ColumnDescriptor mockPath = getMockColumn(typeName); + ValuesWriterFactory factory = getFactory(version, enableDictionary); + ValuesWriter writer = factory.newValuesWriter(mockPath); + + validateWriterType(writer, expectedValueWriterClass); + } + + private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, Class initialValueWriterClass, Class fallbackValueWriterClass) { + ColumnDescriptor mockPath = getMockColumn(typeName); + ValuesWriterFactory factory = getFactory(version, enableDictionary); + ValuesWriter writer = factory.newValuesWriter(mockPath); + + validateFallbackWriter(writer, initialValueWriterClass, fallbackValueWriterClass); + } + + private ColumnDescriptor getMockColumn(PrimitiveTypeName typeName) { + ColumnDescriptor mockPath = mock(ColumnDescriptor.class); + when(mockPath.getType()).thenReturn(typeName); + return mockPath; + } + + private ValuesWriterFactory getFactory(WriterVersion writerVersion, boolean enableDictionary) { + return new ValuesWriterFactory(writerVersion, 128, ParquetProperties.DEFAULT_PAGE_SIZE, null, 0, enableDictionary); + } + + private void validateWriterType(ValuesWriter writer, Class valuesWriterClass) { + assertTrue("Not instance of: " + valuesWriterClass.getName(), valuesWriterClass.isInstance(writer)); + } + + private void validateFallbackWriter(ValuesWriter writer, Class initialWriterClass, Class fallbackWriterClass) { + validateWriterType(writer, FallbackValuesWriter.class); + + FallbackValuesWriter wr = (FallbackValuesWriter) writer; + validateWriterType(wr.initialWriter, initialWriterClass); + validateWriterType(wr.fallBackWriter, fallbackWriterClass); + } +} From b9d6c13119965ef565b5985990de7aac56b726ff Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Fri, 29 Apr 2016 14:17:26 -0700 Subject: [PATCH 02/15] Refactor code in ValuesWriterFactory a bit --- .../column/values/ValuesWriterFactory.java | 137 +++++++++++------- 1 file changed, 83 insertions(+), 54 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java index ca47e92bf9..f9be7682fc 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java @@ -59,43 +59,100 @@ public ValuesWriterFactory(WriterVersion writerVersion, int initialSlabSize, int public ValuesWriter newValuesWriter(ColumnDescriptor path) { switch (path.getType()) { - case BOOLEAN: // no dictionary encoding for boolean - return writerToFallbackTo(path); + case BOOLEAN: + return getBooleanValuesWriter(); case FIXED_LEN_BYTE_ARRAY: - // dictionary encoding for that type was not enabled in PARQUET 1.0 - if (writerVersion == WriterVersion.PARQUET_2_0) { - return dictWriterWithFallBack(path); - } else { - return writerToFallbackTo(path); - } + return getFixedLenByteArrayValuesWriter(path); case BINARY: + return getBinaryValuesWriter(path); case INT32: + return getInt32ValuesWriter(path); case INT64: + return getInt64ValuesWriter(path); case INT96: + return getInt96ValuesWriter(path); case DOUBLE: + return getDoubleValuesWriter(path); case FLOAT: - return dictWriterWithFallBack(path); + return getFloatValuesWriter(path); default: throw new IllegalArgumentException("Unknown type " + path.getType()); } } - private ValuesWriter plainWriter(ColumnDescriptor path) { - switch (path.getType()) { - case BOOLEAN: - return new BooleanPlainValuesWriter(); - case INT96: - return new FixedLenByteArrayPlainValuesWriter(12, initialSlabSize, pageSizeThreshold, allocator); - case FIXED_LEN_BYTE_ARRAY: - return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), initialSlabSize, pageSizeThreshold, allocator); - case BINARY: - case INT32: - case INT64: - case DOUBLE: - case FLOAT: - return new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); - default: - throw new IllegalArgumentException("Unknown type " + path.getType()); + private ValuesWriter getBooleanValuesWriter() { + // no dictionary encoding for boolean + if(writerVersion == WriterVersion.PARQUET_1_0) { + return new BooleanPlainValuesWriter(); + } else { + return new RunLengthBitPackingHybridValuesWriter(1, initialSlabSize, pageSizeThreshold, allocator); + } + } + + private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { + if (writerVersion == WriterVersion.PARQUET_1_0) { + // dictionary encoding was not enabled in PARQUET 1.0 + return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), initialSlabSize, pageSizeThreshold, allocator); + } else { + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } + } + + private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { + if(writerVersion == WriterVersion.PARQUET_1_0) { + ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } else { + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } + } + + private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { + if(writerVersion == WriterVersion.PARQUET_1_0) { + ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } else { + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } + } + + private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { + if(writerVersion == WriterVersion.PARQUET_1_0) { + ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } else { + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(initialSlabSize, pageSizeThreshold, allocator); + return dictWriterWithFallBack(path, fallbackWriter); + } + } + + private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, initialSlabSize, pageSizeThreshold, allocator); + if(writerVersion == WriterVersion.PARQUET_1_0) { + return dictWriterWithFallBack(path, fallbackWriter); + } else { + return dictWriterWithFallBack(path, fallbackWriter); + } + } + + private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + if(writerVersion == WriterVersion.PARQUET_1_0) { + return dictWriterWithFallBack(path, fallbackWriter); + } else { + return dictWriterWithFallBack(path, fallbackWriter); + } + } + + private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + if(writerVersion == WriterVersion.PARQUET_1_0) { + return dictWriterWithFallBack(path, fallbackWriter); + } else { + return dictWriterWithFallBack(path, fallbackWriter); } } @@ -137,35 +194,7 @@ private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { } } - private ValuesWriter writerToFallbackTo(ColumnDescriptor path) { - switch(writerVersion) { - case PARQUET_1_0: - return plainWriter(path); - case PARQUET_2_0: - switch (path.getType()) { - case BOOLEAN: - return new RunLengthBitPackingHybridValuesWriter(1, initialSlabSize, pageSizeThreshold, allocator); - case BINARY: - case FIXED_LEN_BYTE_ARRAY: - return new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); - case INT32: - return new DeltaBinaryPackingValuesWriterForInteger(initialSlabSize, pageSizeThreshold, allocator); - case INT64: - return new DeltaBinaryPackingValuesWriterForLong(initialSlabSize, pageSizeThreshold, allocator); - case INT96: - case DOUBLE: - case FLOAT: - return plainWriter(path); - default: - throw new IllegalArgumentException("Unknown type " + path.getType()); - } - default: - throw new IllegalArgumentException("Unknown version: " + writerVersion); - } - } - - private ValuesWriter dictWriterWithFallBack(ColumnDescriptor path) { - ValuesWriter writerToFallBackTo = writerToFallbackTo(path); + private ValuesWriter dictWriterWithFallBack(ColumnDescriptor path, ValuesWriter writerToFallBackTo) { if (enableDictionary) { return FallbackValuesWriter.of( dictionaryWriter(path), From 5c636c739f6df38c677fafdc7cedbb85c2b21de3 Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Mon, 2 May 2016 18:30:44 -0700 Subject: [PATCH 03/15] Add encoding-overrides config to ParquetOutputFormat config --- .../parquet/column/ParquetProperties.java | 20 +++- .../parquet/hadoop/ParquetOutputFormat.java | 78 +++++++++++++-- ...tParquetOutputFormatEncodingOverrides.java | 97 +++++++++++++++++++ 3 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 257f6f7b39..4ed01a6ad9 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -18,6 +18,10 @@ */ package org.apache.parquet.column; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.parquet.Preconditions; import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; @@ -33,6 +37,7 @@ import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; /** * This class represents all the configurable Parquet properties. @@ -85,7 +90,8 @@ public static WriterVersion fromString(String name) { private final ValuesWriterFactory valuesWriterFactory; private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPageSize, boolean enableDict, int minRowCountForPageSizeCheck, - int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator) { + int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator, + Map> encodingOverrides) { this.pageSizeThreshold = pageSize; int initialSlabSize = CapacityByteArrayOutputStream .initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10); @@ -98,7 +104,7 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag this.allocator = allocator; this.valuesWriterFactory = new ValuesWriterFactory(writerVersion, initialSlabSize, pageSizeThreshold, allocator, - dictionaryPageSizeThreshold, enableDictionary); + dictionaryPageSizeThreshold, enableDictionary, encodingOverrides); } public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) { @@ -196,6 +202,8 @@ public static class Builder { private int maxRowCountForPageSizeCheck = DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK; private boolean estimateNextSizeCheck = DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK; private ByteBufferAllocator allocator = new HeapByteBufferAllocator(); + private Map> encodingOverrides = + new HashMap>(); private Builder() { } @@ -284,10 +292,16 @@ public Builder withAllocator(ByteBufferAllocator allocator) { return this; } + public Builder withEncodingOverrides(Map> encodingOverrides) { + this.encodingOverrides = encodingOverrides; + return this; + } + public ParquetProperties build() { return new ParquetProperties(writerVersion, pageSize, dictPageSize, enableDict, minRowCountForPageSizeCheck, maxRowCountForPageSizeCheck, - estimateNextSizeCheck, allocator); + estimateNextSizeCheck, allocator, encodingOverrides); } + } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 6cfa8e93c3..533c698ced 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -24,7 +24,12 @@ import static org.apache.parquet.hadoop.util.ContextUtil.getConfiguration; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; @@ -36,6 +41,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.parquet.Log; +import org.apache.parquet.column.Encoding; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; import org.apache.parquet.hadoop.ParquetFileWriter.Mode; @@ -44,6 +50,7 @@ import org.apache.parquet.hadoop.codec.CodecConfig; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.hadoop.util.ConfigurationUtil; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; /** * OutputFormat to write to a Parquet file @@ -145,6 +152,20 @@ public static enum JobSummaryLevel { public static final String MAX_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.max"; public static final String ESTIMATE_PAGE_SIZE_CHECK = "parquet.page.size.check.estimate"; + /** + * Used to override the writer encodings for various types. + * See {@link org.apache.parquet.column.Encoding} for a list of valid Encoding choices to use. + * For e.g. "parquet.writer.encoding-override.boolean" = "plain" will ensure that we use + * {@link org.apache.parquet.column.Encoding.PLAIN} for boolean values. + * We can also specify fallbacks: + * parquet.writer.encoding-override.binary = "plain_dictionary,plain". This results in a + * {@link org.apache.parquet.column.values.fallback.FallbackValuesWriter} with Plain dictionary as + * the initial writer and Plain encoding used as the fallback for binary values. + * Note: If fallbacks are specified, the initial writer must implement + * {@link org.apache.parquet.column.values.RequiresFallback}. + */ + public static final String WRITER_ENCODING_OVERRIDE_PREFIX = "parquet.writer.encoding-override."; + // default to no padding for now private static final int DEFAULT_MAX_PADDING_SIZE = 0; @@ -317,6 +338,40 @@ private static int getMaxPaddingSize(Configuration conf) { return conf.getInt(MAX_PADDING_BYTES, DEFAULT_MAX_PADDING_SIZE); } + public static Map> getEncodingOverrides(Configuration conf) { + Map> typeToEncoding = new HashMap>(); + + for(PrimitiveTypeName name : PrimitiveTypeName.values()) { + String typeOverride = conf.get(WRITER_ENCODING_OVERRIDE_PREFIX + name.name().toLowerCase()); + typeToEncoding.put(name, getEncodingOverridesForType(name, typeOverride)); + } + + return typeToEncoding; + } + + private static List getEncodingOverridesForType(PrimitiveTypeName typeName, String typeOverride) { + List encodings = new ArrayList(); + if( StringUtils.isEmpty(typeOverride) ) { + return encodings; + } + + String [] overrides = typeOverride.split(","); + if( overrides.length > 2 ) { + //maybe in the future we could chain more + throw new BadConfigurationException("For : " + typeName + " too many overrides specified, must not be more than 2"); + } + + for(String override : overrides) { + try { + Encoding encoding = Encoding.valueOf(override.toUpperCase()); + encodings.add(encoding); + } catch(IllegalArgumentException e) { + throw new BadConfigurationException("For type: " + typeName + "Invalid encoding type chosen: " + override); + } + } + + return encodings; + } private WriteSupport writeSupport; private ParquetOutputCommitter committer; @@ -368,22 +423,25 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) + .withEncodingOverrides(getEncodingOverrides(conf)) .build(); long blockSize = getLongBlockSize(conf); int maxPaddingSize = getMaxPaddingSize(conf); boolean validating = getValidation(conf); - if (INFO) LOG.info("Parquet block size to " + blockSize); - if (INFO) LOG.info("Parquet page size to " + props.getPageSizeThreshold()); - if (INFO) LOG.info("Parquet dictionary page size to " + props.getDictionaryPageSizeThreshold()); - if (INFO) LOG.info("Dictionary is " + (props.isEnableDictionary() ? "on" : "off")); - if (INFO) LOG.info("Validation is " + (validating ? "on" : "off")); - if (INFO) LOG.info("Writer version is: " + props.getWriterVersion()); - if (INFO) LOG.info("Maximum row group padding size is " + maxPaddingSize + " bytes"); - if (INFO) LOG.info("Page size checking is: " + (props.estimateNextSizeCheck() ? "estimated" : "constant")); - if (INFO) LOG.info("Min row count for page size check is: " + props.getMinRowCountForPageSizeCheck()); - if (INFO) LOG.info("Max row count for page size check is: " + props.getMaxRowCountForPageSizeCheck()); + if (INFO) { + LOG.info("Parquet block size to " + blockSize); + LOG.info("Parquet page size to " + props.getPageSizeThreshold()); + LOG.info("Parquet dictionary page size to " + props.getDictionaryPageSizeThreshold()); + LOG.info("Dictionary is " + (props.isEnableDictionary() ? "on" : "off")); + LOG.info("Validation is " + (validating ? "on" : "off")); + LOG.info("Writer version is: " + props.getWriterVersion()); + LOG.info("Maximum row group padding size is " + maxPaddingSize + " bytes"); + LOG.info("Page size checking is: " + (props.estimateNextSizeCheck() ? "estimated" : "constant")); + LOG.info("Min row count for page size check is: " + props.getMinRowCountForPageSizeCheck()); + LOG.info("Max row count for page size check is: " + props.getMaxRowCountForPageSizeCheck()); + } WriteContext init = writeSupport.init(conf); ParquetFileWriter w = new ParquetFileWriter( diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java new file mode 100644 index 0000000000..e088a747fc --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestParquetOutputFormatEncodingOverrides { + + @Test + public void testEmptyOverrideString() { + Configuration conf = new Configuration(); + Map> encodings = ParquetOutputFormat.getEncodingOverrides(conf); + assertEquals("Incorrect number of encoding entries", PrimitiveTypeName.values().length, encodings.size()); + + for(Map.Entry> entry : encodings.entrySet()) { + assertTrue("Non-empty encoding list for: " + entry.getKey(), entry.getValue().isEmpty()); + } + } + + @Test + public void testOneValidEncoding() { + Configuration conf = new Configuration(); + conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), Encoding.PLAIN.name()); + + Map> encodings = ParquetOutputFormat.getEncodingOverrides(conf); + List encodingList = encodings.get(PrimitiveTypeName.BOOLEAN); + assertTrue(encodingList.size() == 1); + assertEquals(Encoding.PLAIN, encodingList.get(0)); + } + + @Test(expected = BadConfigurationException.class) + public void testOneBrokenEncoding() { + Configuration conf = new Configuration(); + conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), "foo-encoding"); + + // should fail due to an exception + ParquetOutputFormat.getEncodingOverrides(conf); + } + + @Test(expected = BadConfigurationException.class) + public void testOneGood_OneBrokenEncoding() { + Configuration conf = new Configuration(); + String encodingStr = Encoding.PLAIN.name() + "," + "foo-encoding"; + conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), encodingStr); + + // should fail due to an exception + ParquetOutputFormat.getEncodingOverrides(conf); + } + + @Test + public void testTwoEncodings() { + Configuration conf = new Configuration(); + String encodingStr = Encoding.PLAIN_DICTIONARY.name() + "," + Encoding.PLAIN; + conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), encodingStr); + + Map> encodings = ParquetOutputFormat.getEncodingOverrides(conf); + List encodingList = encodings.get(PrimitiveTypeName.BOOLEAN); + assertTrue(encodingList.size() == 2); + assertEquals(Encoding.PLAIN_DICTIONARY, encodingList.get(0)); + assertEquals(Encoding.PLAIN, encodingList.get(1)); + } + + @Test(expected = BadConfigurationException.class) + public void testThreeEncodings() { + Configuration conf = new Configuration(); + String encodingStr = Encoding.PLAIN_DICTIONARY.name() + "," + Encoding.PLAIN + "," + Encoding.DELTA_BINARY_PACKED; + conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), encodingStr); + + // should fail due to an exception + ParquetOutputFormat.getEncodingOverrides(conf); + } +} From 9ead61d0b9f436651a83ab11dafe84c3cf8183ae Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Tue, 3 May 2016 18:12:59 -0700 Subject: [PATCH 04/15] Add guava test dep --- parquet-avro/pom.xml | 2 +- parquet-column/pom.xml | 6 ++++++ parquet-hadoop/pom.xml | 2 +- parquet-pig/pom.xml | 2 +- parquet-tools/pom.xml | 2 +- pom.xml | 1 + 6 files changed, 11 insertions(+), 4 deletions(-) diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml index 50c37db75f..109cc3875d 100644 --- a/parquet-avro/pom.xml +++ b/parquet-avro/pom.xml @@ -67,7 +67,7 @@ com.google.guava guava - 11.0 + ${guava.version} test diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index 3653a87d71..80ed21d076 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -89,6 +89,12 @@ 1.9.5 test + + com.google.guava + guava + ${guava.version} + test + diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index 4c4318212f..1e6729f1bf 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -83,7 +83,7 @@ com.google.guava guava - 11.0 + ${guava.version} test diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml index 9b6371e251..4142c3bcd6 100644 --- a/parquet-pig/pom.xml +++ b/parquet-pig/pom.xml @@ -101,7 +101,7 @@ com.google.guava guava - 11.0 + ${guava.version} test diff --git a/parquet-tools/pom.xml b/parquet-tools/pom.xml index 5d3f7c958a..66abaa9d0f 100644 --- a/parquet-tools/pom.xml +++ b/parquet-tools/pom.xml @@ -70,7 +70,7 @@ com.google.guava guava - 11.0 + ${guava.version} org.slf4j diff --git a/pom.xml b/pom.xml index 510c329749..1347910786 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,7 @@ 0.9.33 1.7.5 1.8.0 + 11.0 From 0f8cd09a629078673db332a29862466abe4b73e2 Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Mon, 16 May 2016 18:14:20 -0700 Subject: [PATCH 05/15] Refactor mockito version --- parquet-cascading/pom.xml | 2 +- parquet-cascading3/pom.xml | 2 +- parquet-column/pom.xml | 2 +- parquet-hadoop/pom.xml | 2 +- .../java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 2 +- parquet-protobuf/pom.xml | 2 +- pom.xml | 1 + 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/parquet-cascading/pom.xml b/parquet-cascading/pom.xml index 0e1e1e1d79..0573aba664 100644 --- a/parquet-cascading/pom.xml +++ b/parquet-cascading/pom.xml @@ -77,7 +77,7 @@ org.mockito mockito-all - 1.9.5 + ${mockito.version} test diff --git a/parquet-cascading3/pom.xml b/parquet-cascading3/pom.xml index 67b5e0955f..9aa8991e9b 100644 --- a/parquet-cascading3/pom.xml +++ b/parquet-cascading3/pom.xml @@ -88,7 +88,7 @@ org.mockito mockito-all - 1.9.5 + ${mockito.version} test diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml index 80ed21d076..014921c3df 100644 --- a/parquet-column/pom.xml +++ b/parquet-column/pom.xml @@ -86,7 +86,7 @@ org.mockito mockito-all - 1.9.5 + ${mockito.version} test diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml index 1e6729f1bf..79ca7f48e4 100644 --- a/parquet-hadoop/pom.xml +++ b/parquet-hadoop/pom.xml @@ -89,7 +89,7 @@ org.mockito mockito-all - 1.9.5 + ${mockito.version} test diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 533c698ced..b62e90ff88 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -366,7 +366,7 @@ private static List getEncodingOverridesForType(PrimitiveTypeName type Encoding encoding = Encoding.valueOf(override.toUpperCase()); encodings.add(encoding); } catch(IllegalArgumentException e) { - throw new BadConfigurationException("For type: " + typeName + "Invalid encoding type chosen: " + override); + throw new BadConfigurationException("For type: " + typeName + " Invalid encoding type chosen: " + override); } } diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml index b3e4e501f9..0c9cae4437 100644 --- a/parquet-protobuf/pom.xml +++ b/parquet-protobuf/pom.xml @@ -42,7 +42,7 @@ org.mockito mockito-core - 1.9.5 + ${mockito.version} test diff --git a/pom.xml b/pom.xml index 1347910786..ccc4675f83 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,7 @@ 1.7.5 1.8.0 11.0 + 1.9.5 From 6a5428ffbc68a579b74b651fee658affcf2e034e Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Mon, 16 May 2016 18:15:15 -0700 Subject: [PATCH 06/15] Clean up some stuff in ValuesWriterFactory --- .../column/values/ValuesWriterFactory.java | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java index f9be7682fc..e0f43d56f0 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java @@ -100,7 +100,7 @@ private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { } private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - if(writerVersion == WriterVersion.PARQUET_1_0) { + if (writerVersion == WriterVersion.PARQUET_1_0) { ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); return dictWriterWithFallBack(path, fallbackWriter); } else { @@ -110,7 +110,7 @@ private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { } private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { - if(writerVersion == WriterVersion.PARQUET_1_0) { + if (writerVersion == WriterVersion.PARQUET_1_0) { ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); return dictWriterWithFallBack(path, fallbackWriter); } else { @@ -120,7 +120,7 @@ private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { } private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { - if(writerVersion == WriterVersion.PARQUET_1_0) { + if (writerVersion == WriterVersion.PARQUET_1_0) { ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); return dictWriterWithFallBack(path, fallbackWriter); } else { @@ -131,29 +131,17 @@ private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, initialSlabSize, pageSizeThreshold, allocator); - if(writerVersion == WriterVersion.PARQUET_1_0) { - return dictWriterWithFallBack(path, fallbackWriter); - } else { - return dictWriterWithFallBack(path, fallbackWriter); - } + return dictWriterWithFallBack(path, fallbackWriter); } private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); - if(writerVersion == WriterVersion.PARQUET_1_0) { - return dictWriterWithFallBack(path, fallbackWriter); - } else { - return dictWriterWithFallBack(path, fallbackWriter); - } + return dictWriterWithFallBack(path, fallbackWriter); } private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); - if(writerVersion == WriterVersion.PARQUET_1_0) { - return dictWriterWithFallBack(path, fallbackWriter); - } else { - return dictWriterWithFallBack(path, fallbackWriter); - } + return dictWriterWithFallBack(path, fallbackWriter); } @SuppressWarnings("deprecation") @@ -176,19 +164,19 @@ private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { case BOOLEAN: throw new IllegalArgumentException("no dictionary encoding for BOOLEAN"); case BINARY: - return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); case INT32: - return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); case INT64: - return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); case INT96: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, 12, encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, 12, encodingForDataPage, encodingForDictionaryPage, allocator); case DOUBLE: - return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); case FLOAT: - return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); case FIXED_LEN_BYTE_ARRAY: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, this.allocator); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, allocator); default: throw new IllegalArgumentException("Unknown type " + path.getType()); } From b46cccd85340cc48836b237798877eed99d894dc Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Tue, 17 May 2016 15:02:01 -0700 Subject: [PATCH 07/15] Add class based factory override --- .../parquet/column/ParquetProperties.java | 44 ++++--- .../DefaultValuesWriterFactory.java} | 107 ++++++++---------- .../values/factory/ValuesWriterFactory.java | 40 +++++++ .../factory/ValuesWriterFactoryParams.java | 75 ++++++++++++ .../DefaultValuesWriterFactoryTest.java} | 17 +-- .../parquet/hadoop/ParquetOutputFormat.java | 70 ++++-------- .../hadoop/StubValuesWriterFactory.java | 37 ++++++ ...tParquetOutputFormatEncodingOverrides.java | 97 ---------------- ...stParquetOutputFormatFactoryOverrides.java | 59 ++++++++++ 9 files changed, 317 insertions(+), 229 deletions(-) rename parquet-column/src/main/java/org/apache/parquet/column/values/{ValuesWriterFactory.java => factory/DefaultValuesWriterFactory.java} (55%) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java rename parquet-column/src/test/java/org/apache/parquet/column/values/{ValuesWriterFactoryTest.java => factory/DefaultValuesWriterFactoryTest.java} (92%) create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java delete mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 4ed01a6ad9..d3dc580601 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -18,10 +18,7 @@ */ package org.apache.parquet.column; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import org.apache.parquet.Log; import org.apache.parquet.Preconditions; import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; @@ -32,12 +29,13 @@ import org.apache.parquet.column.impl.ColumnWriteStoreV2; import org.apache.parquet.column.page.PageWriteStore; import org.apache.parquet.column.values.ValuesWriter; -import org.apache.parquet.column.values.ValuesWriterFactory; import org.apache.parquet.column.values.bitpacking.DevNullValuesWriter; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; +import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; /** * This class represents all the configurable Parquet properties. @@ -57,6 +55,8 @@ public class ParquetProperties { private static final int MIN_SLAB_SIZE = 64; + private static final Log LOG = Log.getLog(ParquetProperties.class); + public enum WriterVersion { PARQUET_1_0 ("v1"), PARQUET_2_0 ("v2"); @@ -86,12 +86,11 @@ public static WriterVersion fromString(String name) { private final int maxRowCountForPageSizeCheck; private final boolean estimateNextSizeCheck; private final ByteBufferAllocator allocator; - private final ValuesWriterFactory valuesWriterFactory; private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPageSize, boolean enableDict, int minRowCountForPageSizeCheck, int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator, - Map> encodingOverrides) { + Class factoryOverride) { this.pageSizeThreshold = pageSize; int initialSlabSize = CapacityByteArrayOutputStream .initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10); @@ -102,9 +101,23 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag this.maxRowCountForPageSizeCheck = maxRowCountForPageSizeCheck; this.estimateNextSizeCheck = estimateNextSizeCheck; this.allocator = allocator; - this.valuesWriterFactory = - new ValuesWriterFactory(writerVersion, initialSlabSize, pageSizeThreshold, allocator, - dictionaryPageSizeThreshold, enableDictionary, encodingOverrides); + + ValuesWriterFactoryParams params = + new ValuesWriterFactoryParams(writerVersion, initialSlabSize, pageSizeThreshold, allocator, + enableDictionary, dictionaryPageSizeThreshold); + this.valuesWriterFactory = initValuesWriterFactory(factoryOverride, params); + } + + private ValuesWriterFactory initValuesWriterFactory(Class factoryOverride, ValuesWriterFactoryParams params) { + ValuesWriterFactory factory; + try { + factory = factoryOverride.newInstance(); + } catch (Exception e) { + LOG.error("Falling back to default values writer as we're unable to instantiate ValuesWriterFactory: " + factoryOverride, e); + factory = new DefaultValuesWriterFactory(); + } + factory.initialize(params); + return factory; } public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) { @@ -202,8 +215,7 @@ public static class Builder { private int maxRowCountForPageSizeCheck = DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK; private boolean estimateNextSizeCheck = DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK; private ByteBufferAllocator allocator = new HeapByteBufferAllocator(); - private Map> encodingOverrides = - new HashMap>(); + private Class factoryOverride = DefaultValuesWriterFactory.class; private Builder() { } @@ -292,15 +304,15 @@ public Builder withAllocator(ByteBufferAllocator allocator) { return this; } - public Builder withEncodingOverrides(Map> encodingOverrides) { - this.encodingOverrides = encodingOverrides; + public Builder withFactoryOverride(Class factoryOverride) { + this.factoryOverride = factoryOverride; return this; } public ParquetProperties build() { return new ParquetProperties(writerVersion, pageSize, dictPageSize, enableDict, minRowCountForPageSizeCheck, maxRowCountForPageSizeCheck, - estimateNextSizeCheck, allocator, encodingOverrides); + estimateNextSizeCheck, allocator, factoryOverride); } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java similarity index 55% rename from parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java rename to parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java index e0f43d56f0..9714008249 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.parquet.column.values; +package org.apache.parquet.column.values.factory; -import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; -import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; @@ -36,111 +36,100 @@ import static org.apache.parquet.column.Encoding.PLAIN_DICTIONARY; import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; -public class ValuesWriterFactory { - - private final WriterVersion writerVersion; - - private final boolean enableDictionary; - private final int initialSlabSize; - private final int pageSizeThreshold; - private final ByteBufferAllocator allocator; - private final int dictionaryPageSizeThreshold; - - public ValuesWriterFactory(WriterVersion writerVersion, int initialSlabSize, int pageSizeThreshold, - ByteBufferAllocator allocator, int dictionaryPageSizeThreshold, - boolean enableDictionary) { - this.writerVersion = writerVersion; - this.initialSlabSize = initialSlabSize; - this.pageSizeThreshold = pageSizeThreshold; - this.allocator = allocator; - this.dictionaryPageSizeThreshold = dictionaryPageSizeThreshold; - this.enableDictionary = enableDictionary; +public class DefaultValuesWriterFactory implements ValuesWriterFactory { + + private ValuesWriterFactoryParams selectionParams; + + @Override + public void initialize(ValuesWriterFactoryParams params) { + this.selectionParams = params; } - public ValuesWriter newValuesWriter(ColumnDescriptor path) { - switch (path.getType()) { + @Override + public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { + switch (descriptor.getType()) { case BOOLEAN: return getBooleanValuesWriter(); case FIXED_LEN_BYTE_ARRAY: - return getFixedLenByteArrayValuesWriter(path); + return getFixedLenByteArrayValuesWriter(descriptor); case BINARY: - return getBinaryValuesWriter(path); + return getBinaryValuesWriter(descriptor); case INT32: - return getInt32ValuesWriter(path); + return getInt32ValuesWriter(descriptor); case INT64: - return getInt64ValuesWriter(path); + return getInt64ValuesWriter(descriptor); case INT96: - return getInt96ValuesWriter(path); + return getInt96ValuesWriter(descriptor); case DOUBLE: - return getDoubleValuesWriter(path); + return getDoubleValuesWriter(descriptor); case FLOAT: - return getFloatValuesWriter(path); + return getFloatValuesWriter(descriptor); default: - throw new IllegalArgumentException("Unknown type " + path.getType()); + throw new IllegalArgumentException("Unknown type " + descriptor.getType()); } } private ValuesWriter getBooleanValuesWriter() { // no dictionary encoding for boolean - if(writerVersion == WriterVersion.PARQUET_1_0) { + if(selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { return new BooleanPlainValuesWriter(); } else { - return new RunLengthBitPackingHybridValuesWriter(1, initialSlabSize, pageSizeThreshold, allocator); + return new RunLengthBitPackingHybridValuesWriter(1, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); } } private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { - if (writerVersion == WriterVersion.PARQUET_1_0) { + if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { // dictionary encoding was not enabled in PARQUET 1.0 - return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), initialSlabSize, pageSizeThreshold, allocator); + return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); } else { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } } private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - if (writerVersion == WriterVersion.PARQUET_1_0) { - ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } else { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter(initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } } private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { - if (writerVersion == WriterVersion.PARQUET_1_0) { - ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } else { - ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } } private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { - if (writerVersion == WriterVersion.PARQUET_1_0) { - ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } else { - ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } } private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(initialSlabSize, pageSizeThreshold, allocator); + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); return dictWriterWithFallBack(path, fallbackWriter); } @@ -148,7 +137,7 @@ private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { Encoding encodingForDataPage; Encoding encodingForDictionaryPage; - switch(writerVersion) { + switch(selectionParams.getWriterVersion()) { case PARQUET_1_0: encodingForDataPage = PLAIN_DICTIONARY; encodingForDictionaryPage = PLAIN_DICTIONARY; @@ -158,32 +147,32 @@ private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { encodingForDictionaryPage = PLAIN; break; default: - throw new IllegalArgumentException("Unknown version: " + writerVersion); + throw new IllegalArgumentException("Unknown version: " + selectionParams.getWriterVersion()); } switch (path.getType()) { case BOOLEAN: throw new IllegalArgumentException("no dictionary encoding for BOOLEAN"); case BINARY: - return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); case INT32: - return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); case INT64: - return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); case INT96: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, 12, encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), 12, encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); case DOUBLE: - return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); case FLOAT: - return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(dictionaryPageSizeThreshold, encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); case FIXED_LEN_BYTE_ARRAY: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(dictionaryPageSizeThreshold, path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, allocator); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); default: throw new IllegalArgumentException("Unknown type " + path.getType()); } } private ValuesWriter dictWriterWithFallBack(ColumnDescriptor path, ValuesWriter writerToFallBackTo) { - if (enableDictionary) { + if (selectionParams.getEnableDictionary()) { return FallbackValuesWriter.of( dictionaryWriter(path), writerToFallBackTo); diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java new file mode 100644 index 0000000000..4f4fbdc017 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.factory; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.values.ValuesWriter; + +/** + * Can be overridden to allow users to specify how they want their ValuesWriters to be created. + * ValuesWriterFactories are created using reflection in {@link org.apache.parquet.column.ParquetProperties}. + * ValuesWriterFactories can in turn read additional config to create appropriate ValuesWriters. + */ +public interface ValuesWriterFactory { + + /** + * Used to initialize the factory. This method is called before newValuesWriter() + */ + void initialize(ValuesWriterFactoryParams params); + + /** + * Creates a ValuesWriter to help write the given column. + */ + ValuesWriter newValuesWriter(ColumnDescriptor descriptor); +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java new file mode 100644 index 0000000000..3a1e9cd5e7 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.factory; + +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.column.ParquetProperties.WriterVersion; + +/** + * Encapsulates parameters needed to create new ValuesWriter classes as part of + * ValuesWriterFactory + */ +public class ValuesWriterFactoryParams { + + public ValuesWriterFactoryParams( + WriterVersion writerVersion, + int initialCapacity, + int pageSize, + ByteBufferAllocator allocator, + boolean enableDictionary, + int maxDictionaryByteSize) { + + this.writerVersion = writerVersion; + this.initialCapacity = initialCapacity; + this.pageSize = pageSize; + this.allocator = allocator; + this.enableDictionary = enableDictionary; + this.maxDictionaryByteSize = maxDictionaryByteSize; + } + + private WriterVersion writerVersion; + public WriterVersion getWriterVersion() { + return writerVersion; + } + + private int initialCapacity; + public int getInitialCapacity() { + return initialCapacity; + } + + private int pageSize; + public int getPageSize() { + return pageSize; + } + + private ByteBufferAllocator allocator; + public ByteBufferAllocator getAllocator() { + return allocator; + } + + private int maxDictionaryByteSize; + public int getMaxDictionaryByteSize() { + return maxDictionaryByteSize; + } + + private boolean enableDictionary; + public boolean getEnableDictionary() { + return enableDictionary; + } +} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java similarity index 92% rename from parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java rename to parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java index b689d5ed04..08096b9aa3 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/ValuesWriterFactoryTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java @@ -16,11 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.parquet.column.values; +package org.apache.parquet.column.values.factory; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriter; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; @@ -39,7 +40,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class ValuesWriterFactoryTest { +public class DefaultValuesWriterFactoryTest { @Test public void testBoolean() { @@ -304,15 +305,15 @@ public void testFloat_V2_NoDict() { private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, Class expectedValueWriterClass) { ColumnDescriptor mockPath = getMockColumn(typeName); - ValuesWriterFactory factory = getFactory(version, enableDictionary); - ValuesWriter writer = factory.newValuesWriter(mockPath); + ValuesWriterFactory selectionStrategy = getDefaultFactory(version, enableDictionary); + ValuesWriter writer = selectionStrategy.newValuesWriter(mockPath); validateWriterType(writer, expectedValueWriterClass); } private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, Class initialValueWriterClass, Class fallbackValueWriterClass) { ColumnDescriptor mockPath = getMockColumn(typeName); - ValuesWriterFactory factory = getFactory(version, enableDictionary); + ValuesWriterFactory factory = getDefaultFactory(version, enableDictionary); ValuesWriter writer = factory.newValuesWriter(mockPath); validateFallbackWriter(writer, initialValueWriterClass, fallbackValueWriterClass); @@ -324,8 +325,10 @@ private ColumnDescriptor getMockColumn(PrimitiveTypeName typeName) { return mockPath; } - private ValuesWriterFactory getFactory(WriterVersion writerVersion, boolean enableDictionary) { - return new ValuesWriterFactory(writerVersion, 128, ParquetProperties.DEFAULT_PAGE_SIZE, null, 0, enableDictionary); + private ValuesWriterFactory getDefaultFactory(WriterVersion writerVersion, boolean enableDictionary) { + ValuesWriterFactory factory = new DefaultValuesWriterFactory(); + factory.initialize(new ValuesWriterFactoryParams(writerVersion, 128, ParquetProperties.DEFAULT_PAGE_SIZE, null, enableDictionary, 0)); + return factory; } private void validateWriterType(ValuesWriter writer, Class valuesWriterClass) { diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index b62e90ff88..6a668c476a 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -24,12 +24,7 @@ import static org.apache.parquet.hadoop.util.ContextUtil.getConfiguration; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; @@ -41,16 +36,16 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.parquet.Log; -import org.apache.parquet.column.Encoding; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.hadoop.ParquetFileWriter.Mode; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.hadoop.api.WriteSupport.WriteContext; import org.apache.parquet.hadoop.codec.CodecConfig; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.hadoop.util.ConfigurationUtil; -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; /** * OutputFormat to write to a Parquet file @@ -153,18 +148,16 @@ public static enum JobSummaryLevel { public static final String ESTIMATE_PAGE_SIZE_CHECK = "parquet.page.size.check.estimate"; /** - * Used to override the writer encodings for various types. - * See {@link org.apache.parquet.column.Encoding} for a list of valid Encoding choices to use. - * For e.g. "parquet.writer.encoding-override.boolean" = "plain" will ensure that we use - * {@link org.apache.parquet.column.Encoding.PLAIN} for boolean values. - * We can also specify fallbacks: - * parquet.writer.encoding-override.binary = "plain_dictionary,plain". This results in a - * {@link org.apache.parquet.column.values.fallback.FallbackValuesWriter} with Plain dictionary as - * the initial writer and Plain encoding used as the fallback for binary values. - * Note: If fallbacks are specified, the initial writer must implement - * {@link org.apache.parquet.column.values.RequiresFallback}. + * Used to override the value writer factory used to create Values Writers. + * This allows users to plug in their own factory class to create custom ValuesWriters rather + * use the Parquet provided default - {@link org.apache.parquet.column.values.factory.DefaultValuesWriterFactory}. + * Factory classes provided must implement {@link org.apache.parquet.column.values.factory.ValuesWriterFactory}. + * + * For e.g. "parquet.writer.factory-override" = "org.apache.parquet.column.values.factory.MyValuesWriterFactory" + * will ensure that we use MyValuesWriterFactory to create ValuesWriters for writing out data. + * See the documentation for ValuesWriterFactory for more details. */ - public static final String WRITER_ENCODING_OVERRIDE_PREFIX = "parquet.writer.encoding-override."; + public static final String WRITER_FACTORY_OVERRIDE = "parquet.writer.factory-override"; // default to no padding for now private static final int DEFAULT_MAX_PADDING_SIZE = 0; @@ -338,39 +331,16 @@ private static int getMaxPaddingSize(Configuration conf) { return conf.getInt(MAX_PADDING_BYTES, DEFAULT_MAX_PADDING_SIZE); } - public static Map> getEncodingOverrides(Configuration conf) { - Map> typeToEncoding = new HashMap>(); - - for(PrimitiveTypeName name : PrimitiveTypeName.values()) { - String typeOverride = conf.get(WRITER_ENCODING_OVERRIDE_PREFIX + name.name().toLowerCase()); - typeToEncoding.put(name, getEncodingOverridesForType(name, typeOverride)); - } - - return typeToEncoding; - } - - private static List getEncodingOverridesForType(PrimitiveTypeName typeName, String typeOverride) { - List encodings = new ArrayList(); - if( StringUtils.isEmpty(typeOverride) ) { - return encodings; - } - - String [] overrides = typeOverride.split(","); - if( overrides.length > 2 ) { - //maybe in the future we could chain more - throw new BadConfigurationException("For : " + typeName + " too many overrides specified, must not be more than 2"); - } - - for(String override : overrides) { - try { - Encoding encoding = Encoding.valueOf(override.toUpperCase()); - encodings.add(encoding); - } catch(IllegalArgumentException e) { - throw new BadConfigurationException("For type: " + typeName + " Invalid encoding type chosen: " + override); - } + public static Class getFactoryOverride(Configuration conf) { + Class factoryClass = DefaultValuesWriterFactory.class; + try { + factoryClass = conf.getClass(WRITER_FACTORY_OVERRIDE, DefaultValuesWriterFactory.class, ValuesWriterFactory.class); + } catch (Exception e) { + // can be due to the class not being found / incorrect parent interface + LOG.warn("Unable to load factory override. Falling back to default factory", e); } - return encodings; + return factoryClass; } private WriteSupport writeSupport; @@ -423,7 +393,7 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) - .withEncodingOverrides(getEncodingOverrides(conf)) + .withFactoryOverride(getFactoryOverride(conf)) .build(); long blockSize = getLongBlockSize(conf); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java new file mode 100644 index 0000000000..cb2d47f4b2 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.factory.ValuesWriterFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; + +/** + * ValuesWriter factory test class for verifying overrides + */ +public class StubValuesWriterFactory implements ValuesWriterFactory { + @Override + public void initialize(ValuesWriterFactoryParams params) {} + + @Override + public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { + return null; + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java deleted file mode 100644 index e088a747fc..0000000000 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatEncodingOverrides.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.hadoop; - -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.parquet.column.Encoding; -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestParquetOutputFormatEncodingOverrides { - - @Test - public void testEmptyOverrideString() { - Configuration conf = new Configuration(); - Map> encodings = ParquetOutputFormat.getEncodingOverrides(conf); - assertEquals("Incorrect number of encoding entries", PrimitiveTypeName.values().length, encodings.size()); - - for(Map.Entry> entry : encodings.entrySet()) { - assertTrue("Non-empty encoding list for: " + entry.getKey(), entry.getValue().isEmpty()); - } - } - - @Test - public void testOneValidEncoding() { - Configuration conf = new Configuration(); - conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), Encoding.PLAIN.name()); - - Map> encodings = ParquetOutputFormat.getEncodingOverrides(conf); - List encodingList = encodings.get(PrimitiveTypeName.BOOLEAN); - assertTrue(encodingList.size() == 1); - assertEquals(Encoding.PLAIN, encodingList.get(0)); - } - - @Test(expected = BadConfigurationException.class) - public void testOneBrokenEncoding() { - Configuration conf = new Configuration(); - conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), "foo-encoding"); - - // should fail due to an exception - ParquetOutputFormat.getEncodingOverrides(conf); - } - - @Test(expected = BadConfigurationException.class) - public void testOneGood_OneBrokenEncoding() { - Configuration conf = new Configuration(); - String encodingStr = Encoding.PLAIN.name() + "," + "foo-encoding"; - conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), encodingStr); - - // should fail due to an exception - ParquetOutputFormat.getEncodingOverrides(conf); - } - - @Test - public void testTwoEncodings() { - Configuration conf = new Configuration(); - String encodingStr = Encoding.PLAIN_DICTIONARY.name() + "," + Encoding.PLAIN; - conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), encodingStr); - - Map> encodings = ParquetOutputFormat.getEncodingOverrides(conf); - List encodingList = encodings.get(PrimitiveTypeName.BOOLEAN); - assertTrue(encodingList.size() == 2); - assertEquals(Encoding.PLAIN_DICTIONARY, encodingList.get(0)); - assertEquals(Encoding.PLAIN, encodingList.get(1)); - } - - @Test(expected = BadConfigurationException.class) - public void testThreeEncodings() { - Configuration conf = new Configuration(); - String encodingStr = Encoding.PLAIN_DICTIONARY.name() + "," + Encoding.PLAIN + "," + Encoding.DELTA_BINARY_PACKED; - conf.set(ParquetOutputFormat.WRITER_ENCODING_OVERRIDE_PREFIX + PrimitiveTypeName.BOOLEAN.name().toLowerCase(), encodingStr); - - // should fail due to an exception - ParquetOutputFormat.getEncodingOverrides(conf); - } -} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java new file mode 100644 index 0000000000..ca21fea74d --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestParquetOutputFormatFactoryOverrides { + + @Test + public void testMissingOverrideParam() { + Configuration conf = new Configuration(); + Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); + assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factoryOverride); + } + + @Test + public void testMissingFactoryOverrideClass() { + Configuration conf = new Configuration(); + conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "MyMissingFactory"); + Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); + assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factoryOverride); + } + + @Test + public void testFactoryClassIncorrectInterface() { + Configuration conf = new Configuration(); + conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.column.values.factory.ValuesWriterFactoryParams"); + Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); + assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factoryOverride); + } + + @Test + public void testValidFactoryOverride() { + Configuration conf = new Configuration(); + conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.hadoop.StubValuesWriterFactory"); + Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); + assertEquals("Incorrect factory override chosen", StubValuesWriterFactory.class, factoryOverride); + } +} From e4b61a45e5cf52cb6008490b0a139ffa5d188a2d Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Tue, 17 May 2016 16:43:57 -0700 Subject: [PATCH 08/15] Tweak factory instantiation a bit --- .../parquet/column/ParquetProperties.java | 29 +++++-------------- .../parquet/hadoop/ParquetOutputFormat.java | 18 +++++++----- ...stParquetOutputFormatFactoryOverrides.java | 23 ++++++++------- 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index d3dc580601..f35fd41894 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -18,7 +18,6 @@ */ package org.apache.parquet.column; -import org.apache.parquet.Log; import org.apache.parquet.Preconditions; import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.bytes.CapacityByteArrayOutputStream; @@ -52,11 +51,10 @@ public class ParquetProperties { public static final boolean DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK = true; public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100; public static final int DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000; + public static final Class DEFAULT_VALUES_WRITER_FACTORY = DefaultValuesWriterFactory.class; private static final int MIN_SLAB_SIZE = 64; - private static final Log LOG = Log.getLog(ParquetProperties.class); - public enum WriterVersion { PARQUET_1_0 ("v1"), PARQUET_2_0 ("v2"); @@ -90,7 +88,7 @@ public static WriterVersion fromString(String name) { private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPageSize, boolean enableDict, int minRowCountForPageSizeCheck, int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator, - Class factoryOverride) { + ValuesWriterFactory writerFactory) { this.pageSizeThreshold = pageSize; int initialSlabSize = CapacityByteArrayOutputStream .initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10); @@ -105,19 +103,8 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag ValuesWriterFactoryParams params = new ValuesWriterFactoryParams(writerVersion, initialSlabSize, pageSizeThreshold, allocator, enableDictionary, dictionaryPageSizeThreshold); - this.valuesWriterFactory = initValuesWriterFactory(factoryOverride, params); - } - - private ValuesWriterFactory initValuesWriterFactory(Class factoryOverride, ValuesWriterFactoryParams params) { - ValuesWriterFactory factory; - try { - factory = factoryOverride.newInstance(); - } catch (Exception e) { - LOG.error("Falling back to default values writer as we're unable to instantiate ValuesWriterFactory: " + factoryOverride, e); - factory = new DefaultValuesWriterFactory(); - } - factory.initialize(params); - return factory; + valuesWriterFactory = writerFactory; + valuesWriterFactory.initialize(params); } public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) { @@ -215,7 +202,7 @@ public static class Builder { private int maxRowCountForPageSizeCheck = DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK; private boolean estimateNextSizeCheck = DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK; private ByteBufferAllocator allocator = new HeapByteBufferAllocator(); - private Class factoryOverride = DefaultValuesWriterFactory.class; + private ValuesWriterFactory valuesWriterFactory = new DefaultValuesWriterFactory(); private Builder() { } @@ -304,15 +291,15 @@ public Builder withAllocator(ByteBufferAllocator allocator) { return this; } - public Builder withFactoryOverride(Class factoryOverride) { - this.factoryOverride = factoryOverride; + public Builder withValuesWriterFactory(ValuesWriterFactory factory) { + this.valuesWriterFactory = factory; return this; } public ParquetProperties build() { return new ParquetProperties(writerVersion, pageSize, dictPageSize, enableDict, minRowCountForPageSizeCheck, maxRowCountForPageSizeCheck, - estimateNextSizeCheck, allocator, factoryOverride); + estimateNextSizeCheck, allocator, valuesWriterFactory); } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 6a668c476a..0820451bea 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -331,16 +331,18 @@ private static int getMaxPaddingSize(Configuration conf) { return conf.getInt(MAX_PADDING_BYTES, DEFAULT_MAX_PADDING_SIZE); } - public static Class getFactoryOverride(Configuration conf) { - Class factoryClass = DefaultValuesWriterFactory.class; + public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) { + Class factoryOverride = + (Class) ConfigurationUtil.getClassFromConfig(conf, WRITER_FACTORY_OVERRIDE, ValuesWriterFactory.class); + if (factoryOverride == null) { + factoryOverride = ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY; + } + try { - factoryClass = conf.getClass(WRITER_FACTORY_OVERRIDE, DefaultValuesWriterFactory.class, ValuesWriterFactory.class); + return factoryOverride.newInstance(); } catch (Exception e) { - // can be due to the class not being found / incorrect parent interface - LOG.warn("Unable to load factory override. Falling back to default factory", e); + throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e); } - - return factoryClass; } private WriteSupport writeSupport; @@ -393,7 +395,7 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) - .withFactoryOverride(getFactoryOverride(conf)) + .withValuesWriterFactory(getValuesWriterFactory(conf)) .build(); long blockSize = getLongBlockSize(conf); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java index ca21fea74d..06a4b523c3 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -29,31 +30,33 @@ public class TestParquetOutputFormatFactoryOverrides { @Test public void testMissingOverrideParam() { Configuration conf = new Configuration(); - Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); - assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factoryOverride); + ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); + assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factory.getClass()); } - @Test + @Test(expected = BadConfigurationException.class) public void testMissingFactoryOverrideClass() { Configuration conf = new Configuration(); conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "MyMissingFactory"); - Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); - assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factoryOverride); + + // should throw as we can't find MyMissingFactory + ParquetOutputFormat.getValuesWriterFactory(conf); } - @Test + @Test(expected = BadConfigurationException.class) public void testFactoryClassIncorrectInterface() { Configuration conf = new Configuration(); conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.column.values.factory.ValuesWriterFactoryParams"); - Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); - assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factoryOverride); + + // should throw as ValuesWriterFactoryParams isn't implementing ValuesWriterFactory + ParquetOutputFormat.getValuesWriterFactory(conf); } @Test public void testValidFactoryOverride() { Configuration conf = new Configuration(); conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.hadoop.StubValuesWriterFactory"); - Class factoryOverride = ParquetOutputFormat.getFactoryOverride(conf); - assertEquals("Incorrect factory override chosen", StubValuesWriterFactory.class, factoryOverride); + ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); + assertEquals("Incorrect factory override chosen", StubValuesWriterFactory.class, factory.getClass()); } } From 8a852a3aa27ca0cde97c9ba29a9502955dbca815 Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Tue, 17 May 2016 16:56:09 -0700 Subject: [PATCH 09/15] Log values writer factory chosen --- .../java/org/apache/parquet/column/ParquetProperties.java | 4 ++++ .../java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 2 ++ 2 files changed, 6 insertions(+) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index f35fd41894..73437d12a6 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -181,6 +181,10 @@ public int getMaxRowCountForPageSizeCheck() { return maxRowCountForPageSizeCheck; } + public ValuesWriterFactory getValuesWriterFactory() { + return valuesWriterFactory; + } + public boolean estimateNextSizeCheck() { return estimateNextSizeCheck; } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 0820451bea..fd6dffafaf 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -335,6 +335,7 @@ public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) { Class factoryOverride = (Class) ConfigurationUtil.getClassFromConfig(conf, WRITER_FACTORY_OVERRIDE, ValuesWriterFactory.class); if (factoryOverride == null) { + // no override specified, fall back to default factoryOverride = ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY; } @@ -413,6 +414,7 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp LOG.info("Page size checking is: " + (props.estimateNextSizeCheck() ? "estimated" : "constant")); LOG.info("Min row count for page size check is: " + props.getMinRowCountForPageSizeCheck()); LOG.info("Max row count for page size check is: " + props.getMaxRowCountForPageSizeCheck()); + LOG.info("ValueWriterFactory chosen is: " + props.getValuesWriterFactory()); } WriteContext init = writeSupport.init(conf); From bf4bc6de7b5ee3221ea239fd48b9d8780674be5f Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Wed, 18 May 2016 13:21:26 -0700 Subject: [PATCH 10/15] Add support for Config setting in ValuesWriter factory --- .../DefaultValuesWriterFactoryTest.java | 4 +- .../values/factory/ConfigurableFactory.java | 31 +++++++++++ .../parquet/hadoop/ParquetOutputFormat.java | 10 +++- .../StubConfigurableValuesWriterFactory.java | 53 +++++++++++++++++++ ...stParquetOutputFormatFactoryOverrides.java | 16 ++++++ 5 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java index 08096b9aa3..c8d4aaf291 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java @@ -305,8 +305,8 @@ public void testFloat_V2_NoDict() { private void doTestValueWriter(PrimitiveTypeName typeName, WriterVersion version, boolean enableDictionary, Class expectedValueWriterClass) { ColumnDescriptor mockPath = getMockColumn(typeName); - ValuesWriterFactory selectionStrategy = getDefaultFactory(version, enableDictionary); - ValuesWriter writer = selectionStrategy.newValuesWriter(mockPath); + ValuesWriterFactory factory = getDefaultFactory(version, enableDictionary); + ValuesWriter writer = factory.newValuesWriter(mockPath); validateWriterType(writer, expectedValueWriterClass); } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java b/parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java new file mode 100644 index 0000000000..bf389c8d8b --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.factory; + +import org.apache.hadoop.conf.Configuration; + +/** + * Used with {@link ValuesWriterFactory} to indicate a factory that has Hadoop config included. + */ +public interface ConfigurableFactory { + + void setConfiguration(Configuration config); + + Configuration getConfiguration(); +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index fd6dffafaf..c49ab26cce 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -38,7 +38,7 @@ import org.apache.parquet.Log; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; -import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; +import org.apache.parquet.column.values.factory.ConfigurableFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.hadoop.ParquetFileWriter.Mode; import org.apache.parquet.hadoop.api.WriteSupport; @@ -340,7 +340,13 @@ public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) { } try { - return factoryOverride.newInstance(); + ValuesWriterFactory factory = factoryOverride.newInstance(); + if (factory instanceof ConfigurableFactory) { + ConfigurableFactory configurableFactory = (ConfigurableFactory) factory; + configurableFactory.setConfiguration(conf); + } + + return factory; } catch (Exception e) { throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e); } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java new file mode 100644 index 0000000000..77b3c40101 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.factory.ConfigurableFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactory; +import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; + +/** + * ValuesWriter factory test class for verifying overrides and configuration + */ +public class StubConfigurableValuesWriterFactory implements ValuesWriterFactory, ConfigurableFactory { + + private Configuration config; + + @Override + public void setConfiguration(Configuration config) { + this.config = config; + } + + @Override + public Configuration getConfiguration() { + return config; + } + + @Override + public void initialize(ValuesWriterFactoryParams params) { + } + + @Override + public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { + return null; + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java index 06a4b523c3..ae46e70c9d 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java @@ -19,11 +19,13 @@ package org.apache.parquet.hadoop; import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.column.values.factory.ConfigurableFactory; import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class TestParquetOutputFormatFactoryOverrides { @@ -59,4 +61,18 @@ public void testValidFactoryOverride() { ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); assertEquals("Incorrect factory override chosen", StubValuesWriterFactory.class, factory.getClass()); } + + @Test + public void testFactoryOverrideWithCfg() { + Configuration conf = new Configuration(); + conf.set("foo", "bar"); + conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.hadoop.StubConfigurableValuesWriterFactory"); + + ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); + assertEquals("Incorrect factory override chosen", StubConfigurableValuesWriterFactory.class, factory.getClass()); + + ConfigurableFactory configurable = (ConfigurableFactory) factory; + assertNotNull("Not a ConfigurableFactory", configurable); + assertEquals("Incorrect config value found", "bar", configurable.getConfiguration().get("foo")); + } } From cb02ea09c14f518838a44c42744c5f36adf86fdd Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Fri, 17 Jun 2016 18:10:50 -0700 Subject: [PATCH 11/15] Fix review comments --- .../parquet/column/ParquetProperties.java | 8 +- .../factory/DefaultV1ValuesWriterFactory.java | 110 +++++++++++++ .../factory/DefaultV2ValuesWriterFactory.java | 114 ++++++++++++++ .../factory/DefaultValuesWriterFactory.java | 145 +++--------------- .../values/factory/ValuesWriterFactory.java | 9 +- .../factory/ValuesWriterFactoryParams.java | 19 +-- .../DefaultValuesWriterFactoryTest.java | 6 +- .../values/factory/ConfigurableFactory.java | 31 ---- .../parquet/hadoop/ParquetOutputFormat.java | 24 +-- .../StubConfigurableValuesWriterFactory.java | 8 +- ...stParquetOutputFormatFactoryOverrides.java | 6 +- 11 files changed, 296 insertions(+), 184 deletions(-) create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java delete mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 73437d12a6..1748e14595 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -29,9 +29,9 @@ import org.apache.parquet.column.page.PageWriteStore; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.bitpacking.DevNullValuesWriter; +import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; -import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; import org.apache.parquet.schema.MessageType; @@ -51,7 +51,8 @@ public class ParquetProperties { public static final boolean DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK = true; public static final int DEFAULT_MINIMUM_RECORD_COUNT_FOR_CHECK = 100; public static final int DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK = 10000; - public static final Class DEFAULT_VALUES_WRITER_FACTORY = DefaultValuesWriterFactory.class; + + public static final ValuesWriterFactory DEFAULT_VALUES_WRITER_FACTORY = new DefaultValuesWriterFactory(); private static final int MIN_SLAB_SIZE = 64; @@ -206,7 +207,7 @@ public static class Builder { private int maxRowCountForPageSizeCheck = DEFAULT_MAXIMUM_RECORD_COUNT_FOR_CHECK; private boolean estimateNextSizeCheck = DEFAULT_ESTIMATE_ROW_COUNT_FOR_PAGE_SIZE_CHECK; private ByteBufferAllocator allocator = new HeapByteBufferAllocator(); - private ValuesWriterFactory valuesWriterFactory = new DefaultValuesWriterFactory(); + private ValuesWriterFactory valuesWriterFactory = DEFAULT_VALUES_WRITER_FACTORY; private Builder() { } @@ -296,6 +297,7 @@ public Builder withAllocator(ByteBufferAllocator allocator) { } public Builder withValuesWriterFactory(ValuesWriterFactory factory) { + Preconditions.checkNotNull(factory, "ValuesWriterFactory"); this.valuesWriterFactory = factory; return this; } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java new file mode 100644 index 0000000000..01530cff19 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.factory; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter; +import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; +import org.apache.parquet.column.values.plain.PlainValuesWriter; + +import static org.apache.parquet.column.Encoding.PLAIN_DICTIONARY; + +public class DefaultV1ValuesWriterFactory implements ValuesWriterFactory { + + private ValuesWriterFactoryParams selectionParams; + + @Override + public void initialize(ValuesWriterFactoryParams params) { + this.selectionParams = params; + } + + private Encoding getEncodingForDataPage() { + return PLAIN_DICTIONARY; + } + + private Encoding getEncodingForDictionaryPage() { + return PLAIN_DICTIONARY; + } + + @Override + public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { + switch (descriptor.getType()) { + case BOOLEAN: + return getBooleanValuesWriter(); + case FIXED_LEN_BYTE_ARRAY: + return getFixedLenByteArrayValuesWriter(descriptor); + case BINARY: + return getBinaryValuesWriter(descriptor); + case INT32: + return getInt32ValuesWriter(descriptor); + case INT64: + return getInt64ValuesWriter(descriptor); + case INT96: + return getInt96ValuesWriter(descriptor); + case DOUBLE: + return getDoubleValuesWriter(descriptor); + case FLOAT: + return getFloatValuesWriter(descriptor); + default: + throw new IllegalArgumentException("Unknown type " + descriptor.getType()); + } + } + + private ValuesWriter getBooleanValuesWriter() { + // no dictionary encoding for boolean + return new BooleanPlainValuesWriter(); + } + + private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { + // dictionary encoding was not enabled in PARQUET 1.0 + return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + } + + private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java new file mode 100644 index 0000000000..50635a9e93 --- /dev/null +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.values.factory; + +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; +import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; +import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; +import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; +import org.apache.parquet.column.values.plain.PlainValuesWriter; +import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; + +import static org.apache.parquet.column.Encoding.PLAIN; +import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; + +public class DefaultV2ValuesWriterFactory implements ValuesWriterFactory { + + private ValuesWriterFactoryParams selectionParams; + + @Override + public void initialize(ValuesWriterFactoryParams params) { + this.selectionParams = params; + } + + private Encoding getEncodingForDataPage() { + return RLE_DICTIONARY; + } + + private Encoding getEncodingForDictionaryPage() { + return PLAIN; + } + + @Override + public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { + switch (descriptor.getType()) { + case BOOLEAN: + return getBooleanValuesWriter(); + case FIXED_LEN_BYTE_ARRAY: + return getFixedLenByteArrayValuesWriter(descriptor); + case BINARY: + return getBinaryValuesWriter(descriptor); + case INT32: + return getInt32ValuesWriter(descriptor); + case INT64: + return getInt64ValuesWriter(descriptor); + case INT96: + return getInt96ValuesWriter(descriptor); + case DOUBLE: + return getDoubleValuesWriter(descriptor); + case FLOAT: + return getFloatValuesWriter(descriptor); + default: + throw new IllegalArgumentException("Unknown type " + descriptor.getType()); + } + } + + private ValuesWriter getBooleanValuesWriter() { + // no dictionary encoding for boolean + return new RunLengthBitPackingHybridValuesWriter(1, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + } + + private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } + + private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { + ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + } +} diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java index 9714008249..80aff8b4fe 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java @@ -20,161 +20,66 @@ import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; -import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.column.ParquetProperties.WriterVersion; import org.apache.parquet.column.values.ValuesWriter; -import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; -import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; -import org.apache.parquet.column.values.deltastrings.DeltaByteArrayWriter; import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; import org.apache.parquet.column.values.fallback.FallbackValuesWriter; -import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter; -import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; -import org.apache.parquet.column.values.plain.PlainValuesWriter; -import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; - -import static org.apache.parquet.column.Encoding.PLAIN; -import static org.apache.parquet.column.Encoding.PLAIN_DICTIONARY; -import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; +/** + * Handles ValuesWriter creation statically based on the types of the columns and the writer version. + */ public class DefaultValuesWriterFactory implements ValuesWriterFactory { private ValuesWriterFactoryParams selectionParams; + private ValuesWriterFactory delegateFactory; + + private static final ValuesWriterFactory DEFAULT_V1_WRITER_FACTORY = new DefaultV1ValuesWriterFactory(); + private static final ValuesWriterFactory DEFAULT_V2_WRITER_FACTORY = new DefaultV2ValuesWriterFactory(); @Override public void initialize(ValuesWriterFactoryParams params) { this.selectionParams = params; - } - - @Override - public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { - switch (descriptor.getType()) { - case BOOLEAN: - return getBooleanValuesWriter(); - case FIXED_LEN_BYTE_ARRAY: - return getFixedLenByteArrayValuesWriter(descriptor); - case BINARY: - return getBinaryValuesWriter(descriptor); - case INT32: - return getInt32ValuesWriter(descriptor); - case INT64: - return getInt64ValuesWriter(descriptor); - case INT96: - return getInt96ValuesWriter(descriptor); - case DOUBLE: - return getDoubleValuesWriter(descriptor); - case FLOAT: - return getFloatValuesWriter(descriptor); - default: - throw new IllegalArgumentException("Unknown type " + descriptor.getType()); - } - } - - private ValuesWriter getBooleanValuesWriter() { - // no dictionary encoding for boolean - if(selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { - return new BooleanPlainValuesWriter(); - } else { - return new RunLengthBitPackingHybridValuesWriter(1, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - } - } - - private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { - if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { - // dictionary encoding was not enabled in PARQUET 1.0 - return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + if (selectionParams.getWriterVersion() == WriterVersion.PARQUET_1_0) { + delegateFactory = DEFAULT_V1_WRITER_FACTORY; } else { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); + delegateFactory = DEFAULT_V2_WRITER_FACTORY; } - } - private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } else { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } + delegateFactory.initialize(params); } - private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { - if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } else { - ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } - } - - private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { - if (selectionParams.getWriterVersion() == ParquetProperties.WriterVersion.PARQUET_1_0) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } else { - ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } - } - - private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } - - private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); - } - - private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return dictWriterWithFallBack(path, fallbackWriter); + @Override + public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { + return delegateFactory.newValuesWriter(descriptor); } - @SuppressWarnings("deprecation") - private DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path) { - Encoding encodingForDataPage; - Encoding encodingForDictionaryPage; - switch(selectionParams.getWriterVersion()) { - case PARQUET_1_0: - encodingForDataPage = PLAIN_DICTIONARY; - encodingForDictionaryPage = PLAIN_DICTIONARY; - break; - case PARQUET_2_0: - encodingForDataPage = RLE_DICTIONARY; - encodingForDictionaryPage = PLAIN; - break; - default: - throw new IllegalArgumentException("Unknown version: " + selectionParams.getWriterVersion()); - } + public static DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path, ValuesWriterFactoryParams selectionParams, Encoding dictPageEncoding, Encoding dataPageEncoding) { switch (path.getType()) { case BOOLEAN: throw new IllegalArgumentException("no dictionary encoding for BOOLEAN"); case BINARY: - return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); case INT32: - return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); case INT64: - return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); case INT96: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), 12, encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), 12, dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); case DOUBLE: - return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); case FLOAT: - return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); case FIXED_LEN_BYTE_ARRAY: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), path.getTypeLength(), encodingForDataPage, encodingForDictionaryPage, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), path.getTypeLength(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); default: throw new IllegalArgumentException("Unknown type " + path.getType()); } } - private ValuesWriter dictWriterWithFallBack(ColumnDescriptor path, ValuesWriter writerToFallBackTo) { + public static ValuesWriter dictWriterWithFallBack(ColumnDescriptor path, ValuesWriterFactoryParams selectionParams, Encoding dictPageEncoding, Encoding dataPageEncoding, ValuesWriter writerToFallBackTo) { if (selectionParams.getEnableDictionary()) { return FallbackValuesWriter.of( - dictionaryWriter(path), + dictionaryWriter(path, selectionParams, dictPageEncoding, dataPageEncoding), writerToFallBackTo); } else { return writerToFallBackTo; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java index 4f4fbdc017..6610593689 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java @@ -24,7 +24,14 @@ /** * Can be overridden to allow users to specify how they want their ValuesWriters to be created. * ValuesWriterFactories are created using reflection in {@link org.apache.parquet.column.ParquetProperties}. - * ValuesWriterFactories can in turn read additional config to create appropriate ValuesWriters. + * Due to this, they must provide a default constructor. + * Lifecycle of ValuesWriterFactories is: + * 1) Created via reflection while creating a {@link org.apache.parquet.column.ParquetProperties} + * 2) If the factory is Configurable (needs Hadoop conf), that is set, initialize is also called. This is done + * just once for the lifetime of the factory. As Hadoop config can be set, ValuesWriterFactories can + * read additional config to create appropriate ValuesWriters. + * 3) newValuesWriter is called once per column for every block of data. + * */ public interface ValuesWriterFactory { diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java index 3a1e9cd5e7..566eb1155b 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java @@ -20,6 +20,7 @@ import org.apache.parquet.bytes.ByteBufferAllocator; import org.apache.parquet.column.ParquetProperties.WriterVersion; +import static org.apache.parquet.Preconditions.checkNotNull; /** * Encapsulates parameters needed to create new ValuesWriter classes as part of @@ -34,41 +35,41 @@ public ValuesWriterFactoryParams( ByteBufferAllocator allocator, boolean enableDictionary, int maxDictionaryByteSize) { - - this.writerVersion = writerVersion; + this.writerVersion = checkNotNull(writerVersion, "writerVersion"); this.initialCapacity = initialCapacity; this.pageSize = pageSize; - this.allocator = allocator; + this.allocator = checkNotNull(allocator, "allocator"); this.enableDictionary = enableDictionary; this.maxDictionaryByteSize = maxDictionaryByteSize; } - private WriterVersion writerVersion; + private final WriterVersion writerVersion; + private final boolean enableDictionary; + private final int initialCapacity; + private final int pageSize; + private final ByteBufferAllocator allocator; + private final int maxDictionaryByteSize; + public WriterVersion getWriterVersion() { return writerVersion; } - private int initialCapacity; public int getInitialCapacity() { return initialCapacity; } - private int pageSize; public int getPageSize() { return pageSize; } - private ByteBufferAllocator allocator; public ByteBufferAllocator getAllocator() { return allocator; } - private int maxDictionaryByteSize; public int getMaxDictionaryByteSize() { return maxDictionaryByteSize; } - private boolean enableDictionary; public boolean getEnableDictionary() { return enableDictionary; } diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java index c8d4aaf291..ab4dfc187d 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java @@ -18,6 +18,8 @@ */ package org.apache.parquet.column.values.factory; +import org.apache.parquet.bytes.ByteBufferAllocator; +import org.apache.parquet.bytes.HeapByteBufferAllocator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; @@ -42,6 +44,8 @@ public class DefaultValuesWriterFactoryTest { + private static final ByteBufferAllocator ALLOCATOR = new HeapByteBufferAllocator(); + @Test public void testBoolean() { doTestValueWriter( @@ -327,7 +331,7 @@ private ColumnDescriptor getMockColumn(PrimitiveTypeName typeName) { private ValuesWriterFactory getDefaultFactory(WriterVersion writerVersion, boolean enableDictionary) { ValuesWriterFactory factory = new DefaultValuesWriterFactory(); - factory.initialize(new ValuesWriterFactoryParams(writerVersion, 128, ParquetProperties.DEFAULT_PAGE_SIZE, null, enableDictionary, 0)); + factory.initialize(new ValuesWriterFactoryParams(writerVersion, 128, ParquetProperties.DEFAULT_PAGE_SIZE, ALLOCATOR, enableDictionary, 0)); return factory; } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java b/parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java deleted file mode 100644 index bf389c8d8b..0000000000 --- a/parquet-hadoop/src/main/java/org/apache/parquet/column/values/factory/ConfigurableFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.column.values.factory; - -import org.apache.hadoop.conf.Configuration; - -/** - * Used with {@link ValuesWriterFactory} to indicate a factory that has Hadoop config included. - */ -public interface ConfigurableFactory { - - void setConfiguration(Configuration config); - - Configuration getConfiguration(); -} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index c49ab26cce..b646866538 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -25,6 +25,7 @@ import java.io.IOException; +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; @@ -38,7 +39,6 @@ import org.apache.parquet.Log; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; -import org.apache.parquet.column.values.factory.ConfigurableFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.hadoop.ParquetFileWriter.Mode; import org.apache.parquet.hadoop.api.WriteSupport; @@ -153,11 +153,11 @@ public static enum JobSummaryLevel { * use the Parquet provided default - {@link org.apache.parquet.column.values.factory.DefaultValuesWriterFactory}. * Factory classes provided must implement {@link org.apache.parquet.column.values.factory.ValuesWriterFactory}. * - * For e.g. "parquet.writer.factory-override" = "org.apache.parquet.column.values.factory.MyValuesWriterFactory" + * For e.g. "parquet.write.value.writer.factory" = "org.apache.parquet.column.values.factory.MyValuesWriterFactory" * will ensure that we use MyValuesWriterFactory to create ValuesWriters for writing out data. * See the documentation for ValuesWriterFactory for more details. */ - public static final String WRITER_FACTORY_OVERRIDE = "parquet.writer.factory-override"; + public static final String WRITER_FACTORY_OVERRIDE = "parquet.write.value.writer.factory"; // default to no padding for now private static final int DEFAULT_MAX_PADDING_SIZE = 0; @@ -334,20 +334,20 @@ private static int getMaxPaddingSize(Configuration conf) { public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) { Class factoryOverride = (Class) ConfigurationUtil.getClassFromConfig(conf, WRITER_FACTORY_OVERRIDE, ValuesWriterFactory.class); - if (factoryOverride == null) { - // no override specified, fall back to default - factoryOverride = ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY; - } try { - ValuesWriterFactory factory = factoryOverride.newInstance(); - if (factory instanceof ConfigurableFactory) { - ConfigurableFactory configurableFactory = (ConfigurableFactory) factory; - configurableFactory.setConfiguration(conf); + ValuesWriterFactory factory = + factoryOverride == null ? ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY : factoryOverride.newInstance() ; + + if (factory instanceof Configurable) { + Configurable configurableFactory = (Configurable) factory; + configurableFactory.setConf(conf); } return factory; - } catch (Exception e) { + } catch (InstantiationException e) { + throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e); + } catch (IllegalAccessException e) { throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e); } } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java index 77b3c40101..4420e93ad5 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java @@ -18,27 +18,27 @@ */ package org.apache.parquet.hadoop; +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.values.ValuesWriter; -import org.apache.parquet.column.values.factory.ConfigurableFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; /** * ValuesWriter factory test class for verifying overrides and configuration */ -public class StubConfigurableValuesWriterFactory implements ValuesWriterFactory, ConfigurableFactory { +public class StubConfigurableValuesWriterFactory implements ValuesWriterFactory, Configurable { private Configuration config; @Override - public void setConfiguration(Configuration config) { + public void setConf(Configuration config) { this.config = config; } @Override - public Configuration getConfiguration() { + public Configuration getConf() { return config; } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java index ae46e70c9d..bd3b437295 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java @@ -18,8 +18,8 @@ */ package org.apache.parquet.hadoop; +import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.parquet.column.values.factory.ConfigurableFactory; import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.junit.Test; @@ -71,8 +71,8 @@ public void testFactoryOverrideWithCfg() { ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); assertEquals("Incorrect factory override chosen", StubConfigurableValuesWriterFactory.class, factory.getClass()); - ConfigurableFactory configurable = (ConfigurableFactory) factory; + Configurable configurable = (Configurable) factory; assertNotNull("Not a ConfigurableFactory", configurable); - assertEquals("Incorrect config value found", "bar", configurable.getConfiguration().get("foo")); + assertEquals("Incorrect config value found", "bar", configurable.getConf().get("foo")); } } From f021ed2ec1aa73dc9c45b591cf72efd0660b27b0 Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Thu, 30 Jun 2016 14:54:40 -0700 Subject: [PATCH 12/15] Tweak comment in ValuesWriterFactory --- .../parquet/column/values/factory/ValuesWriterFactory.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java index 6610593689..9d4e2d6ffa 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java @@ -27,11 +27,10 @@ * Due to this, they must provide a default constructor. * Lifecycle of ValuesWriterFactories is: * 1) Created via reflection while creating a {@link org.apache.parquet.column.ParquetProperties} - * 2) If the factory is Configurable (needs Hadoop conf), that is set, initialize is also called. This is done - * just once for the lifetime of the factory. As Hadoop config can be set, ValuesWriterFactories can - * read additional config to create appropriate ValuesWriters. + * 2) If the factory is Configurable (needs Hadoop conf), that is set by calling setConfig(). + * initialize() is also called. This is done just once for the lifetime of the factory. As Hadoop + * config can be set, ValuesWriterFactories can read additional config to create appropriate ValuesWriters. * 3) newValuesWriter is called once per column for every block of data. - * */ public interface ValuesWriterFactory { From 0b78e04d126f96dfbc8e10ea1130c3cf7cf33bc1 Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Tue, 2 Aug 2016 18:48:25 -0700 Subject: [PATCH 13/15] Address Ryan's feedback --- .../parquet/column/ParquetProperties.java | 24 ++++-- .../factory/DefaultV1ValuesWriterFactory.java | 33 ++++---- .../factory/DefaultV2ValuesWriterFactory.java | 37 ++++----- .../factory/DefaultValuesWriterFactory.java | 31 ++++---- .../values/factory/ValuesWriterFactory.java | 21 ++--- .../factory/ValuesWriterFactoryParams.java | 76 ------------------ .../DefaultValuesWriterFactoryTest.java | 11 +-- .../parquet/hadoop/ParquetOutputFormat.java | 39 +++------- .../StubConfigurableValuesWriterFactory.java | 53 ------------- .../hadoop/StubValuesWriterFactory.java | 37 --------- ...stParquetOutputFormatFactoryOverrides.java | 78 ------------------- 11 files changed, 93 insertions(+), 347 deletions(-) delete mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java delete mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java delete mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java delete mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java diff --git a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java index 1748e14595..e7468115d3 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/ParquetProperties.java @@ -33,7 +33,6 @@ import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; import org.apache.parquet.column.values.factory.ValuesWriterFactory; -import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; import org.apache.parquet.schema.MessageType; /** @@ -77,6 +76,7 @@ public static WriterVersion fromString(String name) { } } + private final int initialSlabSize; private final int pageSizeThreshold; private final int dictionaryPageSizeThreshold; private final WriterVersion writerVersion; @@ -91,7 +91,7 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag int maxRowCountForPageSizeCheck, boolean estimateNextSizeCheck, ByteBufferAllocator allocator, ValuesWriterFactory writerFactory) { this.pageSizeThreshold = pageSize; - int initialSlabSize = CapacityByteArrayOutputStream + this.initialSlabSize = CapacityByteArrayOutputStream .initialSlabSizeHeuristic(MIN_SLAB_SIZE, pageSizeThreshold, 10); this.dictionaryPageSizeThreshold = dictPageSize; this.writerVersion = writerVersion; @@ -101,11 +101,7 @@ private ParquetProperties(WriterVersion writerVersion, int pageSize, int dictPag this.estimateNextSizeCheck = estimateNextSizeCheck; this.allocator = allocator; - ValuesWriterFactoryParams params = - new ValuesWriterFactoryParams(writerVersion, initialSlabSize, pageSizeThreshold, allocator, - enableDictionary, dictionaryPageSizeThreshold); - valuesWriterFactory = writerFactory; - valuesWriterFactory.initialize(params); + this.valuesWriterFactory = writerFactory; } public ValuesWriter newRepetitionLevelWriter(ColumnDescriptor path) { @@ -146,6 +142,10 @@ public int getPageSizeThreshold() { return pageSizeThreshold; } + public int getInitialSlabSize() { + return initialSlabSize; + } + public int getDictionaryPageSizeThreshold() { return dictionaryPageSizeThreshold; } @@ -303,9 +303,17 @@ public Builder withValuesWriterFactory(ValuesWriterFactory factory) { } public ParquetProperties build() { - return new ParquetProperties(writerVersion, pageSize, dictPageSize, + ParquetProperties properties = + new ParquetProperties(writerVersion, pageSize, dictPageSize, enableDict, minRowCountForPageSizeCheck, maxRowCountForPageSizeCheck, estimateNextSizeCheck, allocator, valuesWriterFactory); + // we pass a constructed but uninitialized factory to ParquetProperties above as currently + // creation of ValuesWriters is invoked from within ParquetProperties. In the future + // we'd like to decouple that and won't need to pass an object to properties and then pass the + // properties to the object. + valuesWriterFactory.initialize(properties); + + return properties; } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java index 01530cff19..ffbd9505f1 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV1ValuesWriterFactory.java @@ -20,6 +20,7 @@ import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.plain.BooleanPlainValuesWriter; import org.apache.parquet.column.values.plain.FixedLenByteArrayPlainValuesWriter; @@ -29,11 +30,11 @@ public class DefaultV1ValuesWriterFactory implements ValuesWriterFactory { - private ValuesWriterFactoryParams selectionParams; + private ParquetProperties parquetProperties; @Override - public void initialize(ValuesWriterFactoryParams params) { - this.selectionParams = params; + public void initialize(ParquetProperties properties) { + this.parquetProperties = properties; } private Encoding getEncodingForDataPage() { @@ -75,36 +76,36 @@ private ValuesWriter getBooleanValuesWriter() { private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { // dictionary encoding was not enabled in PARQUET 1.0 - return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return new FixedLenByteArrayPlainValuesWriter(path.getTypeLength(), parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); } private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java index 50635a9e93..8348a024af 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultV2ValuesWriterFactory.java @@ -20,6 +20,7 @@ import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForInteger; import org.apache.parquet.column.values.delta.DeltaBinaryPackingValuesWriterForLong; @@ -33,11 +34,11 @@ public class DefaultV2ValuesWriterFactory implements ValuesWriterFactory { - private ValuesWriterFactoryParams selectionParams; + private ParquetProperties parquetProperties; @Override - public void initialize(ValuesWriterFactoryParams params) { - this.selectionParams = params; + public void initialize(ParquetProperties properties) { + this.parquetProperties = properties; } private Encoding getEncodingForDataPage() { @@ -74,41 +75,41 @@ public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { private ValuesWriter getBooleanValuesWriter() { // no dictionary encoding for boolean - return new RunLengthBitPackingHybridValuesWriter(1, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); + return new RunLengthBitPackingHybridValuesWriter(1, parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); } private ValuesWriter getFixedLenByteArrayValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getBinaryValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new DeltaByteArrayWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new DeltaByteArrayWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getInt32ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForInteger(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getInt64ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new DeltaBinaryPackingValuesWriterForLong(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getInt96ValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new FixedLenByteArrayPlainValuesWriter(12, parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getDoubleValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } private ValuesWriter getFloatValuesWriter(ColumnDescriptor path) { - ValuesWriter fallbackWriter = new PlainValuesWriter(selectionParams.getInitialCapacity(), selectionParams.getPageSize(), selectionParams.getAllocator()); - return DefaultValuesWriterFactory.dictWriterWithFallBack(path, selectionParams, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); + ValuesWriter fallbackWriter = new PlainValuesWriter(parquetProperties.getInitialSlabSize(), parquetProperties.getPageSizeThreshold(), parquetProperties.getAllocator()); + return DefaultValuesWriterFactory.dictWriterWithFallBack(path, parquetProperties, getEncodingForDictionaryPage(), getEncodingForDataPage(), fallbackWriter); } } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java index 80aff8b4fe..65848945f1 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactory.java @@ -20,6 +20,7 @@ import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; import org.apache.parquet.column.values.ValuesWriter; import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; @@ -30,22 +31,20 @@ */ public class DefaultValuesWriterFactory implements ValuesWriterFactory { - private ValuesWriterFactoryParams selectionParams; private ValuesWriterFactory delegateFactory; private static final ValuesWriterFactory DEFAULT_V1_WRITER_FACTORY = new DefaultV1ValuesWriterFactory(); private static final ValuesWriterFactory DEFAULT_V2_WRITER_FACTORY = new DefaultV2ValuesWriterFactory(); @Override - public void initialize(ValuesWriterFactoryParams params) { - this.selectionParams = params; - if (selectionParams.getWriterVersion() == WriterVersion.PARQUET_1_0) { + public void initialize(ParquetProperties properties) { + if (properties.getWriterVersion() == WriterVersion.PARQUET_1_0) { delegateFactory = DEFAULT_V1_WRITER_FACTORY; } else { delegateFactory = DEFAULT_V2_WRITER_FACTORY; } - delegateFactory.initialize(params); + delegateFactory.initialize(properties); } @Override @@ -53,33 +52,33 @@ public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { return delegateFactory.newValuesWriter(descriptor); } - public static DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path, ValuesWriterFactoryParams selectionParams, Encoding dictPageEncoding, Encoding dataPageEncoding) { + static DictionaryValuesWriter dictionaryWriter(ColumnDescriptor path, ParquetProperties properties, Encoding dictPageEncoding, Encoding dataPageEncoding) { switch (path.getType()) { case BOOLEAN: throw new IllegalArgumentException("no dictionary encoding for BOOLEAN"); case BINARY: - return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainBinaryDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), dataPageEncoding, dictPageEncoding, properties.getAllocator()); case INT32: - return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainIntegerDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), dataPageEncoding, dictPageEncoding, properties.getAllocator()); case INT64: - return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainLongDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), dataPageEncoding, dictPageEncoding, properties.getAllocator()); case INT96: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), 12, dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), 12, dataPageEncoding, dictPageEncoding, properties.getAllocator()); case DOUBLE: - return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainDoubleDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), dataPageEncoding, dictPageEncoding, properties.getAllocator()); case FLOAT: - return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainFloatDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), dataPageEncoding, dictPageEncoding, properties.getAllocator()); case FIXED_LEN_BYTE_ARRAY: - return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(selectionParams.getMaxDictionaryByteSize(), path.getTypeLength(), dataPageEncoding, dictPageEncoding, selectionParams.getAllocator()); + return new DictionaryValuesWriter.PlainFixedLenArrayDictionaryValuesWriter(properties.getDictionaryPageSizeThreshold(), path.getTypeLength(), dataPageEncoding, dictPageEncoding, properties.getAllocator()); default: throw new IllegalArgumentException("Unknown type " + path.getType()); } } - public static ValuesWriter dictWriterWithFallBack(ColumnDescriptor path, ValuesWriterFactoryParams selectionParams, Encoding dictPageEncoding, Encoding dataPageEncoding, ValuesWriter writerToFallBackTo) { - if (selectionParams.getEnableDictionary()) { + static ValuesWriter dictWriterWithFallBack(ColumnDescriptor path, ParquetProperties parquetProperties, Encoding dictPageEncoding, Encoding dataPageEncoding, ValuesWriter writerToFallBackTo) { + if (parquetProperties.isEnableDictionary()) { return FallbackValuesWriter.of( - dictionaryWriter(path, selectionParams, dictPageEncoding, dataPageEncoding), + dictionaryWriter(path, parquetProperties, dictPageEncoding, dataPageEncoding), writerToFallBackTo); } else { return writerToFallBackTo; diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java index 9d4e2d6ffa..3b9fcfb41f 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java @@ -19,28 +19,29 @@ package org.apache.parquet.column.values.factory; import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.values.ValuesWriter; /** - * Can be overridden to allow users to specify how they want their ValuesWriters to be created. - * ValuesWriterFactories are created using reflection in {@link org.apache.parquet.column.ParquetProperties}. - * Due to this, they must provide a default constructor. - * Lifecycle of ValuesWriterFactories is: - * 1) Created via reflection while creating a {@link org.apache.parquet.column.ParquetProperties} - * 2) If the factory is Configurable (needs Hadoop conf), that is set by calling setConfig(). + * Can be overridden to allow users to manually test different strategies to create ValuesWriters. + * To do this, the ValuesWriterFactory to be used must be passed to the {@link org.apache.parquet.column.ParquetProperties.Builder}. + *
    Lifecycle of ValuesWriterFactories is: + *
  • Initialized while creating a {@link org.apache.parquet.column.ParquetProperties} using the Builder
    • + *
  • If the factory is Configurable (needs Hadoop conf), that is set by calling setConfig(). * initialize() is also called. This is done just once for the lifetime of the factory. As Hadoop - * config can be set, ValuesWriterFactories can read additional config to create appropriate ValuesWriters. - * 3) newValuesWriter is called once per column for every block of data. + * config can be set, ValuesWriterFactories can read additional config to create appropriate ValuesWriters.
    • + *
  • newValuesWriter is called once per column for every block of data.
    • + *
*/ public interface ValuesWriterFactory { /** * Used to initialize the factory. This method is called before newValuesWriter() */ - void initialize(ValuesWriterFactoryParams params); + void initialize(ParquetProperties parquetProperties); /** - * Creates a ValuesWriter to help write the given column. + * Creates a ValuesWriter to write values for the given column. */ ValuesWriter newValuesWriter(ColumnDescriptor descriptor); } diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java deleted file mode 100644 index 566eb1155b..0000000000 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactoryParams.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.column.values.factory; - -import org.apache.parquet.bytes.ByteBufferAllocator; -import org.apache.parquet.column.ParquetProperties.WriterVersion; -import static org.apache.parquet.Preconditions.checkNotNull; - -/** - * Encapsulates parameters needed to create new ValuesWriter classes as part of - * ValuesWriterFactory - */ -public class ValuesWriterFactoryParams { - - public ValuesWriterFactoryParams( - WriterVersion writerVersion, - int initialCapacity, - int pageSize, - ByteBufferAllocator allocator, - boolean enableDictionary, - int maxDictionaryByteSize) { - this.writerVersion = checkNotNull(writerVersion, "writerVersion"); - this.initialCapacity = initialCapacity; - this.pageSize = pageSize; - this.allocator = checkNotNull(allocator, "allocator"); - this.enableDictionary = enableDictionary; - this.maxDictionaryByteSize = maxDictionaryByteSize; - } - - private final WriterVersion writerVersion; - private final boolean enableDictionary; - private final int initialCapacity; - private final int pageSize; - private final ByteBufferAllocator allocator; - private final int maxDictionaryByteSize; - - public WriterVersion getWriterVersion() { - return writerVersion; - } - - public int getInitialCapacity() { - return initialCapacity; - } - - public int getPageSize() { - return pageSize; - } - - public ByteBufferAllocator getAllocator() { - return allocator; - } - - public int getMaxDictionaryByteSize() { - return maxDictionaryByteSize; - } - - public boolean getEnableDictionary() { - return enableDictionary; - } -} diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java index ab4dfc187d..d6865e2d65 100644 --- a/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java +++ b/parquet-column/src/test/java/org/apache/parquet/column/values/factory/DefaultValuesWriterFactoryTest.java @@ -18,8 +18,6 @@ */ package org.apache.parquet.column.values.factory; -import org.apache.parquet.bytes.ByteBufferAllocator; -import org.apache.parquet.bytes.HeapByteBufferAllocator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; @@ -44,8 +42,6 @@ public class DefaultValuesWriterFactoryTest { - private static final ByteBufferAllocator ALLOCATOR = new HeapByteBufferAllocator(); - @Test public void testBoolean() { doTestValueWriter( @@ -331,7 +327,12 @@ private ColumnDescriptor getMockColumn(PrimitiveTypeName typeName) { private ValuesWriterFactory getDefaultFactory(WriterVersion writerVersion, boolean enableDictionary) { ValuesWriterFactory factory = new DefaultValuesWriterFactory(); - factory.initialize(new ValuesWriterFactoryParams(writerVersion, 128, ParquetProperties.DEFAULT_PAGE_SIZE, ALLOCATOR, enableDictionary, 0)); + ParquetProperties.builder() + .withDictionaryEncoding(enableDictionary) + .withWriterVersion(writerVersion) + .withValuesWriterFactory(factory) + .build(); + return factory; } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index b646866538..67f79924b9 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -147,18 +147,6 @@ public static enum JobSummaryLevel { public static final String MAX_ROW_COUNT_FOR_PAGE_SIZE_CHECK = "parquet.page.size.row.check.max"; public static final String ESTIMATE_PAGE_SIZE_CHECK = "parquet.page.size.check.estimate"; - /** - * Used to override the value writer factory used to create Values Writers. - * This allows users to plug in their own factory class to create custom ValuesWriters rather - * use the Parquet provided default - {@link org.apache.parquet.column.values.factory.DefaultValuesWriterFactory}. - * Factory classes provided must implement {@link org.apache.parquet.column.values.factory.ValuesWriterFactory}. - * - * For e.g. "parquet.write.value.writer.factory" = "org.apache.parquet.column.values.factory.MyValuesWriterFactory" - * will ensure that we use MyValuesWriterFactory to create ValuesWriters for writing out data. - * See the documentation for ValuesWriterFactory for more details. - */ - public static final String WRITER_FACTORY_OVERRIDE = "parquet.write.value.writer.factory"; - // default to no padding for now private static final int DEFAULT_MAX_PADDING_SIZE = 0; @@ -332,24 +320,14 @@ private static int getMaxPaddingSize(Configuration conf) { } public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) { - Class factoryOverride = - (Class) ConfigurationUtil.getClassFromConfig(conf, WRITER_FACTORY_OVERRIDE, ValuesWriterFactory.class); + ValuesWriterFactory factory = ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY; - try { - ValuesWriterFactory factory = - factoryOverride == null ? ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY : factoryOverride.newInstance() ; - - if (factory instanceof Configurable) { - Configurable configurableFactory = (Configurable) factory; - configurableFactory.setConf(conf); - } - - return factory; - } catch (InstantiationException e) { - throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e); - } catch (IllegalAccessException e) { - throw new BadConfigurationException("Unable to instantiate ValuesWriterFactory: " + factoryOverride, e); + if (factory instanceof Configurable) { + Configurable configurableFactory = (Configurable) factory; + configurableFactory.setConf(conf); } + + return factory; } private WriteSupport writeSupport; @@ -394,6 +372,8 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp throws IOException, InterruptedException { final WriteSupport writeSupport = getWriteSupport(conf); + ValuesWriterFactory valuesWriterFactory = getValuesWriterFactory(conf); + ParquetProperties props = ParquetProperties.builder() .withPageSize(getPageSize(conf)) .withDictionaryPageSize(getDictionaryPageSize(conf)) @@ -402,7 +382,7 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) - .withValuesWriterFactory(getValuesWriterFactory(conf)) + .withValuesWriterFactory(valuesWriterFactory) .build(); long blockSize = getLongBlockSize(conf); @@ -420,7 +400,6 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp LOG.info("Page size checking is: " + (props.estimateNextSizeCheck() ? "estimated" : "constant")); LOG.info("Min row count for page size check is: " + props.getMinRowCountForPageSizeCheck()); LOG.info("Max row count for page size check is: " + props.getMaxRowCountForPageSizeCheck()); - LOG.info("ValueWriterFactory chosen is: " + props.getValuesWriterFactory()); } WriteContext init = writeSupport.init(conf); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java deleted file mode 100644 index 4420e93ad5..0000000000 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubConfigurableValuesWriterFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.hadoop; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.values.ValuesWriter; -import org.apache.parquet.column.values.factory.ValuesWriterFactory; -import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; - -/** - * ValuesWriter factory test class for verifying overrides and configuration - */ -public class StubConfigurableValuesWriterFactory implements ValuesWriterFactory, Configurable { - - private Configuration config; - - @Override - public void setConf(Configuration config) { - this.config = config; - } - - @Override - public Configuration getConf() { - return config; - } - - @Override - public void initialize(ValuesWriterFactoryParams params) { - } - - @Override - public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { - return null; - } -} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java deleted file mode 100644 index cb2d47f4b2..0000000000 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/StubValuesWriterFactory.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.hadoop; - -import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.values.ValuesWriter; -import org.apache.parquet.column.values.factory.ValuesWriterFactory; -import org.apache.parquet.column.values.factory.ValuesWriterFactoryParams; - -/** - * ValuesWriter factory test class for verifying overrides - */ -public class StubValuesWriterFactory implements ValuesWriterFactory { - @Override - public void initialize(ValuesWriterFactoryParams params) {} - - @Override - public ValuesWriter newValuesWriter(ColumnDescriptor descriptor) { - return null; - } -} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java deleted file mode 100644 index bd3b437295..0000000000 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetOutputFormatFactoryOverrides.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.hadoop; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.parquet.column.values.factory.DefaultValuesWriterFactory; -import org.apache.parquet.column.values.factory.ValuesWriterFactory; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class TestParquetOutputFormatFactoryOverrides { - - @Test - public void testMissingOverrideParam() { - Configuration conf = new Configuration(); - ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); - assertEquals("Incorrect default factory override", DefaultValuesWriterFactory.class, factory.getClass()); - } - - @Test(expected = BadConfigurationException.class) - public void testMissingFactoryOverrideClass() { - Configuration conf = new Configuration(); - conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "MyMissingFactory"); - - // should throw as we can't find MyMissingFactory - ParquetOutputFormat.getValuesWriterFactory(conf); - } - - @Test(expected = BadConfigurationException.class) - public void testFactoryClassIncorrectInterface() { - Configuration conf = new Configuration(); - conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.column.values.factory.ValuesWriterFactoryParams"); - - // should throw as ValuesWriterFactoryParams isn't implementing ValuesWriterFactory - ParquetOutputFormat.getValuesWriterFactory(conf); - } - - @Test - public void testValidFactoryOverride() { - Configuration conf = new Configuration(); - conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.hadoop.StubValuesWriterFactory"); - ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); - assertEquals("Incorrect factory override chosen", StubValuesWriterFactory.class, factory.getClass()); - } - - @Test - public void testFactoryOverrideWithCfg() { - Configuration conf = new Configuration(); - conf.set("foo", "bar"); - conf.set(ParquetOutputFormat.WRITER_FACTORY_OVERRIDE, "org.apache.parquet.hadoop.StubConfigurableValuesWriterFactory"); - - ValuesWriterFactory factory = ParquetOutputFormat.getValuesWriterFactory(conf); - assertEquals("Incorrect factory override chosen", StubConfigurableValuesWriterFactory.class, factory.getClass()); - - Configurable configurable = (Configurable) factory; - assertNotNull("Not a ConfigurableFactory", configurable); - assertEquals("Incorrect config value found", "bar", configurable.getConf().get("foo")); - } -} From 149bb986e25b67c5148891abcd4501451cfddf0a Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Thu, 4 Aug 2016 11:39:29 -0700 Subject: [PATCH 14/15] Switch to getValuesWriterFactory call to non-static --- .../java/org/apache/parquet/hadoop/ParquetOutputFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 67f79924b9..58c219f6f4 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -319,7 +319,7 @@ private static int getMaxPaddingSize(Configuration conf) { return conf.getInt(MAX_PADDING_BYTES, DEFAULT_MAX_PADDING_SIZE); } - public static ValuesWriterFactory getValuesWriterFactory(Configuration conf) { + public ValuesWriterFactory getValuesWriterFactory(Configuration conf) { ValuesWriterFactory factory = ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY; if (factory instanceof Configurable) { From 3ebab28b538946df13ac1621fd65640780988c72 Mon Sep 17 00:00:00 2001 From: Piyush Narang Date: Thu, 4 Aug 2016 18:50:01 -0700 Subject: [PATCH 15/15] Remove Configurable --- .../values/factory/ValuesWriterFactory.java | 6 +++--- .../parquet/hadoop/ParquetOutputFormat.java | 16 ---------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java index 3b9fcfb41f..8f06e7b2f4 100644 --- a/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java +++ b/parquet-column/src/main/java/org/apache/parquet/column/values/factory/ValuesWriterFactory.java @@ -27,9 +27,9 @@ * To do this, the ValuesWriterFactory to be used must be passed to the {@link org.apache.parquet.column.ParquetProperties.Builder}. *
    Lifecycle of ValuesWriterFactories is: *
  • Initialized while creating a {@link org.apache.parquet.column.ParquetProperties} using the Builder
    • - *
  • If the factory is Configurable (needs Hadoop conf), that is set by calling setConfig(). - * initialize() is also called. This is done just once for the lifetime of the factory. As Hadoop - * config can be set, ValuesWriterFactories can read additional config to create appropriate ValuesWriters.
    • + *
  • If the factory must read Hadoop config, it needs to implement the Configurable interface. + * In addition to that, ParquetOutputFormat needs to be updated to pass in the Hadoop config via the setConf() + * method on the Configurable interface.
    • *
  • newValuesWriter is called once per column for every block of data.
    • *
*/ diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 58c219f6f4..d05d41f69e 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -25,7 +25,6 @@ import java.io.IOException; -import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; @@ -39,7 +38,6 @@ import org.apache.parquet.Log; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; -import org.apache.parquet.column.values.factory.ValuesWriterFactory; import org.apache.parquet.hadoop.ParquetFileWriter.Mode; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.hadoop.api.WriteSupport.WriteContext; @@ -319,17 +317,6 @@ private static int getMaxPaddingSize(Configuration conf) { return conf.getInt(MAX_PADDING_BYTES, DEFAULT_MAX_PADDING_SIZE); } - public ValuesWriterFactory getValuesWriterFactory(Configuration conf) { - ValuesWriterFactory factory = ParquetProperties.DEFAULT_VALUES_WRITER_FACTORY; - - if (factory instanceof Configurable) { - Configurable configurableFactory = (Configurable) factory; - configurableFactory.setConf(conf); - } - - return factory; - } - private WriteSupport writeSupport; private ParquetOutputCommitter committer; @@ -372,8 +359,6 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp throws IOException, InterruptedException { final WriteSupport writeSupport = getWriteSupport(conf); - ValuesWriterFactory valuesWriterFactory = getValuesWriterFactory(conf); - ParquetProperties props = ParquetProperties.builder() .withPageSize(getPageSize(conf)) .withDictionaryPageSize(getDictionaryPageSize(conf)) @@ -382,7 +367,6 @@ public RecordWriter getRecordWriter(Configuration conf, Path file, Comp .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf)) .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf)) .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf)) - .withValuesWriterFactory(valuesWriterFactory) .build(); long blockSize = getLongBlockSize(conf);