From 3edb8c1cb87f1725d121326e10edcaa9fe9fca42 Mon Sep 17 00:00:00 2001 From: Kunal Chawla Date: Tue, 1 May 2018 13:26:35 -0700 Subject: [PATCH 1/2] Adding constructors to ProtoParquetWriter with writeSpecsCompliant flag so that a specs compliant writer can be instantiated --- .../parquet/proto/ProtoParquetWriter.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java index ef9a5baeb0..e178965010 100644 --- a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java +++ b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java @@ -20,6 +20,7 @@ import com.google.protobuf.Message; import com.google.protobuf.MessageOrBuilder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.api.WriteSupport; @@ -81,4 +82,63 @@ public ProtoParquetWriter(Path file, Class protoMessage) thro DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE); } + /** + * Create a new {@link ProtoParquetWriter}. + * + * @param file The file name to write to. + * @param protoMessage Protobuf message class + * @param compressionCodecName Compression code to use, or CompressionCodecName.UNCOMPRESSED + * @param blockSize HDFS block size + * @param pageSize See parquet write up. Blocks are subdivided into pages for alignment and other purposes. + * @param enableDictionary Whether to use a dictionary to compress columns. + * @param validating to turn on validation using the schema + * @param writeSpecsCompliant Enable writing specs compliant schemas with parquet-protobuf + * @throws IOException if there is an error while writing + */ + public ProtoParquetWriter(Path file, Class protoMessage, + CompressionCodecName compressionCodecName, int blockSize, int pageSize, boolean enableDictionary, + boolean validating, boolean writeSpecsCompliant) throws IOException { + super(file, new ProtoWriteSupport(protoMessage), compressionCodecName, blockSize, pageSize, pageSize, + enableDictionary, validating, DEFAULT_WRITER_VERSION, + getConfigWithWriteSpecsCompliant(writeSpecsCompliant)); + } + + /** + * Create a new {@link ProtoParquetWriter}. + * + * @param file The file name to write to. + * @param protoMessage Protobuf message class + * @param compressionCodecName Compression code to use, or CompressionCodecName.UNCOMPRESSED + * @param blockSize HDFS block size + * @param pageSize See parquet write up. Blocks are subdivided into pages for alignment and other purposes. + * @param writeSpecsCompliant Enable writing specs compliant schemas with parquet-protobuf + * @throws IOException if there is an error while writing + */ + public ProtoParquetWriter(Path file, Class protoMessage, + CompressionCodecName compressionCodecName, int blockSize, + int pageSize, boolean writeSpecsCompliant) throws IOException { + this(file, protoMessage, compressionCodecName, blockSize, pageSize, DEFAULT_IS_DICTIONARY_ENABLED, + DEFAULT_IS_VALIDATING_ENABLED, writeSpecsCompliant); + } + + /** + * Create a new {@link ProtoParquetWriter}. The default block size is 50 MB.The default + * page size is 1 MB. Default compression is no compression. (Inherited from {@link ParquetWriter}) + * + * @param file The file name to write to. + * @param protoMessage Protobuf message class + * @param writeSpecsCompliant Enable writing specs compliant schemas with parquet-protobuf + * @throws IOException if there is an error while writing + */ + public ProtoParquetWriter(Path file, Class protoMessage, boolean writeSpecsCompliant) throws IOException { + this(file, protoMessage, CompressionCodecName.UNCOMPRESSED, + DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE, writeSpecsCompliant); + } + + private static Configuration getConfigWithWriteSpecsCompliant(boolean writeSpecsCompliant) { + Configuration config = new Configuration(); + ProtoWriteSupport.setWriteSpecsCompliant(config, writeSpecsCompliant); + return config; + } + } From fef8b287c021093f15a919a4fac90733ce36b64c Mon Sep 17 00:00:00 2001 From: Kunal Chawla Date: Thu, 3 May 2018 08:49:22 -0700 Subject: [PATCH 2/2] Updating the comments for default page size --- .../parquet/proto/ProtoParquetWriter.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java index e178965010..22fb87d4f9 100644 --- a/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java +++ b/parquet-protobuf/src/main/java/org/apache/parquet/proto/ProtoParquetWriter.java @@ -69,14 +69,15 @@ public ProtoParquetWriter(Path file, Class protoMessage, compressionCodecName, blockSize, pageSize, enableDictionary, validating); } - /** - * Create a new {@link ProtoParquetWriter}. The default block size is 50 MB.The default - * page size is 1 MB. Default compression is no compression. (Inherited from {@link ParquetWriter}) - * - * @param file The file name to write to. - * @param protoMessage Protobuf message class - * @throws IOException if there is an error while writing - */ + /** + * Create a new {@link ProtoParquetWriter}. The default block size is {@link ParquetWriter#DEFAULT_BLOCK_SIZE} bytes.The + * default page size is {@link ParquetWriter#DEFAULT_PAGE_SIZE} bytes. Default compression is no compression. (Inherited from + * {@link ParquetWriter}) + * + * @param file The file name to write to. + * @param protoMessage Protobuf message class + * @throws IOException if there is an error while writing + */ public ProtoParquetWriter(Path file, Class protoMessage) throws IOException { this(file, protoMessage, CompressionCodecName.UNCOMPRESSED, DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE); @@ -122,8 +123,8 @@ public ProtoParquetWriter(Path file, Class protoMessage, } /** - * Create a new {@link ProtoParquetWriter}. The default block size is 50 MB.The default - * page size is 1 MB. Default compression is no compression. (Inherited from {@link ParquetWriter}) + * Create a new {@link ProtoParquetWriter}. The default block size is {@link ParquetWriter#DEFAULT_BLOCK_SIZE} bytes.The + * default page size is {@link ParquetWriter#DEFAULT_PAGE_SIZE} bytes. Default compression is no compression. * * @param file The file name to write to. * @param protoMessage Protobuf message class