From ba08b3f9b9fd76e7d12a9fd1702e036d3459d7f5 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 28 Jun 2016 17:46:57 -0700 Subject: [PATCH 01/11] PARQUET-400: Replace CompatibilityUtil with SeekableInputStream. --- .travis.yml | 4 +- .../parquet/hadoop/ParquetFileReader.java | 37 +++-- .../parquet/hadoop/ParquetFileWriter.java | 16 +- .../hadoop/util/CompatibilityUtil.java | 114 -------------- .../hadoop/util/H1SeekableInputStream.java | 142 ++++++++++++++++++ .../hadoop/util/H2SeekableInputStream.java | 88 +++++++++++ .../hadoop/util/SeekableInputStream.java | 109 ++++++++++++++ pom.xml | 14 +- 8 files changed, 385 insertions(+), 139 deletions(-) delete mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/CompatibilityUtil.java create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java diff --git a/.travis.yml b/.travis.yml index 890a3720f6..ff9b356935 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,8 @@ before_install: - cd .. env: - - HADOOP_PROFILE=default TEST_CODECS=uncompressed - - HADOOP_PROFILE=hadoop-2 TEST_CODECS=gzip,snappy + - HADOOP_PROFILE=hadoop-1 TEST_CODECS=uncompressed + - HADOOP_PROFILE=default TEST_CODECS=gzip,snappy install: mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true > mvn_install.log || (cat mvn_install.log && false) script: mvn test -P $HADOOP_PROFILE diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 83542d5193..05228d52f6 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -54,7 +54,6 @@ import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -66,7 +65,6 @@ import org.apache.parquet.column.page.DictionaryPageReadStore; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.filter2.compat.RowGroupFilter; -import org.apache.parquet.hadoop.util.CompatibilityUtil; import org.apache.parquet.Log; import org.apache.parquet.bytes.BytesInput; @@ -91,6 +89,7 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.HiddenFileFilter; +import org.apache.parquet.hadoop.util.SeekableInputStream; import org.apache.parquet.hadoop.util.counters.BenchmarkCounter; import org.apache.parquet.io.ParquetDecodingException; @@ -106,6 +105,13 @@ public class ParquetFileReader implements Closeable { public static String PARQUET_READ_PARALLELISM = "parquet.metadata.read.parallelism"; + // configure if we want to use Hadoop's V2 read(bytebuffer) API. + // If true, we try to read using the new Hadoop read(ByteBuffer) api. This reads data into the provided + // byteBuffer and allows us to potentially take advantage zero-copy read path in Hadoop. + // Else, we skip and either read into the byteBuffer's array (if its present) or allocate one and copy + // into it. + public static String PARQUET_HADOOP_BYTEBUFFER_READ = "parquet.read.use.byte.buffer"; + private static ParquetMetadataConverter converter = new ParquetMetadataConverter(); /** @@ -432,7 +438,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File */ public static final ParquetMetadata readFooter(Configuration configuration, FileStatus file, MetadataFilter filter) throws IOException { FileSystem fileSystem = file.getPath().getFileSystem(configuration); - FSDataInputStream in = fileSystem.open(file.getPath()); + SeekableInputStream in = SeekableInputStream.wrap(fileSystem.open(file.getPath())); try { return readFooter(file.getLen(), file.getPath().toString(), in, filter); } finally { @@ -449,7 +455,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File * @return the metadata blocks in the footer * @throws IOException if an error occurs while reading the file */ - public static final ParquetMetadata readFooter(long fileLen, String filePath, FSDataInputStream f, MetadataFilter filter) throws IOException { + public static final ParquetMetadata readFooter(long fileLen, String filePath, SeekableInputStream f, MetadataFilter filter) throws IOException { if (Log.DEBUG) { LOG.debug("File length " + fileLen); } @@ -493,7 +499,7 @@ public static ParquetFileReader open(Configuration conf, Path file, ParquetMetad } private final CodecFactory codecFactory; - private final FSDataInputStream f; + private final SeekableInputStream f; private final FileStatus fileStatus; private final Map paths = new HashMap(); private final FileMetaData fileMetaData; // may be null @@ -531,7 +537,7 @@ public ParquetFileReader( this.conf = configuration; this.fileMetaData = fileMetaData; FileSystem fs = filePath.getFileSystem(configuration); - this.f = fs.open(filePath); + this.f = SeekableInputStream.wrap(fs.open(filePath)); this.fileStatus = fs.getFileStatus(filePath); this.blocks = blocks; for (ColumnDescriptor col : columns) { @@ -562,7 +568,7 @@ public ParquetFileReader(Configuration conf, Path file, MetadataFilter filter) t this.conf = conf; FileSystem fs = file.getFileSystem(conf); this.fileStatus = fs.getFileStatus(file); - this.f = fs.open(file); + this.f = SeekableInputStream.wrap(fs.open(file)); this.footer = readFooter(fileStatus.getLen(), fileStatus.getPath().toString(), f, filter); this.fileMetaData = footer.getFileMetaData(); this.blocks = footer.getBlocks(); @@ -585,7 +591,7 @@ public ParquetFileReader(Configuration conf, Path file, ParquetMetadata footer) this.conf = conf; FileSystem fs = file.getFileSystem(conf); this.fileStatus = fs.getFileStatus(file); - this.f = fs.open(file); + this.f = SeekableInputStream.wrap(fs.open(file)); this.footer = footer; this.fileMetaData = footer.getFileMetaData(); this.blocks = footer.getBlocks(); @@ -772,7 +778,7 @@ DictionaryPage readDictionary(ColumnChunkMetaData meta) throws IOException { } private static DictionaryPage readCompressedDictionary( - PageHeader pageHeader, FSDataInputStream fin) throws IOException { + PageHeader pageHeader, SeekableInputStream fin) throws IOException { DictionaryPageHeader dictHeader = pageHeader.getDictionary_page_header(); int uncompressedPageSize = pageHeader.getUncompressed_page_size(); @@ -940,7 +946,7 @@ public BytesInput readAsBytesInput(int size) throws IOException { */ private class WorkaroundChunk extends Chunk { - private final FSDataInputStream f; + private final SeekableInputStream f; /** * @param descriptor the descriptor of the chunk @@ -948,7 +954,7 @@ private class WorkaroundChunk extends Chunk { * @param offset where the chunk starts in data * @param f the file stream positioned at the end of this chunk */ - private WorkaroundChunk(ChunkDescriptor descriptor, ByteBuffer byteBuf, int offset, FSDataInputStream f) { + private WorkaroundChunk(ChunkDescriptor descriptor, ByteBuffer byteBuf, int offset, SeekableInputStream f) { super(descriptor, byteBuf, offset); this.f = f; } @@ -964,7 +970,7 @@ protected PageHeader readPageHeader() throws IOException { // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing // if the last page is smaller than this, the page header itself is truncated in the buffer. - this.byteBuf.rewind(); // resetting the buffer to the position before we got the error + this.byteBuf.position(initialPos); // resetting the buffer to the position before we got the error LOG.info("completing the column chunk to read the page header"); pageHeader = Util.readPageHeader(new SequenceInputStream(this, f)); // trying again from the buffer + remainder of the stream. } @@ -1050,11 +1056,14 @@ public void addChunk(ChunkDescriptor descriptor) { * @return the chunks * @throws IOException */ - public List readAll(FSDataInputStream f) throws IOException { + public List readAll(SeekableInputStream f) throws IOException { List result = new ArrayList(chunks.size()); f.seek(offset); + + // Allocate the bytebuffer based on whether the FS can support it. ByteBuffer chunksByteBuffer = allocator.allocate(length); - CompatibilityUtil.getBuf(f, chunksByteBuffer, length); + f.readFully(chunksByteBuffer); + // report in a counter the data we just scanned BenchmarkCounter.incrementBytesRead(length); int currentChunkOffset = 0; diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index 523d01f56a..617f85722f 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -61,6 +61,7 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.GlobalMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.hadoop.util.SeekableInputStream; import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; @@ -495,6 +496,12 @@ public void appendFile(Configuration conf, Path file) throws IOException { public void appendRowGroups(FSDataInputStream file, List rowGroups, boolean dropColumns) throws IOException { + appendRowGroups(SeekableInputStream.wrap(file), rowGroups, dropColumns); + } + + public void appendRowGroups(SeekableInputStream file, + List rowGroups, + boolean dropColumns) throws IOException { for (BlockMetaData block : rowGroups) { appendRowGroup(file, block, dropColumns); } @@ -502,6 +509,11 @@ public void appendRowGroups(FSDataInputStream file, public void appendRowGroup(FSDataInputStream from, BlockMetaData rowGroup, boolean dropColumns) throws IOException { + appendRowGroup(from, rowGroup, dropColumns); + } + + public void appendRowGroup(SeekableInputStream from, BlockMetaData rowGroup, + boolean dropColumns) throws IOException { startBlock(rowGroup.getRowCount()); Map columnsToCopy = @@ -596,8 +608,8 @@ protected byte[] initialValue() { * @param length the number of bytes to copy * @throws IOException */ - private static void copy(FSDataInputStream from, FSDataOutputStream to, - long start, long length) throws IOException{ + private static void copy(SeekableInputStream from, FSDataOutputStream to, + long start, long length) throws IOException{ if (DEBUG) LOG.debug( "Copying " + length + " bytes at " + start + " to " + to.getPos()); from.seek(start); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/CompatibilityUtil.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/CompatibilityUtil.java deleted file mode 100644 index bacf222a24..0000000000 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/CompatibilityUtil.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.hadoop.util; - -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.parquet.ShouldNeverHappenException; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -public class CompatibilityUtil { - - // Will be set to true if the implementation of FSDataInputSteam supports - // the 2.x APIs, in particular reading using a provided ByteBuffer - private static boolean useV21; - public static final V21FileAPI fileAPI; - - private static class V21FileAPI { - private final Method PROVIDE_BUF_READ_METHOD; - private final Class FSDataInputStreamCls; - - private V21FileAPI() throws ReflectiveOperationException { - final String PACKAGE = "org.apache.hadoop"; - FSDataInputStreamCls = Class.forName(PACKAGE + ".fs.FSDataInputStream"); - PROVIDE_BUF_READ_METHOD = FSDataInputStreamCls.getMethod("read", ByteBuffer.class); - } - } - - static { - // Test to see if a class from the Hadoop 2.x API is available - boolean v21 = true; - try { - Class.forName("org.apache.hadoop.io.compress.DirectDecompressor"); - } catch (ClassNotFoundException cnfe) { - v21 = false; - } - - useV21 = v21; - try { - if (v21) { - fileAPI = new V21FileAPI(); - } else { - fileAPI = null; - } - - } catch (ReflectiveOperationException e) { - throw new IllegalArgumentException("Error finding appropriate interfaces using reflection.", e); - } - } - - private static Object invoke(Method method, String errorMsg, Object instance, Object... args) { - try { - return method.invoke(instance, args); - } catch (IllegalAccessException e) { - throw new IllegalArgumentException(errorMsg, e); - } catch (InvocationTargetException e) { - throw new IllegalArgumentException(errorMsg, e); - } - } - - public static int getBuf(FSDataInputStream f, ByteBuffer readBuf, int maxSize) throws IOException { - int res; - if (useV21) { - try { - res = (Integer) fileAPI.PROVIDE_BUF_READ_METHOD.invoke(f, readBuf); - } catch (InvocationTargetException e) { - if (e.getCause() instanceof UnsupportedOperationException) { - // the FSDataInputStream docs say specifically that implementations - // can choose to throw UnsupportedOperationException, so this should - // be a reasonable check to make to see if the interface is - // present but not implemented and we should be falling back - useV21 = false; - return getBuf(f, readBuf, maxSize); - } else if (e.getCause() instanceof IOException) { - throw (IOException) e.getCause(); - } else { - // To handle any cases where a Runtime exception occurs and provide - // some additional context information. A stacktrace would just give - // a line number, this at least tells them we were using the version - // of the read method designed for using a ByteBuffer. - throw new IOException("Error reading out of an FSDataInputStream " + - "using the Hadoop 2 ByteBuffer based read method.", e.getCause()); - } - } catch (IllegalAccessException e) { - // This method is public because it is defined in an interface, - // there should be no problems accessing it - throw new ShouldNeverHappenException(e); - } - } else { - byte[] buf = new byte[maxSize]; - res = f.read(buf); - readBuf.put(buf, 0, res); - } - return res; - } -} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java new file mode 100644 index 0000000000..cbb42a260e --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.FSDataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class H1SeekableInputStream extends SeekableInputStream { + + private final int COPY_BUFFER_SIZE = 8192; + private final byte[] temp = new byte[COPY_BUFFER_SIZE]; + + private final FSDataInputStream stream; + + public H1SeekableInputStream(FSDataInputStream stream) { + this.stream = stream; + } + + @Override + public void close() throws IOException { + stream.close(); + } + + @Override + public long getPos() throws IOException { + return stream.getPos(); + } + + @Override + public void seek(long newPos) throws IOException { + stream.seek(newPos); + } + + @Override + public int read() throws IOException { + return stream.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return stream.read(b, off, len); + } + + @Override + public void readFully(byte[] bytes) throws IOException { + stream.readFully(bytes, 0, bytes.length); + } + + @Override + public void readFully(byte[] bytes, int start, int len) throws IOException { + stream.readFully(bytes); + } + + @Override + public int read(ByteBuffer buf) throws IOException { + if (buf.hasArray()) { + return readHeapBuffer(stream, buf); + } else { + return readDirectBuffer(stream, buf, temp); + } + } + + @Override + public void readFully(ByteBuffer buf) throws IOException { + if (buf.hasArray()) { + readFullyHeapBuffer(stream, buf); + } else { + readFullyDirectBuffer(stream, buf, temp); + } + } + + // Visisble for testing + static int readHeapBuffer(FSDataInputStream f, ByteBuffer buf) throws IOException { + int bytesRead = f.read(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining()); + buf.position(buf.position() + bytesRead); + return bytesRead; + } + + // Visisble for testing + static void readFullyHeapBuffer(FSDataInputStream f, ByteBuffer buf) throws IOException { + f.readFully(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining()); + buf.position(buf.limit()); + } + + // Visisble for testing + static int readDirectBuffer(FSDataInputStream f, ByteBuffer buf, byte[] temp) throws IOException { + // copy all the bytes that return immediately, stopping at the first + // read that doesn't return a full buffer. + int nextReadLength = Math.min(buf.remaining(), temp.length); + int totalBytesRead = 0; + int bytesRead; + + while ((bytesRead = f.read(temp, 0, nextReadLength)) == temp.length) { + buf.put(temp); + totalBytesRead += bytesRead; + nextReadLength = Math.min(buf.remaining(), temp.length); + } + + if (bytesRead < 0) { + // return -1 if nothing was read + return totalBytesRead == 0 ? -1 : totalBytesRead; + } else { + // copy the last partial buffer + buf.put(temp, 0, bytesRead); + totalBytesRead += bytesRead; + return totalBytesRead; + } + } + + // Visisble for testing + static void readFullyDirectBuffer(FSDataInputStream f, ByteBuffer buf, byte[] temp) throws IOException { + int nextReadLength = Math.min(buf.remaining(), temp.length); + int bytesRead = 0; + while (nextReadLength > 0 && (bytesRead = f.read(temp, 0, nextReadLength)) > 0) { + buf.put(temp, 0, bytesRead); + nextReadLength = Math.min(buf.remaining(), temp.length); + } + if (bytesRead < 0 && buf.remaining() > 0) { + throw new EOFException( + "Reached the end of stream. Still have: " + buf.remaining() + " bytes left"); + } + } +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java new file mode 100644 index 0000000000..7f9a9c40a1 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.ByteBufferReadable; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.parquet.Preconditions; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class H2SeekableInputStream extends SeekableInputStream { + + private final FSDataInputStream stream; + + public H2SeekableInputStream(FSDataInputStream stream) { + Preconditions.checkArgument(stream instanceof ByteBufferReadable, + "Input stream must be ByteBufferReadable. Try using H1SeekableInputStream"); + this.stream = stream; + } + + @Override + public long getPos() throws IOException { + return stream.getPos(); + } + + @Override + public void seek(long newPos) throws IOException { + stream.seek(newPos); + } + + @Override + public int read() throws IOException { + return stream.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return stream.read(b, off, len); + } + + @Override + public void readFully(byte[] bytes) throws IOException { + stream.readFully(bytes, 0, bytes.length); + } + + @Override + public void readFully(byte[] bytes, int start, int len) throws IOException { + stream.readFully(bytes); + } + + @Override + public int read(ByteBuffer buf) throws IOException { + return stream.read(buf); + } + + @Override + public void readFully(ByteBuffer buf) throws IOException { + // unfortunately the Hadoop APIs seem to not have a 'readFully' equivalent for the byteBuffer read + // calls. The read(ByteBuffer) call might read fewer than byteBuffer.hasRemaining() bytes. Thus we + // have to loop to ensure we read them. + while (buf.hasRemaining()) { + int readCount = read(buf); + if (readCount == -1) { + // this is probably a bug in the ParquetReader. We shouldn't have called readFully with a buffer + // that has more remaining than the amount of data in the stream. + throw new EOFException("Reached the end of stream. Still have: " + buf.remaining() + " bytes left"); + } + } + } +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java new file mode 100644 index 0000000000..baefd7ec95 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.parquet.Log; +import org.apache.parquet.io.ParquetDecodingException; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; + +public abstract class SeekableInputStream extends InputStream { + + private static final Log LOG = Log.getLog(SeekableInputStream.class); + + private static final Class byteBufferReadableClass; + + private static final Constructor h2SeekableConstructor; + + static { + byteBufferReadableClass = getReadableClass(); + h2SeekableConstructor = getH2SeekableConstructor(); + } + + public static SeekableInputStream wrap(FSDataInputStream stream) { + if (byteBufferReadableClass != null && h2SeekableConstructor != null && + byteBufferReadableClass.isInstance(stream)) { + try { + return h2SeekableConstructor.newInstance(stream); + } catch (InstantiationException e) { + LOG.warn("Could not instantiate H2SeekableInputStream, falling back to byte array reads", e); + return new H1SeekableInputStream(stream); + } catch (IllegalAccessException e) { + LOG.warn("Could not instantiate H2SeekableInputStream, falling back to byte array reads", e); + return new H1SeekableInputStream(stream); + } catch (InvocationTargetException e) { + throw new ParquetDecodingException( + "Could not instantiate H2SeekableInputStream", e.getTargetException()); + } + } else { + return new H1SeekableInputStream(stream); + } + } + + public abstract long getPos() throws IOException; + + public abstract void seek(long newPos) throws IOException; + + public abstract void readFully(byte[] bytes) throws IOException; + + public abstract void readFully(byte[] bytes, int start, int len) throws IOException; + + public abstract int read(ByteBuffer buf) throws IOException; + + public abstract void readFully(ByteBuffer buf) throws IOException; + + private static Class getReadableClass() { + try { + return Class.forName("org.apache.hadoop.fs.ByteBufferReadable"); + } catch (ClassNotFoundException e) { + return null; + } catch (NoClassDefFoundError e) { + return null; + } + } + + @SuppressWarnings("unchecked") + private static Class getH2SeekableClass() { + try { + return (Class) Class.forName( + "org.apache.parquet.hadoop.util.H2SeekableInputStream"); + } catch (ClassNotFoundException e) { + return null; + } catch (NoClassDefFoundError e) { + return null; + } + } + + private static Constructor getH2SeekableConstructor() { + Class h2SeekableClass = getH2SeekableClass(); + if (h2SeekableClass != null) { + try { + return h2SeekableClass.getConstructor(FSDataInputStream.class); + } catch (NoSuchMethodException e) { + return null; + } + } + return null; + } +} diff --git a/pom.xml b/pom.xml index 510c329749..acad5475ec 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,8 @@ 1.9.11 org.codehaus.jackson shaded.parquet - 1.1.0 + 2.3.0 + 1.1.0 2.5.3 3.0.3 2.3.1 @@ -80,7 +81,7 @@ 2.10 false 0.14.0 - + h2 0.7.0 6.5.7 0.9.33 @@ -507,19 +508,18 @@ - hadoop-2 + hadoop-1 hadoop.profile - hadoop2 + hadoop1 true - 2.3.0 - 0.14.0 - h2 + ${hadoop1.version} + From c80580c240a3d7a66857b9b7d8dd41ef26f12593 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 29 Jun 2016 10:36:27 -0700 Subject: [PATCH 02/11] PARQUET-400: Handle UnsupportedOperationException from read(ByteBuffer). --- .../hadoop/util/SeekableInputStream.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java index baefd7ec95..791f302dd2 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java @@ -43,7 +43,8 @@ public abstract class SeekableInputStream extends InputStream { public static SeekableInputStream wrap(FSDataInputStream stream) { if (byteBufferReadableClass != null && h2SeekableConstructor != null && - byteBufferReadableClass.isInstance(stream)) { + byteBufferReadableClass.isInstance(stream) && + supportsByteBufferReads(stream)) { try { return h2SeekableConstructor.newInstance(stream); } catch (InstantiationException e) { @@ -73,6 +74,23 @@ public static SeekableInputStream wrap(FSDataInputStream stream) { public abstract void readFully(ByteBuffer buf) throws IOException; + private static final ByteBuffer ZERO_LEN_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]); + + private static boolean supportsByteBufferReads(FSDataInputStream stream) { + // FSDataInputStream implements ByteBufferReadable, but may throw + // UnsupportedOperationException if it isn't actually supported. This tests + // whether the method throws by trying to read a 0-length buffer, which has + // no effect on the stream when it is supported. + try { + stream.read(ZERO_LEN_BYTE_BUFFER); + return true; + } catch (UnsupportedOperationException e) { + return false; + } catch (IOException e) { + return false; + } + } + private static Class getReadableClass() { try { return Class.forName("org.apache.hadoop.fs.ByteBufferReadable"); From 506a556cf9a57d8871a21dc64a3e774223eff62f Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 29 Jun 2016 10:45:26 -0700 Subject: [PATCH 03/11] PARQUET-400: Remove Hadoop dependencies from SeekableInputStream. --- .../parquet/hadoop/ParquetFileReader.java | 9 +- .../parquet/hadoop/ParquetFileWriter.java | 3 +- .../hadoop/util/HadoopInputStreams.java | 115 ++++++++++++++++++ .../hadoop/util/SeekableInputStream.java | 87 ------------- 4 files changed, 122 insertions(+), 92 deletions(-) create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 05228d52f6..9e12d557ce 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -89,6 +89,7 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.HiddenFileFilter; +import org.apache.parquet.hadoop.util.HadoopInputStreams; import org.apache.parquet.hadoop.util.SeekableInputStream; import org.apache.parquet.hadoop.util.counters.BenchmarkCounter; import org.apache.parquet.io.ParquetDecodingException; @@ -438,7 +439,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File */ public static final ParquetMetadata readFooter(Configuration configuration, FileStatus file, MetadataFilter filter) throws IOException { FileSystem fileSystem = file.getPath().getFileSystem(configuration); - SeekableInputStream in = SeekableInputStream.wrap(fileSystem.open(file.getPath())); + SeekableInputStream in = HadoopInputStreams.wrap(fileSystem.open(file.getPath())); try { return readFooter(file.getLen(), file.getPath().toString(), in, filter); } finally { @@ -537,7 +538,7 @@ public ParquetFileReader( this.conf = configuration; this.fileMetaData = fileMetaData; FileSystem fs = filePath.getFileSystem(configuration); - this.f = SeekableInputStream.wrap(fs.open(filePath)); + this.f = HadoopInputStreams.wrap(fs.open(filePath)); this.fileStatus = fs.getFileStatus(filePath); this.blocks = blocks; for (ColumnDescriptor col : columns) { @@ -568,7 +569,7 @@ public ParquetFileReader(Configuration conf, Path file, MetadataFilter filter) t this.conf = conf; FileSystem fs = file.getFileSystem(conf); this.fileStatus = fs.getFileStatus(file); - this.f = SeekableInputStream.wrap(fs.open(file)); + this.f = HadoopInputStreams.wrap(fs.open(file)); this.footer = readFooter(fileStatus.getLen(), fileStatus.getPath().toString(), f, filter); this.fileMetaData = footer.getFileMetaData(); this.blocks = footer.getBlocks(); @@ -591,7 +592,7 @@ public ParquetFileReader(Configuration conf, Path file, ParquetMetadata footer) this.conf = conf; FileSystem fs = file.getFileSystem(conf); this.fileStatus = fs.getFileStatus(file); - this.f = SeekableInputStream.wrap(fs.open(file)); + this.f = HadoopInputStreams.wrap(fs.open(file)); this.footer = footer; this.fileMetaData = footer.getFileMetaData(); this.blocks = footer.getBlocks(); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index 617f85722f..d6064d1168 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -61,6 +61,7 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.GlobalMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.hadoop.util.HadoopInputStreams; import org.apache.parquet.hadoop.util.SeekableInputStream; import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.schema.MessageType; @@ -496,7 +497,7 @@ public void appendFile(Configuration conf, Path file) throws IOException { public void appendRowGroups(FSDataInputStream file, List rowGroups, boolean dropColumns) throws IOException { - appendRowGroups(SeekableInputStream.wrap(file), rowGroups, dropColumns); + appendRowGroups(HadoopInputStreams.wrap(file), rowGroups, dropColumns); } public void appendRowGroups(SeekableInputStream file, diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java new file mode 100644 index 0000000000..22adbd3405 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.parquet.Log; +import org.apache.parquet.io.ParquetDecodingException; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; + +public class HadoopInputStreams { + + private static final Log LOG = Log.getLog(SeekableInputStream.class); + + private static final Class byteBufferReadableClass; + + private static final Constructor h2SeekableConstructor; + + static { + byteBufferReadableClass = getReadableClass(); + h2SeekableConstructor = getH2SeekableConstructor(); + } + + public static SeekableInputStream wrap(FSDataInputStream stream) { + if (byteBufferReadableClass != null && h2SeekableConstructor != null && + byteBufferReadableClass.isInstance(stream) && + supportsByteBufferReads(stream)) { + try { + return h2SeekableConstructor.newInstance(stream); + } catch (InstantiationException e) { + LOG.warn("Could not instantiate H2SeekableInputStream, falling back to byte array reads", e); + return new H1SeekableInputStream(stream); + } catch (IllegalAccessException e) { + LOG.warn("Could not instantiate H2SeekableInputStream, falling back to byte array reads", e); + return new H1SeekableInputStream(stream); + } catch (InvocationTargetException e) { + throw new ParquetDecodingException( + "Could not instantiate H2SeekableInputStream", e.getTargetException()); + } + } else { + return new H1SeekableInputStream(stream); + } + } + + private static final ByteBuffer ZERO_LEN_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]); + + private static boolean supportsByteBufferReads(FSDataInputStream stream) { + // FSDataInputStream implements ByteBufferReadable, but may throw + // UnsupportedOperationException if it isn't actually supported. This tests + // whether the method throws by trying to read a 0-length buffer, which has + // no effect on the stream when it is supported. + try { + stream.read(ZERO_LEN_BYTE_BUFFER); + return true; + } catch (UnsupportedOperationException e) { + return false; + } catch (IOException e) { + return false; + } + } + + private static Class getReadableClass() { + try { + return Class.forName("org.apache.hadoop.fs.ByteBufferReadable"); + } catch (ClassNotFoundException e) { + return null; + } catch (NoClassDefFoundError e) { + return null; + } + } + + @SuppressWarnings("unchecked") + private static Class getH2SeekableClass() { + try { + return (Class) Class.forName( + "org.apache.parquet.hadoop.util.H2SeekableInputStream"); + } catch (ClassNotFoundException e) { + return null; + } catch (NoClassDefFoundError e) { + return null; + } + } + + private static Constructor getH2SeekableConstructor() { + Class h2SeekableClass = getH2SeekableClass(); + if (h2SeekableClass != null) { + try { + return h2SeekableClass.getConstructor(FSDataInputStream.class); + } catch (NoSuchMethodException e) { + return null; + } + } + return null; + } + +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java index 791f302dd2..f319cd252f 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java @@ -19,49 +19,12 @@ package org.apache.parquet.hadoop.util; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.parquet.Log; -import org.apache.parquet.io.ParquetDecodingException; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; public abstract class SeekableInputStream extends InputStream { - private static final Log LOG = Log.getLog(SeekableInputStream.class); - - private static final Class byteBufferReadableClass; - - private static final Constructor h2SeekableConstructor; - - static { - byteBufferReadableClass = getReadableClass(); - h2SeekableConstructor = getH2SeekableConstructor(); - } - - public static SeekableInputStream wrap(FSDataInputStream stream) { - if (byteBufferReadableClass != null && h2SeekableConstructor != null && - byteBufferReadableClass.isInstance(stream) && - supportsByteBufferReads(stream)) { - try { - return h2SeekableConstructor.newInstance(stream); - } catch (InstantiationException e) { - LOG.warn("Could not instantiate H2SeekableInputStream, falling back to byte array reads", e); - return new H1SeekableInputStream(stream); - } catch (IllegalAccessException e) { - LOG.warn("Could not instantiate H2SeekableInputStream, falling back to byte array reads", e); - return new H1SeekableInputStream(stream); - } catch (InvocationTargetException e) { - throw new ParquetDecodingException( - "Could not instantiate H2SeekableInputStream", e.getTargetException()); - } - } else { - return new H1SeekableInputStream(stream); - } - } - public abstract long getPos() throws IOException; public abstract void seek(long newPos) throws IOException; @@ -74,54 +37,4 @@ public static SeekableInputStream wrap(FSDataInputStream stream) { public abstract void readFully(ByteBuffer buf) throws IOException; - private static final ByteBuffer ZERO_LEN_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]); - - private static boolean supportsByteBufferReads(FSDataInputStream stream) { - // FSDataInputStream implements ByteBufferReadable, but may throw - // UnsupportedOperationException if it isn't actually supported. This tests - // whether the method throws by trying to read a 0-length buffer, which has - // no effect on the stream when it is supported. - try { - stream.read(ZERO_LEN_BYTE_BUFFER); - return true; - } catch (UnsupportedOperationException e) { - return false; - } catch (IOException e) { - return false; - } - } - - private static Class getReadableClass() { - try { - return Class.forName("org.apache.hadoop.fs.ByteBufferReadable"); - } catch (ClassNotFoundException e) { - return null; - } catch (NoClassDefFoundError e) { - return null; - } - } - - @SuppressWarnings("unchecked") - private static Class getH2SeekableClass() { - try { - return (Class) Class.forName( - "org.apache.parquet.hadoop.util.H2SeekableInputStream"); - } catch (ClassNotFoundException e) { - return null; - } catch (NoClassDefFoundError e) { - return null; - } - } - - private static Constructor getH2SeekableConstructor() { - Class h2SeekableClass = getH2SeekableClass(); - if (h2SeekableClass != null) { - try { - return h2SeekableClass.getConstructor(FSDataInputStream.class); - } catch (NoSuchMethodException e) { - return null; - } - } - return null; - } } From 730a9e26d9cacef1350d99d56276c158e0799882 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 29 Jun 2016 10:59:42 -0700 Subject: [PATCH 04/11] PARQUET-400: Move SeekableInputStream to io package. --- .../org/apache/parquet/io}/SeekableInputStream.java | 2 +- .../org/apache/parquet/hadoop/ParquetFileReader.java | 12 ++++++------ .../org/apache/parquet/hadoop/ParquetFileWriter.java | 6 +++--- .../parquet/hadoop/util/H1SeekableInputStream.java | 1 + .../parquet/hadoop/util/H2SeekableInputStream.java | 1 + .../{HadoopInputStreams.java => HadoopStreams.java} | 3 ++- 6 files changed, 14 insertions(+), 11 deletions(-) rename {parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util => parquet-common/src/main/java/org/apache/parquet/io}/SeekableInputStream.java (97%) rename parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/{HadoopInputStreams.java => HadoopStreams.java} (98%) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java b/parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java similarity index 97% rename from parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java rename to parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java index f319cd252f..796279b9ee 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/SeekableInputStream.java +++ b/parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.parquet.hadoop.util; +package org.apache.parquet.io; import java.io.IOException; import java.io.InputStream; diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 9e12d557ce..26922d7f2d 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -89,8 +89,8 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.hadoop.util.HiddenFileFilter; -import org.apache.parquet.hadoop.util.HadoopInputStreams; -import org.apache.parquet.hadoop.util.SeekableInputStream; +import org.apache.parquet.hadoop.util.HadoopStreams; +import org.apache.parquet.io.SeekableInputStream; import org.apache.parquet.hadoop.util.counters.BenchmarkCounter; import org.apache.parquet.io.ParquetDecodingException; @@ -439,7 +439,7 @@ public static final ParquetMetadata readFooter(Configuration configuration, File */ public static final ParquetMetadata readFooter(Configuration configuration, FileStatus file, MetadataFilter filter) throws IOException { FileSystem fileSystem = file.getPath().getFileSystem(configuration); - SeekableInputStream in = HadoopInputStreams.wrap(fileSystem.open(file.getPath())); + SeekableInputStream in = HadoopStreams.wrap(fileSystem.open(file.getPath())); try { return readFooter(file.getLen(), file.getPath().toString(), in, filter); } finally { @@ -538,7 +538,7 @@ public ParquetFileReader( this.conf = configuration; this.fileMetaData = fileMetaData; FileSystem fs = filePath.getFileSystem(configuration); - this.f = HadoopInputStreams.wrap(fs.open(filePath)); + this.f = HadoopStreams.wrap(fs.open(filePath)); this.fileStatus = fs.getFileStatus(filePath); this.blocks = blocks; for (ColumnDescriptor col : columns) { @@ -569,7 +569,7 @@ public ParquetFileReader(Configuration conf, Path file, MetadataFilter filter) t this.conf = conf; FileSystem fs = file.getFileSystem(conf); this.fileStatus = fs.getFileStatus(file); - this.f = HadoopInputStreams.wrap(fs.open(file)); + this.f = HadoopStreams.wrap(fs.open(file)); this.footer = readFooter(fileStatus.getLen(), fileStatus.getPath().toString(), f, filter); this.fileMetaData = footer.getFileMetaData(); this.blocks = footer.getBlocks(); @@ -592,7 +592,7 @@ public ParquetFileReader(Configuration conf, Path file, ParquetMetadata footer) this.conf = conf; FileSystem fs = file.getFileSystem(conf); this.fileStatus = fs.getFileStatus(file); - this.f = HadoopInputStreams.wrap(fs.open(file)); + this.f = HadoopStreams.wrap(fs.open(file)); this.footer = footer; this.fileMetaData = footer.getFileMetaData(); this.blocks = footer.getBlocks(); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index d6064d1168..f0fa7f5ace 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -61,8 +61,8 @@ import org.apache.parquet.hadoop.metadata.FileMetaData; import org.apache.parquet.hadoop.metadata.GlobalMetaData; import org.apache.parquet.hadoop.metadata.ParquetMetadata; -import org.apache.parquet.hadoop.util.HadoopInputStreams; -import org.apache.parquet.hadoop.util.SeekableInputStream; +import org.apache.parquet.hadoop.util.HadoopStreams; +import org.apache.parquet.io.SeekableInputStream; import org.apache.parquet.io.ParquetEncodingException; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; @@ -497,7 +497,7 @@ public void appendFile(Configuration conf, Path file) throws IOException { public void appendRowGroups(FSDataInputStream file, List rowGroups, boolean dropColumns) throws IOException { - appendRowGroups(HadoopInputStreams.wrap(file), rowGroups, dropColumns); + appendRowGroups(HadoopStreams.wrap(file), rowGroups, dropColumns); } public void appendRowGroups(SeekableInputStream file, diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java index cbb42a260e..edf8285ba5 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java @@ -20,6 +20,7 @@ package org.apache.parquet.hadoop.util; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.parquet.io.SeekableInputStream; import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java index 7f9a9c40a1..4ed1a1c882 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.ByteBufferReadable; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.parquet.Preconditions; +import org.apache.parquet.io.SeekableInputStream; import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java similarity index 98% rename from parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java rename to parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java index 22adbd3405..d564d10182 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputStreams.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java @@ -22,12 +22,13 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.parquet.Log; import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.io.SeekableInputStream; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; -public class HadoopInputStreams { +public class HadoopStreams { private static final Log LOG = Log.getLog(SeekableInputStream.class); From 5dc50a560035f006e19e0a1d1b1443b83d150153 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 29 Jun 2016 12:40:07 -0700 Subject: [PATCH 05/11] PARQUET-400: Add tests for H1SeekableInputStream methods. --- .../hadoop/util/H1SeekableInputStream.java | 21 +- .../hadoop/util/TestByteBufferReads.java | 826 ++++++++++++++++++ 2 files changed, 840 insertions(+), 7 deletions(-) create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java index edf8285ba5..19ccc66110 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java @@ -89,20 +89,25 @@ public void readFully(ByteBuffer buf) throws IOException { } } - // Visisble for testing + // Visible for testing static int readHeapBuffer(FSDataInputStream f, ByteBuffer buf) throws IOException { int bytesRead = f.read(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining()); - buf.position(buf.position() + bytesRead); - return bytesRead; + if (bytesRead < 0) { + // if this resulted in EOF, don't update position + return bytesRead; + } else { + buf.position(buf.position() + bytesRead); + return bytesRead; + } } - // Visisble for testing + // Visible for testing static void readFullyHeapBuffer(FSDataInputStream f, ByteBuffer buf) throws IOException { f.readFully(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining()); buf.position(buf.limit()); } - // Visisble for testing + // Visible for testing static int readDirectBuffer(FSDataInputStream f, ByteBuffer buf, byte[] temp) throws IOException { // copy all the bytes that return immediately, stopping at the first // read that doesn't return a full buffer. @@ -127,14 +132,16 @@ static int readDirectBuffer(FSDataInputStream f, ByteBuffer buf, byte[] temp) th } } - // Visisble for testing + // Visible for testing static void readFullyDirectBuffer(FSDataInputStream f, ByteBuffer buf, byte[] temp) throws IOException { int nextReadLength = Math.min(buf.remaining(), temp.length); int bytesRead = 0; - while (nextReadLength > 0 && (bytesRead = f.read(temp, 0, nextReadLength)) > 0) { + + while (nextReadLength > 0 && (bytesRead = f.read(temp, 0, nextReadLength)) >= 0) { buf.put(temp, 0, bytesRead); nextReadLength = Math.min(buf.remaining(), temp.length); } + if (bytesRead < 0 && buf.remaining() > 0) { throw new EOFException( "Reached the end of stream. Still have: " + buf.remaining() + " bytes left"); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java new file mode 100644 index 0000000000..afe1de1524 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java @@ -0,0 +1,826 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import org.apache.parquet.hadoop.TestUtils; +import org.junit.Assert; +import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.Callable; + +public class TestByteBufferReads { + + private static final byte[] TEST_ARRAY = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + + private static class MockInputStream extends ByteArrayInputStream + implements Seekable, PositionedReadable { + private int[] lengths; + private int current = 0; + MockInputStream(int... actualReadLengths) { + super(TEST_ARRAY); + this.lengths = actualReadLengths; + } + + @Override + public synchronized int read(byte[] b, int off, int len) { + if (current < lengths.length) { + if (len <= lengths[current]) { + // when len == lengths[current], the next read will by 0 bytes + int bytesRead = super.read(b, off, len); + lengths[current] -= bytesRead; + return bytesRead; + } else { + int bytesRead = super.read(b, off, lengths[current]); + current += 1; + return bytesRead; + } + } else { + return super.read(b, off, len); + } + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) throws IOException { + seek(position); + return read(buffer, offset, length); + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { + throw new UnsupportedOperationException("Not acutally supported."); + } + + @Override + public void readFully(long position, byte[] buffer) throws IOException { + throw new UnsupportedOperationException("Not acutally supported."); + } + + @Override + public void seek(long pos) throws IOException { + this.pos = (int) pos; + } + + @Override + public long getPos() throws IOException { + return this.pos; + } + + @Override + public boolean seekToNewSource(long targetPos) throws IOException { + seek(targetPos); + return true; + } + } + + @Test + public void testHeapRead() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(20); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + int len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, len); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(-1, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapSmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(5); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + int len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(5, len); + Assert.assertEquals(5, readBuffer.position()); + Assert.assertEquals(5, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(0, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 5), readBuffer); + } + + @Test + public void testHeapSmallReads() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + int len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(2, len); + Assert.assertEquals(2, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(3, len); + Assert.assertEquals(5, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(3, len); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(2, len); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapPosition() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(20); + readBuffer.position(10); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(8)); + + int len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(8, len); + Assert.assertEquals(18, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(2, len); + Assert.assertEquals(20, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(-1, len); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(20); + readBuffer.limit(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(7)); + + int len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(7, len); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(1, len); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(0, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testHeapPositionAndLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(20); + readBuffer.position(5); + readBuffer.limit(13); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(7)); + + int len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(7, len); + Assert.assertEquals(12, readBuffer.position()); + Assert.assertEquals(13, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(1, len); + Assert.assertEquals(13, readBuffer.position()); + Assert.assertEquals(13, readBuffer.limit()); + + len = H1SeekableInputStream.readHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(0, len); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + private static final ThreadLocal TEMP = new ThreadLocal() { + @Override + protected byte[] initialValue() { + return new byte[8192]; + } + }; + + @Test + public void testDirectRead() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, len); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(-1, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectSmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(5); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(5, len); + Assert.assertEquals(5, readBuffer.position()); + Assert.assertEquals(5, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(0, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 5), readBuffer); + } + + @Test + public void testDirectSmallReads() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(2, len); + Assert.assertEquals(2, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(3, len); + Assert.assertEquals(5, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(3, len); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(2, len); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectPosition() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + readBuffer.position(10); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(8)); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(8, len); + Assert.assertEquals(18, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(2, len); + Assert.assertEquals(20, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(-1, len); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(20); + readBuffer.limit(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(7)); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(7, len); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(1, len); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(0, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testDirectPositionAndLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + readBuffer.position(5); + readBuffer.limit(13); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(7)); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(7, len); + Assert.assertEquals(12, readBuffer.position()); + Assert.assertEquals(13, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(1, len); + Assert.assertEquals(13, readBuffer.position()); + Assert.assertEquals(13, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(0, len); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testDirectSmallTempBufferSmallReads() throws Exception { + byte[] temp = new byte[2]; // this will cause readDirectBuffer to loop + + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(2, len); + Assert.assertEquals(2, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(3, len); + Assert.assertEquals(5, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(3, len); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(2, len); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(-1, len); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectSmallTempBufferWithPositionAndLimit() throws Exception { + byte[] temp = new byte[2]; // this will cause readDirectBuffer to loop + + ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + readBuffer.position(5); + readBuffer.limit(13); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(7)); + + int len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(7, len); + Assert.assertEquals(12, readBuffer.position()); + Assert.assertEquals(13, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(1, len); + Assert.assertEquals(13, readBuffer.position()); + Assert.assertEquals(13, readBuffer.limit()); + + len = H1SeekableInputStream.readDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(0, len); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testHeapReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testHeapReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(20); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + return null; + } + }); + + Assert.assertEquals(0, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testHeapReadFullyJustRight() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + // reads all of the bytes available without EOFException + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapReadFullySmallReads() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapReadFullyPosition() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testHeapReadFullyLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.limit(7); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapReadFullyPositionAndLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H1SeekableInputStream.readFullyHeapBuffer(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testDirectReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testDirectReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testDirectReadFullyJustRight() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + + // reads all of the bytes available without EOFException + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectReadFullySmallReads() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectReadFullyPosition() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testDirectReadFullyLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.limit(7); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectReadFullyPositionAndLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, TEMP.get()); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testDirectReadFullySmallTempBufferWithPositionAndLimit() throws Exception { + byte[] temp = new byte[2]; // this will cause readFully to loop + + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H1SeekableInputStream.readFullyDirectBuffer(hadoopStream, readBuffer, temp); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + +} From abaa69584993014649a8e12c2bf3c74c28f95312 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 6 Jul 2016 14:29:47 -0700 Subject: [PATCH 06/11] PARQUET-400: Fix review items. This commit: * Makes SeekableInputStream implementations package private * Adds tests for the H2 stream wrapper * Adds javadoc to SeekableInputStream * Simplifies the wrap method by checking the underlying stream --- .../parquet/io/SeekableInputStream.java | 66 +++ .../hadoop/util/H1SeekableInputStream.java | 6 +- .../hadoop/util/H2SeekableInputStream.java | 12 +- .../parquet/hadoop/util/HadoopStreams.java | 42 +- .../hadoop/util/H2MockInputStream.java | 42 ++ .../parquet/hadoop/util/MockInputStream.java | 87 ++++ .../hadoop/util/TestByteBufferReads.java | 468 ++++++++++++++---- 7 files changed, 592 insertions(+), 131 deletions(-) create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/MockInputStream.java diff --git a/parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java b/parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java index 796279b9ee..7247817467 100644 --- a/parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java +++ b/parquet-common/src/main/java/org/apache/parquet/io/SeekableInputStream.java @@ -19,22 +19,88 @@ package org.apache.parquet.io; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +/** + * {@code SeekableInputStream} is an interface with the methods needed by + * Parquet to read data from a file or Hadoop data stream. + */ public abstract class SeekableInputStream extends InputStream { + /** + * Return the current position in the InputStream. + * + * @return current position in bytes from the start of the stream + * @throws IOException If the underlying stream throws IOException + */ public abstract long getPos() throws IOException; + /** + * Seek to a new position in the InputStream. + * + * @param newPos the new position to seek to + * @throws IOException If the underlying stream throws IOException + */ public abstract void seek(long newPos) throws IOException; + /** + * Read a byte array of data, from position 0 to the end of the array. + *

+ * This method is equivalent to {@code read(bytes, 0, bytes.length)}. + *

+ * This method will block until len bytes are available to copy into the + * array, or will throw {@link EOFException} if the stream ends before the + * array is full. + * + * @param bytes a byte array to fill with data from the stream + * @throws IOException If the underlying stream throws IOException + * @throws EOFException If the stream has fewer bytes left than are needed to + * fill the array, {@code bytes.length} + */ public abstract void readFully(byte[] bytes) throws IOException; + /** + * Read {@code len} bytes of data into an array, at position {@code start}. + *

+ * This method will block until len bytes are available to copy into the + * array, or will throw {@link EOFException} if the stream ends before the + * array is full. + * + * @param bytes a byte array to fill with data from the stream + * @throws IOException If the underlying stream throws IOException + * @throws EOFException If the stream has fewer than {@code len} bytes left + */ public abstract void readFully(byte[] bytes, int start, int len) throws IOException; + /** + * Read {@code buf.remaining()} bytes of data into a {@link ByteBuffer}. + *

+ * This method will copy available bytes into the buffer, reading at most + * {@code buf.remaining()} bytes. The number of bytes actually copied is + * returned by the method, or -1 is returned to signal that the end of the + * underlying stream has been reached. + * + * @param buf a byte array to fill with data from the stream + * @return the number of bytes read or -1 if the stream ended + * @throws IOException If the underlying stream throws IOException + */ public abstract int read(ByteBuffer buf) throws IOException; + /** + * Read {@code buf.remaining()} bytes of data into a {@link ByteBuffer}. + *

+ * This method will block until {@code buf.remaining()} bytes are available + * to copy into the buffer, or will throw {@link EOFException} if the stream + * ends before the buffer is full. + * + * @param buf a byte array to fill with data from the stream + * @throws IOException If the underlying stream throws IOException + * @throws EOFException If the stream has fewer bytes left than are needed to + * fill the buffer, {@code buf.remaining()} + */ public abstract void readFully(ByteBuffer buf) throws IOException; } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java index 19ccc66110..4a03b1a80a 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H1SeekableInputStream.java @@ -25,7 +25,11 @@ import java.io.IOException; import java.nio.ByteBuffer; -public class H1SeekableInputStream extends SeekableInputStream { +/** + * SeekableInputStream implementation that implements read(ByteBuffer) for + * Hadoop 1 FSDataInputStream. + */ +class H1SeekableInputStream extends SeekableInputStream { private final int COPY_BUFFER_SIZE = 8192; private final byte[] temp = new byte[COPY_BUFFER_SIZE]; diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java index 4ed1a1c882..474efdb433 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java @@ -27,7 +27,11 @@ import java.io.IOException; import java.nio.ByteBuffer; -public class H2SeekableInputStream extends SeekableInputStream { +/** + * SeekableInputStream implementation for FSDataInputStream that implements + * ByteBufferReadable in Hadoop 2. + */ +class H2SeekableInputStream extends SeekableInputStream { private final FSDataInputStream stream; @@ -74,11 +78,15 @@ public int read(ByteBuffer buf) throws IOException { @Override public void readFully(ByteBuffer buf) throws IOException { + readFully(stream, buf); + } + + public static void readFully(FSDataInputStream stream, ByteBuffer buf) throws IOException { // unfortunately the Hadoop APIs seem to not have a 'readFully' equivalent for the byteBuffer read // calls. The read(ByteBuffer) call might read fewer than byteBuffer.hasRemaining() bytes. Thus we // have to loop to ensure we read them. while (buf.hasRemaining()) { - int readCount = read(buf); + int readCount = stream.read(buf); if (readCount == -1) { // this is probably a bug in the ParquetReader. We shouldn't have called readFully with a buffer // that has more remaining than the amount of data in the stream. diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java index d564d10182..2e82bda8b5 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java @@ -23,28 +23,29 @@ import org.apache.parquet.Log; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.SeekableInputStream; -import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; +/** + * Convenience methods to get Parquet abstractions for Hadoop data streams. + */ public class HadoopStreams { private static final Log LOG = Log.getLog(SeekableInputStream.class); - private static final Class byteBufferReadableClass; - - private static final Constructor h2SeekableConstructor; - - static { - byteBufferReadableClass = getReadableClass(); - h2SeekableConstructor = getH2SeekableConstructor(); - } + private static final Class byteBufferReadableClass = getReadableClass(); + private static final Constructor h2SeekableConstructor = getH2SeekableConstructor(); + /** + * Wraps a {@link FSDataInputStream} in a {@link SeekableInputStream} + * implementation for Parquet readers. + * + * @param stream a Hadoop FSDataInputStream + * @return a SeekableInputStream + */ public static SeekableInputStream wrap(FSDataInputStream stream) { if (byteBufferReadableClass != null && h2SeekableConstructor != null && - byteBufferReadableClass.isInstance(stream) && - supportsByteBufferReads(stream)) { + byteBufferReadableClass.isInstance(stream.getWrappedStream())) { try { return h2SeekableConstructor.newInstance(stream); } catch (InstantiationException e) { @@ -62,23 +63,6 @@ public static SeekableInputStream wrap(FSDataInputStream stream) { } } - private static final ByteBuffer ZERO_LEN_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]); - - private static boolean supportsByteBufferReads(FSDataInputStream stream) { - // FSDataInputStream implements ByteBufferReadable, but may throw - // UnsupportedOperationException if it isn't actually supported. This tests - // whether the method throws by trying to read a 0-length buffer, which has - // no effect on the stream when it is supported. - try { - stream.read(ZERO_LEN_BYTE_BUFFER); - return true; - } catch (UnsupportedOperationException e) { - return false; - } catch (IOException e) { - return false; - } - } - private static Class getReadableClass() { try { return Class.forName("org.apache.hadoop.fs.ByteBufferReadable"); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java new file mode 100644 index 0000000000..f5ea721dee --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.ByteBufferReadable; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class H2MockInputStream extends MockInputStream implements ByteBufferReadable { + public H2MockInputStream(int... actualReadLengths) { + super(actualReadLengths); + } + + @Override + public int read(ByteBuffer buf) throws IOException { + // this is inefficient, but simple for correctness tests of + // readFully(ByteBuffer) + byte[] temp = new byte[buf.remaining()]; + int bytesRead = read(temp, 0, temp.length); + if (bytesRead > 0) { + buf.put(temp, 0, bytesRead); + } + return bytesRead; + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/MockInputStream.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/MockInputStream.java new file mode 100644 index 0000000000..a112288024 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/MockInputStream.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.PositionedReadable; +import org.apache.hadoop.fs.Seekable; +import java.io.ByteArrayInputStream; +import java.io.IOException; + +class MockInputStream extends ByteArrayInputStream + implements Seekable, PositionedReadable { + static final byte[] TEST_ARRAY = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + + private int[] lengths; + private int current = 0; + MockInputStream(int... actualReadLengths) { + super(TEST_ARRAY); + this.lengths = actualReadLengths; + } + + @Override + public synchronized int read(byte[] b, int off, int len) { + if (current < lengths.length) { + if (len <= lengths[current]) { + // when len == lengths[current], the next read will by 0 bytes + int bytesRead = super.read(b, off, len); + lengths[current] -= bytesRead; + return bytesRead; + } else { + int bytesRead = super.read(b, off, lengths[current]); + current += 1; + return bytesRead; + } + } else { + return super.read(b, off, len); + } + } + + @Override + public int read(long position, byte[] buffer, int offset, int length) throws IOException { + seek(position); + return read(buffer, offset, length); + } + + @Override + public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { + throw new UnsupportedOperationException("Not actually supported."); + } + + @Override + public void readFully(long position, byte[] buffer) throws IOException { + throw new UnsupportedOperationException("Not actually supported."); + } + + @Override + public void seek(long pos) throws IOException { + this.pos = (int) pos; + } + + @Override + public long getPos() throws IOException { + return this.pos; + } + + @Override + public boolean seekToNewSource(long targetPos) throws IOException { + seek(targetPos); + return true; + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java index afe1de1524..683ea61d8d 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java @@ -20,83 +20,26 @@ package org.apache.parquet.hadoop.util; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.PositionedReadable; -import org.apache.hadoop.fs.Seekable; import org.apache.parquet.hadoop.TestUtils; import org.junit.Assert; import org.junit.Test; -import java.io.ByteArrayInputStream; import java.io.EOFException; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.Callable; -public class TestByteBufferReads { - - private static final byte[] TEST_ARRAY = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - - private static class MockInputStream extends ByteArrayInputStream - implements Seekable, PositionedReadable { - private int[] lengths; - private int current = 0; - MockInputStream(int... actualReadLengths) { - super(TEST_ARRAY); - this.lengths = actualReadLengths; - } - - @Override - public synchronized int read(byte[] b, int off, int len) { - if (current < lengths.length) { - if (len <= lengths[current]) { - // when len == lengths[current], the next read will by 0 bytes - int bytesRead = super.read(b, off, len); - lengths[current] -= bytesRead; - return bytesRead; - } else { - int bytesRead = super.read(b, off, lengths[current]); - current += 1; - return bytesRead; - } - } else { - return super.read(b, off, len); - } - } - - @Override - public int read(long position, byte[] buffer, int offset, int length) throws IOException { - seek(position); - return read(buffer, offset, length); - } +import static org.apache.parquet.hadoop.util.MockInputStream.TEST_ARRAY; - @Override - public void readFully(long position, byte[] buffer, int offset, int length) throws IOException { - throw new UnsupportedOperationException("Not acutally supported."); - } - - @Override - public void readFully(long position, byte[] buffer) throws IOException { - throw new UnsupportedOperationException("Not acutally supported."); - } - - @Override - public void seek(long pos) throws IOException { - this.pos = (int) pos; - } - - @Override - public long getPos() throws IOException { - return this.pos; - } +public class TestByteBufferReads { + private static final ThreadLocal TEMP = new ThreadLocal() { @Override - public boolean seekToNewSource(long targetPos) throws IOException { - seek(targetPos); - return true; + protected byte[] initialValue() { + return new byte[8192]; } - } + }; @Test - public void testHeapRead() throws Exception { + public void testH1HeapRead() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -115,7 +58,7 @@ public void testHeapRead() throws Exception { } @Test - public void testHeapSmallBuffer() throws Exception { + public void testH1HeapSmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(5); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -134,7 +77,7 @@ public void testHeapSmallBuffer() throws Exception { } @Test - public void testHeapSmallReads() throws Exception { + public void testH1HeapSmallReads() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(10); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -165,7 +108,7 @@ public void testHeapSmallReads() throws Exception { } @Test - public void testHeapPosition() throws Exception { + public void testH1HeapPosition() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.position(10); readBuffer.mark(); @@ -191,7 +134,7 @@ public void testHeapPosition() throws Exception { } @Test - public void testHeapLimit() throws Exception { + public void testH1HeapLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.limit(8); @@ -216,7 +159,7 @@ public void testHeapLimit() throws Exception { } @Test - public void testHeapPositionAndLimit() throws Exception { + public void testH1HeapPositionAndLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.position(5); readBuffer.limit(13); @@ -242,15 +185,8 @@ public void testHeapPositionAndLimit() throws Exception { ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); } - private static final ThreadLocal TEMP = new ThreadLocal() { - @Override - protected byte[] initialValue() { - return new byte[8192]; - } - }; - @Test - public void testDirectRead() throws Exception { + public void testH1DirectRead() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -269,7 +205,7 @@ public void testDirectRead() throws Exception { } @Test - public void testDirectSmallBuffer() throws Exception { + public void testH1DirectSmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(5); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -288,7 +224,7 @@ public void testDirectSmallBuffer() throws Exception { } @Test - public void testDirectSmallReads() throws Exception { + public void testH1DirectSmallReads() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -319,7 +255,7 @@ public void testDirectSmallReads() throws Exception { } @Test - public void testDirectPosition() throws Exception { + public void testH1DirectPosition() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); readBuffer.position(10); readBuffer.mark(); @@ -345,7 +281,7 @@ public void testDirectPosition() throws Exception { } @Test - public void testDirectLimit() throws Exception { + public void testH1DirectLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.limit(8); @@ -370,7 +306,7 @@ public void testDirectLimit() throws Exception { } @Test - public void testDirectPositionAndLimit() throws Exception { + public void testH1DirectPositionAndLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); readBuffer.position(5); readBuffer.limit(13); @@ -397,7 +333,7 @@ public void testDirectPositionAndLimit() throws Exception { } @Test - public void testDirectSmallTempBufferSmallReads() throws Exception { + public void testH1DirectSmallTempBufferSmallReads() throws Exception { byte[] temp = new byte[2]; // this will cause readDirectBuffer to loop ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); @@ -433,7 +369,7 @@ public void testDirectSmallTempBufferSmallReads() throws Exception { } @Test - public void testDirectSmallTempBufferWithPositionAndLimit() throws Exception { + public void testH1DirectSmallTempBufferWithPositionAndLimit() throws Exception { byte[] temp = new byte[2]; // this will cause readDirectBuffer to loop ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); @@ -462,7 +398,7 @@ public void testDirectSmallTempBufferWithPositionAndLimit() throws Exception { } @Test - public void testHeapReadFullySmallBuffer() throws Exception { + public void testH1HeapReadFullySmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(8); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -481,7 +417,7 @@ public void testHeapReadFullySmallBuffer() throws Exception { } @Test - public void testHeapReadFullyLargeBuffer() throws Exception { + public void testH1HeapReadFullyLargeBuffer() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(20); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -500,7 +436,7 @@ public Object call() throws Exception { } @Test - public void testHeapReadFullyJustRight() throws Exception { + public void testH1HeapReadFullyJustRight() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -521,7 +457,7 @@ public void testHeapReadFullyJustRight() throws Exception { } @Test - public void testHeapReadFullySmallReads() throws Exception { + public void testH1HeapReadFullySmallReads() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -540,7 +476,7 @@ public void testHeapReadFullySmallReads() throws Exception { } @Test - public void testHeapReadFullyPosition() throws Exception { + public void testH1HeapReadFullyPosition() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); readBuffer.position(3); readBuffer.mark(); @@ -561,7 +497,7 @@ public void testHeapReadFullyPosition() throws Exception { } @Test - public void testHeapReadFullyLimit() throws Exception { + public void testH1HeapReadFullyLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); readBuffer.limit(7); @@ -591,7 +527,7 @@ public void testHeapReadFullyLimit() throws Exception { } @Test - public void testHeapReadFullyPositionAndLimit() throws Exception { + public void testH1HeapReadFullyPositionAndLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); readBuffer.position(3); readBuffer.limit(7); @@ -623,7 +559,7 @@ public void testHeapReadFullyPositionAndLimit() throws Exception { } @Test - public void testDirectReadFullySmallBuffer() throws Exception { + public void testH1DirectReadFullySmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -642,7 +578,7 @@ public void testDirectReadFullySmallBuffer() throws Exception { } @Test - public void testDirectReadFullyLargeBuffer() throws Exception { + public void testH1DirectReadFullyLargeBuffer() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -667,7 +603,7 @@ public Object call() throws Exception { } @Test - public void testDirectReadFullyJustRight() throws Exception { + public void testH1DirectReadFullyJustRight() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -688,7 +624,7 @@ public void testDirectReadFullyJustRight() throws Exception { } @Test - public void testDirectReadFullySmallReads() throws Exception { + public void testH1DirectReadFullySmallReads() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -707,7 +643,7 @@ public void testDirectReadFullySmallReads() throws Exception { } @Test - public void testDirectReadFullyPosition() throws Exception { + public void testH1DirectReadFullyPosition() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); readBuffer.position(3); readBuffer.mark(); @@ -728,7 +664,7 @@ public void testDirectReadFullyPosition() throws Exception { } @Test - public void testDirectReadFullyLimit() throws Exception { + public void testH1DirectReadFullyLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); readBuffer.limit(7); @@ -758,7 +694,7 @@ public void testDirectReadFullyLimit() throws Exception { } @Test - public void testDirectReadFullyPositionAndLimit() throws Exception { + public void testH1DirectReadFullyPositionAndLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); readBuffer.position(3); readBuffer.limit(7); @@ -790,7 +726,7 @@ public void testDirectReadFullyPositionAndLimit() throws Exception { } @Test - public void testDirectReadFullySmallTempBufferWithPositionAndLimit() throws Exception { + public void testH1DirectReadFullySmallTempBufferWithPositionAndLimit() throws Exception { byte[] temp = new byte[2]; // this will cause readFully to loop final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); @@ -823,4 +759,338 @@ public void testDirectReadFullySmallTempBufferWithPositionAndLimit() throws Exce ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); } + @Test + public void testH2HeapReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testH2HeapReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(20); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testH2HeapReadFullyJustRight() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); + + // reads all of the bytes available without EOFException + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2HeapReadFullySmallReads() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2HeapReadFullyPosition() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testH2HeapReadFullyLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.limit(7); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2HeapReadFullyPositionAndLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testH2DirectReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testH2DirectReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testH2DirectReadFullyJustRight() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); + + // reads all of the bytes available without EOFException + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2DirectReadFullySmallReads() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2DirectReadFullyPosition() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testH2DirectReadFullyLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.limit(7); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2DirectReadFullyPositionAndLimit() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(hadoopStream, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + } From 2cb69345e8946ebbafba23db5682fa5b123e5571 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 6 Jul 2016 14:52:39 -0700 Subject: [PATCH 07/11] PARQUET-400: Remove H2SeekableInputStream tests. These tests fail in Hadoop-1 and would require a new module. --- .../hadoop/util/H2MockInputStream.java | 42 --- .../hadoop/util/TestByteBufferReads.java | 334 ------------------ 2 files changed, 376 deletions(-) delete mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java deleted file mode 100644 index f5ea721dee..0000000000 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/H2MockInputStream.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.parquet.hadoop.util; - -import org.apache.hadoop.fs.ByteBufferReadable; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class H2MockInputStream extends MockInputStream implements ByteBufferReadable { - public H2MockInputStream(int... actualReadLengths) { - super(actualReadLengths); - } - - @Override - public int read(ByteBuffer buf) throws IOException { - // this is inefficient, but simple for correctness tests of - // readFully(ByteBuffer) - byte[] temp = new byte[buf.remaining()]; - int bytesRead = read(temp, 0, temp.length); - if (bytesRead > 0) { - buf.put(temp, 0, bytesRead); - } - return bytesRead; - } -} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java index 683ea61d8d..47038c0e4a 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java @@ -759,338 +759,4 @@ public void testH1DirectReadFullySmallTempBufferWithPositionAndLimit() throws Ex ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); } - @Test - public void testH2HeapReadFullySmallBuffer() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(8); - - FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); - } - - @Test - public void testH2HeapReadFullyLargeBuffer() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(20); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); - - TestUtils.assertThrows("Should throw EOFException", - EOFException.class, new Callable() { - @Override - public Object call() throws Exception { - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - return null; - } - }); - - // NOTE: This behavior differs from readFullyHeapBuffer because direct uses - // several read operations that will read up to the end of the input. This - // is a correct value because the bytes in the buffer are valid. This - // behavior can't be implemented for the heap buffer without using the read - // method instead of the readFully method on the underlying - // FSDataInputStream. - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(20, readBuffer.limit()); - } - - @Test - public void testH2HeapReadFullyJustRight() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(10); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); - - // reads all of the bytes available without EOFException - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - // trying to read 0 more bytes doesn't result in EOFException - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2HeapReadFullySmallReads() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(10); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2HeapReadFullyPosition() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(10); - readBuffer.position(3); - readBuffer.mark(); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - - @Test - public void testH2HeapReadFullyLimit() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(10); - readBuffer.limit(7); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2HeapReadFullyPositionAndLimit() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(10); - readBuffer.position(3); - readBuffer.limit(7); - readBuffer.mark(); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - - @Test - public void testH2DirectReadFullySmallBuffer() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); - - FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); - } - - @Test - public void testH2DirectReadFullyLargeBuffer() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); - - TestUtils.assertThrows("Should throw EOFException", - EOFException.class, new Callable() { - @Override - public Object call() throws Exception { - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - return null; - } - }); - - // NOTE: This behavior differs from readFullyHeapBuffer because direct uses - // several read operations that will read up to the end of the input. This - // is a correct value because the bytes in the buffer are valid. This - // behavior can't be implemented for the heap buffer without using the read - // method instead of the readFully method on the underlying - // FSDataInputStream. - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(20, readBuffer.limit()); - } - - @Test - public void testH2DirectReadFullyJustRight() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream()); - - // reads all of the bytes available without EOFException - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - // trying to read 0 more bytes doesn't result in EOFException - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2DirectReadFullySmallReads() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2DirectReadFullyPosition() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - readBuffer.position(3); - readBuffer.mark(); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - - @Test - public void testH2DirectReadFullyLimit() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - readBuffer.limit(7); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2DirectReadFullyPositionAndLimit() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - readBuffer.position(3); - readBuffer.limit(7); - readBuffer.mark(); - - final FSDataInputStream hadoopStream = new FSDataInputStream(new H2MockInputStream(2, 3, 3)); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(hadoopStream, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - } From b5430131efa4a4ab878fda86a750c5db8f6f673d Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 6 Jul 2016 15:11:20 -0700 Subject: [PATCH 08/11] PARQUET-400: Fix logger for HadoopStreams. --- .../main/java/org/apache/parquet/hadoop/util/HadoopStreams.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java index 2e82bda8b5..6edb041f7c 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java @@ -31,7 +31,7 @@ */ public class HadoopStreams { - private static final Log LOG = Log.getLog(SeekableInputStream.class); + private static final Log LOG = Log.getLog(HadoopStreams.class); private static final Class byteBufferReadableClass = getReadableClass(); private static final Constructor h2SeekableConstructor = getH2SeekableConstructor(); From 02d37097eb94afb85dcf1d63bd3af61f94935452 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 6 Jul 2016 15:14:00 -0700 Subject: [PATCH 09/11] PARQUET-400: Remove unused property. --- .../java/org/apache/parquet/hadoop/ParquetFileReader.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 26922d7f2d..59a7e46cf5 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -106,13 +106,6 @@ public class ParquetFileReader implements Closeable { public static String PARQUET_READ_PARALLELISM = "parquet.metadata.read.parallelism"; - // configure if we want to use Hadoop's V2 read(bytebuffer) API. - // If true, we try to read using the new Hadoop read(ByteBuffer) api. This reads data into the provided - // byteBuffer and allows us to potentially take advantage zero-copy read path in Hadoop. - // Else, we skip and either read into the byteBuffer's array (if its present) or allocate one and copy - // into it. - public static String PARQUET_HADOOP_BYTEBUFFER_READ = "parquet.read.use.byte.buffer"; - private static ParquetMetadataConverter converter = new ParquetMetadataConverter(); /** From 823ca00889c1384d9ee9ba7d0f18dee8ebb2bd61 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Tue, 26 Jul 2016 14:43:01 -0700 Subject: [PATCH 10/11] PARQUET-400: Add tests for Hadoop 2 readFully. This adds a wrapper so that the static method used for readFully can be passed a stand-in for FSDataInputStream. --- .../hadoop/util/H2SeekableInputStream.java | 22 +- .../parquet/hadoop/util/HadoopStreams.java | 2 +- .../hadoop/util/TestByteBufferReads.java | 371 ++++++++++++++++++ 3 files changed, 388 insertions(+), 7 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java index 474efdb433..a04c71cd2a 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java @@ -19,9 +19,7 @@ package org.apache.parquet.hadoop.util; -import org.apache.hadoop.fs.ByteBufferReadable; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.parquet.Preconditions; import org.apache.parquet.io.SeekableInputStream; import java.io.EOFException; import java.io.IOException; @@ -33,12 +31,17 @@ */ class H2SeekableInputStream extends SeekableInputStream { + // Visible for testing + interface Reader { + int read(ByteBuffer buf) throws IOException; + } + private final FSDataInputStream stream; + private final Reader reader; public H2SeekableInputStream(FSDataInputStream stream) { - Preconditions.checkArgument(stream instanceof ByteBufferReadable, - "Input stream must be ByteBufferReadable. Try using H1SeekableInputStream"); this.stream = stream; + this.reader = new H2Reader(); } @Override @@ -78,10 +81,17 @@ public int read(ByteBuffer buf) throws IOException { @Override public void readFully(ByteBuffer buf) throws IOException { - readFully(stream, buf); + readFully(reader, buf); + } + + private class H2Reader implements Reader { + @Override + public int read(ByteBuffer buf) throws IOException { + return stream.read(buf); + } } - public static void readFully(FSDataInputStream stream, ByteBuffer buf) throws IOException { + public static void readFully(Reader stream, ByteBuffer buf) throws IOException { // unfortunately the Hadoop APIs seem to not have a 'readFully' equivalent for the byteBuffer read // calls. The read(ByteBuffer) call might read fewer than byteBuffer.hasRemaining() bytes. Thus we // have to loop to ensure we read them. diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java index 6edb041f7c..7c321cd461 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopStreams.java @@ -34,7 +34,7 @@ public class HadoopStreams { private static final Log LOG = Log.getLog(HadoopStreams.class); private static final Class byteBufferReadableClass = getReadableClass(); - private static final Constructor h2SeekableConstructor = getH2SeekableConstructor(); + static final Constructor h2SeekableConstructor = getH2SeekableConstructor(); /** * Wraps a {@link FSDataInputStream} in a {@link SeekableInputStream} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java index 47038c0e4a..08f0eed29e 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java @@ -24,6 +24,7 @@ import org.junit.Assert; import org.junit.Test; import java.io.EOFException; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.Callable; @@ -38,6 +39,29 @@ protected byte[] initialValue() { } }; + /** + * This mimics ByteBuffer reads from streams in Hadoop 2 + */ + private static class MockBufferReader implements H2SeekableInputStream.Reader { + private final FSDataInputStream stream; + + public MockBufferReader(FSDataInputStream stream) { + this.stream = stream; + } + + @Override + public int read(ByteBuffer buf) throws IOException { + // this is inefficient, but simple for correctness tests of + // readFully(ByteBuffer) + byte[] temp = new byte[buf.remaining()]; + int bytesRead = stream.read(temp, 0, temp.length); + if (bytesRead > 0) { + buf.put(temp, 0, bytesRead); + } + return bytesRead; + } + } + @Test public void testH1HeapRead() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); @@ -759,4 +783,351 @@ public void testH1DirectReadFullySmallTempBufferWithPositionAndLimit() throws Ex ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); } + @Test + public void testH2HeapReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testH2HeapReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(20); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + final MockBufferReader reader = new MockBufferReader(hadoopStream); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H2SeekableInputStream.readFully(reader, readBuffer); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testH2HeapReadFullyJustRight() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + // reads all of the bytes available without EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2HeapReadFullySmallReads() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2HeapReadFullyPosition() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testH2HeapReadFullyLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.limit(7); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2HeapReadFullyPositionAndLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testH2DirectReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testH2DirectReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + final MockBufferReader reader = new MockBufferReader(hadoopStream); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H2SeekableInputStream.readFully(reader, readBuffer); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testH2DirectReadFullyJustRight() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + // reads all of the bytes available without EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2DirectReadFullySmallReads() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2DirectReadFullyPosition() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testH2DirectReadFullyLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.limit(7); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + H2SeekableInputStream.Reader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testH2DirectReadFullyPositionAndLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } } From 1bcb8a8c23346687a209a9e88a1cc04c077cafd6 Mon Sep 17 00:00:00 2001 From: Ryan Blue Date: Wed, 27 Jul 2016 09:22:01 -0700 Subject: [PATCH 11/11] PARQUET-400: Fix review nits. --- .../hadoop/util/H2SeekableInputStream.java | 4 +- ...s.java => TestHadoop1ByteBufferReads.java} | 432 ++---------------- .../util/TestHadoop2ByteBufferReads.java | 405 ++++++++++++++++ 3 files changed, 437 insertions(+), 404 deletions(-) rename parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/{TestByteBufferReads.java => TestHadoop1ByteBufferReads.java} (63%) create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop2ByteBufferReads.java diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java index a04c71cd2a..a7065465f9 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/H2SeekableInputStream.java @@ -91,12 +91,12 @@ public int read(ByteBuffer buf) throws IOException { } } - public static void readFully(Reader stream, ByteBuffer buf) throws IOException { + public static void readFully(Reader reader, ByteBuffer buf) throws IOException { // unfortunately the Hadoop APIs seem to not have a 'readFully' equivalent for the byteBuffer read // calls. The read(ByteBuffer) call might read fewer than byteBuffer.hasRemaining() bytes. Thus we // have to loop to ensure we read them. while (buf.hasRemaining()) { - int readCount = stream.read(buf); + int readCount = reader.read(buf); if (readCount == -1) { // this is probably a bug in the ParquetReader. We shouldn't have called readFully with a buffer // that has more remaining than the amount of data in the stream. diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop1ByteBufferReads.java similarity index 63% rename from parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java rename to parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop1ByteBufferReads.java index 08f0eed29e..9e4e2a9cf7 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestByteBufferReads.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop1ByteBufferReads.java @@ -24,13 +24,12 @@ import org.junit.Assert; import org.junit.Test; import java.io.EOFException; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.Callable; import static org.apache.parquet.hadoop.util.MockInputStream.TEST_ARRAY; -public class TestByteBufferReads { +public class TestHadoop1ByteBufferReads { private static final ThreadLocal TEMP = new ThreadLocal() { @Override @@ -39,31 +38,8 @@ protected byte[] initialValue() { } }; - /** - * This mimics ByteBuffer reads from streams in Hadoop 2 - */ - private static class MockBufferReader implements H2SeekableInputStream.Reader { - private final FSDataInputStream stream; - - public MockBufferReader(FSDataInputStream stream) { - this.stream = stream; - } - - @Override - public int read(ByteBuffer buf) throws IOException { - // this is inefficient, but simple for correctness tests of - // readFully(ByteBuffer) - byte[] temp = new byte[buf.remaining()]; - int bytesRead = stream.read(temp, 0, temp.length); - if (bytesRead > 0) { - buf.put(temp, 0, bytesRead); - } - return bytesRead; - } - } - @Test - public void testH1HeapRead() throws Exception { + public void testHeapRead() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -82,7 +58,7 @@ public void testH1HeapRead() throws Exception { } @Test - public void testH1HeapSmallBuffer() throws Exception { + public void testHeapSmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(5); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -101,7 +77,7 @@ public void testH1HeapSmallBuffer() throws Exception { } @Test - public void testH1HeapSmallReads() throws Exception { + public void testHeapSmallReads() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(10); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -132,7 +108,7 @@ public void testH1HeapSmallReads() throws Exception { } @Test - public void testH1HeapPosition() throws Exception { + public void testHeapPosition() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.position(10); readBuffer.mark(); @@ -158,7 +134,7 @@ public void testH1HeapPosition() throws Exception { } @Test - public void testH1HeapLimit() throws Exception { + public void testHeapLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.limit(8); @@ -183,7 +159,7 @@ public void testH1HeapLimit() throws Exception { } @Test - public void testH1HeapPositionAndLimit() throws Exception { + public void testHeapPositionAndLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.position(5); readBuffer.limit(13); @@ -210,7 +186,7 @@ public void testH1HeapPositionAndLimit() throws Exception { } @Test - public void testH1DirectRead() throws Exception { + public void testDirectRead() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -229,7 +205,7 @@ public void testH1DirectRead() throws Exception { } @Test - public void testH1DirectSmallBuffer() throws Exception { + public void testDirectSmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(5); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -248,7 +224,7 @@ public void testH1DirectSmallBuffer() throws Exception { } @Test - public void testH1DirectSmallReads() throws Exception { + public void testDirectSmallReads() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -279,7 +255,7 @@ public void testH1DirectSmallReads() throws Exception { } @Test - public void testH1DirectPosition() throws Exception { + public void testDirectPosition() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); readBuffer.position(10); readBuffer.mark(); @@ -305,7 +281,7 @@ public void testH1DirectPosition() throws Exception { } @Test - public void testH1DirectLimit() throws Exception { + public void testDirectLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(20); readBuffer.limit(8); @@ -330,7 +306,7 @@ public void testH1DirectLimit() throws Exception { } @Test - public void testH1DirectPositionAndLimit() throws Exception { + public void testDirectPositionAndLimit() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); readBuffer.position(5); readBuffer.limit(13); @@ -357,7 +333,7 @@ public void testH1DirectPositionAndLimit() throws Exception { } @Test - public void testH1DirectSmallTempBufferSmallReads() throws Exception { + public void testDirectSmallTempBufferSmallReads() throws Exception { byte[] temp = new byte[2]; // this will cause readDirectBuffer to loop ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); @@ -393,7 +369,7 @@ public void testH1DirectSmallTempBufferSmallReads() throws Exception { } @Test - public void testH1DirectSmallTempBufferWithPositionAndLimit() throws Exception { + public void testDirectSmallTempBufferWithPositionAndLimit() throws Exception { byte[] temp = new byte[2]; // this will cause readDirectBuffer to loop ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); @@ -422,7 +398,7 @@ public void testH1DirectSmallTempBufferWithPositionAndLimit() throws Exception { } @Test - public void testH1HeapReadFullySmallBuffer() throws Exception { + public void testHeapReadFullySmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocate(8); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -441,7 +417,7 @@ public void testH1HeapReadFullySmallBuffer() throws Exception { } @Test - public void testH1HeapReadFullyLargeBuffer() throws Exception { + public void testHeapReadFullyLargeBuffer() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(20); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -460,7 +436,7 @@ public Object call() throws Exception { } @Test - public void testH1HeapReadFullyJustRight() throws Exception { + public void testHeapReadFullyJustRight() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -481,7 +457,7 @@ public void testH1HeapReadFullyJustRight() throws Exception { } @Test - public void testH1HeapReadFullySmallReads() throws Exception { + public void testHeapReadFullySmallReads() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -500,7 +476,7 @@ public void testH1HeapReadFullySmallReads() throws Exception { } @Test - public void testH1HeapReadFullyPosition() throws Exception { + public void testHeapReadFullyPosition() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); readBuffer.position(3); readBuffer.mark(); @@ -521,7 +497,7 @@ public void testH1HeapReadFullyPosition() throws Exception { } @Test - public void testH1HeapReadFullyLimit() throws Exception { + public void testHeapReadFullyLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); readBuffer.limit(7); @@ -551,7 +527,7 @@ public void testH1HeapReadFullyLimit() throws Exception { } @Test - public void testH1HeapReadFullyPositionAndLimit() throws Exception { + public void testHeapReadFullyPositionAndLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocate(10); readBuffer.position(3); readBuffer.limit(7); @@ -583,7 +559,7 @@ public void testH1HeapReadFullyPositionAndLimit() throws Exception { } @Test - public void testH1DirectReadFullySmallBuffer() throws Exception { + public void testDirectReadFullySmallBuffer() throws Exception { ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -602,7 +578,7 @@ public void testH1DirectReadFullySmallBuffer() throws Exception { } @Test - public void testH1DirectReadFullyLargeBuffer() throws Exception { + public void testDirectReadFullyLargeBuffer() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -627,7 +603,7 @@ public Object call() throws Exception { } @Test - public void testH1DirectReadFullyJustRight() throws Exception { + public void testDirectReadFullyJustRight() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); @@ -648,7 +624,7 @@ public void testH1DirectReadFullyJustRight() throws Exception { } @Test - public void testH1DirectReadFullySmallReads() throws Exception { + public void testDirectReadFullySmallReads() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); final FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); @@ -667,7 +643,7 @@ public void testH1DirectReadFullySmallReads() throws Exception { } @Test - public void testH1DirectReadFullyPosition() throws Exception { + public void testDirectReadFullyPosition() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); readBuffer.position(3); readBuffer.mark(); @@ -688,7 +664,7 @@ public void testH1DirectReadFullyPosition() throws Exception { } @Test - public void testH1DirectReadFullyLimit() throws Exception { + public void testDirectReadFullyLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); readBuffer.limit(7); @@ -718,7 +694,7 @@ public void testH1DirectReadFullyLimit() throws Exception { } @Test - public void testH1DirectReadFullyPositionAndLimit() throws Exception { + public void testDirectReadFullyPositionAndLimit() throws Exception { final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); readBuffer.position(3); readBuffer.limit(7); @@ -750,7 +726,7 @@ public void testH1DirectReadFullyPositionAndLimit() throws Exception { } @Test - public void testH1DirectReadFullySmallTempBufferWithPositionAndLimit() throws Exception { + public void testDirectReadFullySmallTempBufferWithPositionAndLimit() throws Exception { byte[] temp = new byte[2]; // this will cause readFully to loop final ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); @@ -782,352 +758,4 @@ public void testH1DirectReadFullySmallTempBufferWithPositionAndLimit() throws Ex Assert.assertEquals("Buffer contents should match", ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); } - - @Test - public void testH2HeapReadFullySmallBuffer() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(8); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); - } - - @Test - public void testH2HeapReadFullyLargeBuffer() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocate(20); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); - final MockBufferReader reader = new MockBufferReader(hadoopStream); - - TestUtils.assertThrows("Should throw EOFException", - EOFException.class, new Callable() { - @Override - public Object call() throws Exception { - H2SeekableInputStream.readFully(reader, readBuffer); - return null; - } - }); - - // NOTE: This behavior differs from readFullyHeapBuffer because direct uses - // several read operations that will read up to the end of the input. This - // is a correct value because the bytes in the buffer are valid. This - // behavior can't be implemented for the heap buffer without using the read - // method instead of the readFully method on the underlying - // FSDataInputStream. - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(20, readBuffer.limit()); - } - - @Test - public void testH2HeapReadFullyJustRight() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(10); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - // reads all of the bytes available without EOFException - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - // trying to read 0 more bytes doesn't result in EOFException - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2HeapReadFullySmallReads() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(10); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2HeapReadFullyPosition() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(10); - readBuffer.position(3); - readBuffer.mark(); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - - @Test - public void testH2HeapReadFullyLimit() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(10); - readBuffer.limit(7); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2HeapReadFullyPositionAndLimit() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocate(10); - readBuffer.position(3); - readBuffer.limit(7); - readBuffer.mark(); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - - @Test - public void testH2DirectReadFullySmallBuffer() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(8, readBuffer.position()); - Assert.assertEquals(8, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); - } - - @Test - public void testH2DirectReadFullyLargeBuffer() throws Exception { - final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); - final MockBufferReader reader = new MockBufferReader(hadoopStream); - - TestUtils.assertThrows("Should throw EOFException", - EOFException.class, new Callable() { - @Override - public Object call() throws Exception { - H2SeekableInputStream.readFully(reader, readBuffer); - return null; - } - }); - - // NOTE: This behavior differs from readFullyHeapBuffer because direct uses - // several read operations that will read up to the end of the input. This - // is a correct value because the bytes in the buffer are valid. This - // behavior can't be implemented for the heap buffer without using the read - // method instead of the readFully method on the underlying - // FSDataInputStream. - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(20, readBuffer.limit()); - } - - @Test - public void testH2DirectReadFullyJustRight() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - // reads all of the bytes available without EOFException - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - // trying to read 0 more bytes doesn't result in EOFException - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2DirectReadFullySmallReads() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2DirectReadFullyPosition() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - readBuffer.position(3); - readBuffer.mark(); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } - - @Test - public void testH2DirectReadFullyLimit() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - readBuffer.limit(7); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - H2SeekableInputStream.Reader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.flip(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY), readBuffer); - } - - @Test - public void testH2DirectReadFullyPositionAndLimit() throws Exception { - ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); - readBuffer.position(3); - readBuffer.limit(7); - readBuffer.mark(); - - FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); - MockBufferReader reader = new MockBufferReader(hadoopStream); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(7, readBuffer.position()); - Assert.assertEquals(7, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); - - readBuffer.position(7); - readBuffer.limit(10); - H2SeekableInputStream.readFully(reader, readBuffer); - Assert.assertEquals(10, readBuffer.position()); - Assert.assertEquals(10, readBuffer.limit()); - - readBuffer.reset(); - Assert.assertEquals("Buffer contents should match", - ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); - } } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop2ByteBufferReads.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop2ByteBufferReads.java new file mode 100644 index 0000000000..86b903c30e --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoop2ByteBufferReads.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop.util; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.parquet.hadoop.TestUtils; +import org.junit.Assert; +import org.junit.Test; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.Callable; + +import static org.apache.parquet.hadoop.util.MockInputStream.TEST_ARRAY; + +public class TestHadoop2ByteBufferReads { + + /** + * This mimics ByteBuffer reads from streams in Hadoop 2 + */ + private static class MockBufferReader implements H2SeekableInputStream.Reader { + private final FSDataInputStream stream; + + public MockBufferReader(FSDataInputStream stream) { + this.stream = stream; + } + + @Override + public int read(ByteBuffer buf) throws IOException { + // this is inefficient, but simple for correctness tests of + // readFully(ByteBuffer) + byte[] temp = new byte[buf.remaining()]; + int bytesRead = stream.read(temp, 0, temp.length); + if (bytesRead > 0) { + buf.put(temp, 0, bytesRead); + } + return bytesRead; + } + } + + @Test + public void testHeapReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testHeapReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocate(20); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + final MockBufferReader reader = new MockBufferReader(hadoopStream); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H2SeekableInputStream.readFully(reader, readBuffer); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testHeapReadFullyJustRight() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + // reads all of the bytes available without EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapReadFullySmallReads() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapReadFullyPosition() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testHeapReadFullyLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.limit(7); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testHeapReadFullyPositionAndLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocate(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testDirectReadFullySmallBuffer() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(8); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(8, readBuffer.position()); + Assert.assertEquals(8, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 8), readBuffer); + } + + @Test + public void testDirectReadFullyLargeBuffer() throws Exception { + final ByteBuffer readBuffer = ByteBuffer.allocateDirect(20); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + final MockBufferReader reader = new MockBufferReader(hadoopStream); + + TestUtils.assertThrows("Should throw EOFException", + EOFException.class, new Callable() { + @Override + public Object call() throws Exception { + H2SeekableInputStream.readFully(reader, readBuffer); + return null; + } + }); + + // NOTE: This behavior differs from readFullyHeapBuffer because direct uses + // several read operations that will read up to the end of the input. This + // is a correct value because the bytes in the buffer are valid. This + // behavior can't be implemented for the heap buffer without using the read + // method instead of the readFully method on the underlying + // FSDataInputStream. + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(20, readBuffer.limit()); + } + + @Test + public void testDirectReadFullyJustRight() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream()); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + // reads all of the bytes available without EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + // trying to read 0 more bytes doesn't result in EOFException + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectReadFullySmallReads() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectReadFullyPosition() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } + + @Test + public void testDirectReadFullyLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.limit(7); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + H2SeekableInputStream.Reader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.flip(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY), readBuffer); + } + + @Test + public void testDirectReadFullyPositionAndLimit() throws Exception { + ByteBuffer readBuffer = ByteBuffer.allocateDirect(10); + readBuffer.position(3); + readBuffer.limit(7); + readBuffer.mark(); + + FSDataInputStream hadoopStream = new FSDataInputStream(new MockInputStream(2, 3, 3)); + MockBufferReader reader = new MockBufferReader(hadoopStream); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(7, readBuffer.position()); + Assert.assertEquals(7, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 4), readBuffer); + + readBuffer.position(7); + readBuffer.limit(10); + H2SeekableInputStream.readFully(reader, readBuffer); + Assert.assertEquals(10, readBuffer.position()); + Assert.assertEquals(10, readBuffer.limit()); + + readBuffer.reset(); + Assert.assertEquals("Buffer contents should match", + ByteBuffer.wrap(TEST_ARRAY, 0, 7), readBuffer); + } +}