ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 4 commits into from
Closed

ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned #5229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5ee858c
[Java] Make change to ensure flatbuffer reads are aligned
Aug 29, 2019
9a690e4
fix comments and styles
Sep 4, 2019
cd4fd05
fix small bugs
Sep 5, 2019
1eb71d2
ARROW-6461: [Java] Prevent EchoServer from closing the client socket …
BryanCutler Sep 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
*/
public void run() throws IOException {
try {
Socket clientSocket = null;
ClientConnection client = null;
while (!closed) {
LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
clientSocket = serverSocket.accept();
LOGGER.debug("Accepted new client connection.");
try (ClientConnection client = new ClientConnection(clientSocket)) {
try {
client.run();
} catch (IOException e) {
LOGGER.warn("Error handling client connection.", e);
}
client = new ClientConnection(clientSocket);
try {
client.run();
} catch (IOException e) {
LOGGER.warn("Error handling client connection.", e);
}
LOGGER.debug("Closed connection with client");
}
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
*/
public void run() throws IOException {
// Read the entire input stream and write it back
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
VectorSchemaRoot root = reader.getVectorSchemaRoot();
// load the first batch before instantiating the writer so that we have any dictionaries
reader.loadNextBatch();
try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
.getOutputStream())) {
writer.start();
int echoed = 0;
while (true) {
int rowCount = reader.getVectorSchemaRoot().getRowCount();
if (rowCount == 0) {
break;
} else {
writer.writeBatch();
echoed += rowCount;
reader.loadNextBatch();
}
ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
writer.start();
int echoed = 0;
while (true) {
int rowCount = reader.getVectorSchemaRoot().getRowCount();
if (rowCount == 0) {
break;
} else {
writer.writeBatch();
echoed += rowCount;
reader.loadNextBatch();
}
writer.end();
Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
LOGGER.debug(String.format("Echoed %d records", echoed));
}
writer.end();
Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
LOGGER.debug(String.format("Echoed %d records", echoed));
reader.close(false);
}
}

Expand Down
2 changes: 1 addition & 1 deletion java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
}
Assert.assertFalse(reader.loadNextBatch());
assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
assertEquals(reader.bytesRead(), writer.bytesWritten());
assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
}
}

Expand Down
13 changes: 12 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
super(root, provider, out);
}

public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
IpcOption option) {
super(root, provider, out, option);
}

@Override
protected void startInternal(WriteChannel out) throws IOException {
ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

@Override
protected void endInternal(WriteChannel out) throws IOException {
out.writeIntLittleEndian(0);
if (option.write_legacy_ipc_format) {
out.writeIntLittleEndian(0);
} else {
out.writeLongLittleEndian(0);
}

long footerStart = out.getCurrentPosition();
out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down
22 changes: 18 additions & 4 deletions java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
* Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
this(root, provider, Channels.newChannel(out));
}

/**
* Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
*/
public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
this(root, provider, out, new IpcOption());
}

/**
* Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
*
* @param root Existing VectorSchemaRoot with vectors to be written.
* @param provider DictionaryProvider for any vectors that are dictionary encoded.
* (Optional, can be null)
* @param option IPC write options
* @param out WritableByteChannel for writing.
*/
public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
super(root, provider, out);
public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
IpcOption option) {
super(root, provider, out, option);
}

/**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
* @param out Open WriteChannel with an active Arrow stream.
* @throws IOException on error
*/
public static void writeEndOfStream(WriteChannel out) throws IOException {
out.writeIntLittleEndian(0);
public void writeEndOfStream(WriteChannel out) throws IOException {
if (option.write_legacy_ipc_format) {
out.writeIntLittleEndian(0);
} else {
out.writeLongLittleEndian(0);
}
}

@Override
Expand Down
17 changes: 13 additions & 4 deletions java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

private boolean dictWritten = false;

protected IpcOption option;

protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
this (root, provider, out, new IpcOption());
}

/**
* Note: fields are not closed when the writer is closed.
*
* @param root the vectors to write to the output
* @param provider where to find the dictionaries
* @param out the output where to write
* @param option IPC write options
*/
protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
this.unloader = new VectorUnloader(root);
this.out = new WriteChannel(out);
this.option = option;

List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
}

protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
ArrowBlock block = MessageSerializer.serialize(out, batch);
ArrowBlock block = MessageSerializer.serialize(out, batch, option);
LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
block.getOffset(), block.getMetadataLength(), block.getBodyLength());
return block;
}

protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
ArrowBlock block = MessageSerializer.serialize(out, batch);
ArrowBlock block = MessageSerializer.serialize(out, batch, option);
LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
block.getOffset(), block.getMetadataLength(), block.getBodyLength());
return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
startInternal(out);
// write the schema - for file formats this is duplicated in the footer, but matches
// the streaming format
MessageSerializer.serialize(out, schema);
MessageSerializer.serialize(out, schema, option);
}
}

Expand Down
9 changes: 9 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
return write(outBuffer);
}

/**
* Writes <code>v</code> in little-endian format to the underlying channel.
*/
public long writeLongLittleEndian(long v) throws IOException {
byte[] outBuffer = new byte[8];
MessageSerializer.longToBytes(v, outBuffer);
return write(outBuffer);
}

/**
* Writes the buffer to the underlying channel.
*/
Expand Down
28 changes: 28 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.arrow.vector.ipc.message;

/**
* IPC options, now only use for write.
*/
public class IpcOption {

// Write the pre-0.15.0 encapsulated IPC message format
// consisting of a 4-byte prefix instead of 8 byte
public boolean write_legacy_ipc_format = false;
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned#5229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

tianchen92
wants to merge
4
commits into

from

 Closed


ARROW-6315: [Java] Make change to ensure flatbuffer reads are aligned
#5229
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


4 commits


Select commit
Hold shift + click to select a range
5ee858c

[Java] Make change to ensure flatbuffer reads are aligned

Aug 29, 2019
9a690e4

fix comments and styles

Sep 4, 2019
cd4fd05

fix small bugs

Sep 5, 2019
1eb71d2

ARROW-6461: [Java] Prevent EchoServer from closing the client socket …


BryanCutler Sep 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 25 additions & 25 deletions
java/tools/src/main/java/org/apache/arrow/tools/EchoServer.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -69,16 +69,17 @@ public int port() {
 */
 public void run() throws IOException {
 try {
 Socket clientSocket = null;
 ClientConnection client = null;
 while (!closed) {
 LOGGER.debug("Waiting to accept new client connection.");
Socket clientSocket = serverSocket.accept();
 clientSocket = serverSocket.accept();
 LOGGER.debug("Accepted new client connection.");
 try (ClientConnection client = new ClientConnection(clientSocket)) {
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 client = new ClientConnection(clientSocket);
 try {
 client.run();
 } catch (IOException e) {
 LOGGER.warn("Error handling client connection.", e);
 }
 LOGGER.debug("Closed connection with client");
 }
Expand DownExpand Up@@ -112,29 +113,28 @@ public ClientConnection(Socket socket) {
 */
 public void run() throws IOException {
 // Read the entire input stream and write it back
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
 ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator);
 VectorSchemaRoot root = reader.getVectorSchemaRoot();
 // load the first batch before instantiating the writer so that we have any dictionaries
 reader.loadNextBatch();
 try (ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket
 .getOutputStream())) {
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 ArrowStreamWriter writer = new ArrowStreamWriter(root, reader, socket.getOutputStream());
 writer.start();
 int echoed = 0;
 while (true) {
 int rowCount = reader.getVectorSchemaRoot().getRowCount();
 if (rowCount == 0) {
 break;
 } else {
 writer.writeBatch();
 echoed += rowCount;
 reader.loadNextBatch();
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 }
 writer.end();
 Preconditions.checkState(reader.bytesRead() + 4 == writer.bytesWritten());
 LOGGER.debug(String.format("Echoed %d records", echoed));
 reader.close(false);
 }
 }

Expand Down

2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -127,7 +127,7 @@ private void testEchoServer(int serverPort,
 }
 Assert.assertFalse(reader.loadNextBatch());
 assertEquals(0, reader.getVectorSchemaRoot().getRowCount());
 assertEquals(reader.bytesRead(), writer.bytesWritten());
 assertEquals(reader.bytesRead() + 4, writer.bytesWritten());
 }
 }

Expand Down

13 changes: 12 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowFooter;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All@@ -47,6 +48,11 @@ public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, Writa
 super(root, provider, out);
 }

 public ArrowFileWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 @Override
 protected void startInternal(WriteChannel out) throws IOException {
 ArrowMagic.writeMagic(out, true);
Expand All@@ -68,7 +74,12 @@ protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException

 @Override
 protected void endInternal(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }

 long footerStart = out.getCurrentPosition();
 out.write(new ArrowFooter(schema, dictionaryBlocks, recordBlocks), false);
 int footerLength = (int) (out.getCurrentPosition() - footerStart);
Expand Down

22 changes: 18 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowStreamWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@

import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.IpcOption;

/**
 * Writer for the Arrow stream format to send ArrowRecordBatches over a WriteChannel.
Expand All@@ -42,16 +43,25 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Out
 this(root, provider, Channels.newChannel(out));
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this(root, provider, out, new IpcOption());
 }

 /**
 * Construct an ArrowStreamWriter with an optional DictionaryProvider for the WritableByteChannel.
 *
 * @param root Existing VectorSchemaRoot with vectors to be written.
 * @param provider DictionaryProvider for any vectors that are dictionary encoded.
 * (Optional, can be null)
 * @param option IPC write options
 * @param out WritableByteChannel for writing.
 */
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 super(root, provider, out);
 public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out,
 IpcOption option) {
 super(root, provider, out, option);
 }

 /**
Expand All@@ -60,8 +70,12 @@ public ArrowStreamWriter(VectorSchemaRoot root, DictionaryProvider provider, Wri
 * @param out Open WriteChannel with an active Arrow stream.
 * @throws IOException on error
 */
 public static void writeEndOfStream(WriteChannel out) throws IOException {
 out.writeIntLittleEndian(0);
 public void writeEndOfStream(WriteChannel out) throws IOException {
 if (option.write_legacy_ipc_format) {
 out.writeIntLittleEndian(0);
 } else {
 out.writeLongLittleEndian(0);
 }
 }

 @Override
Expand Down

17 changes: 13 additions & 4 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import org.apache.arrow.vector.ipc.message.ArrowBlock;
import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.ipc.message.IpcOption;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
Expand All@@ -59,16 +60,24 @@ public abstract class ArrowWriter implements AutoCloseable {

 private boolean dictWritten = false;

 protected IpcOption option;

 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 this (root, provider, out, new IpcOption());
 }

 /**
 * Note: fields are not closed when the writer is closed.
 *
 * @param root the vectors to write to the output
 * @param provider where to find the dictionaries
 * @param out the output where to write
 * @param option IPC write options
 */
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out) {
 protected ArrowWriter(VectorSchemaRoot root, DictionaryProvider provider, WritableByteChannel out, IpcOption option) {
 this.unloader = new VectorUnloader(root);
 this.out = new WriteChannel(out);
 this.option = option;

 List<Field> fields = new ArrayList<>(root.getSchema().getFields().size());
 Set<Long> dictionaryIdsUsed = new HashSet<>();
Expand DownExpand Up@@ -112,14 +121,14 @@ public void writeBatch() throws IOException {
 }

 protected ArrowBlock writeDictionaryBatch(ArrowDictionaryBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("DictionaryRecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
 }

 protected ArrowBlock writeRecordBatch(ArrowRecordBatch batch) throws IOException {
 ArrowBlock block = MessageSerializer.serialize(out, batch);
 ArrowBlock block = MessageSerializer.serialize(out, batch, option);
 LOGGER.debug("RecordBatch at {}, metadata: {}, body: {}",
 block.getOffset(), block.getMetadataLength(), block.getBodyLength());
 return block;
Expand All@@ -140,7 +149,7 @@ private void ensureStarted() throws IOException {
 startInternal(out);
 // write the schema - for file formats this is duplicated in the footer, but matches
 // the streaming format
 MessageSerializer.serialize(out, schema);
 MessageSerializer.serialize(out, schema, option);
 }
 }

Expand Down

9 changes: 9 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/WriteChannel.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -101,6 +101,15 @@ public long writeIntLittleEndian(int v) throws IOException {
 return write(outBuffer);
 }

 /**
 * Writes <code>v</code> in little-endian format to the underlying channel.
 */
 public long writeLongLittleEndian(long v) throws IOException {
 byte[] outBuffer = new byte[8];
 MessageSerializer.longToBytes(v, outBuffer);
 return write(outBuffer);
 }

 /**
 * Writes the buffer to the underlying channel.
 */
Expand Down

28 changes: 28 additions & 0 deletions
java/vector/src/main/java/org/apache/arrow/vector/ipc/message/IpcOption.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.arrow.vector.ipc.message;

/**
 * IPC options, now only use for write.
 */
public class IpcOption {

 // Write the pre-0.15.0 encapsulated IPC message format
 // consisting of a 4-byte prefix instead of 8 byte
 public boolean write_legacy_ipc_format = false;
}
Loading