ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
elahrvivaz wants to merge 1 commit into from
Closed

ARROW-482 [Java] Exposing custom field metadata #611

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
Integer.MAX_VALUE);
MapVector parent = new MapVector("parent", vectorAllocator, null)) {
MapVector parent = MapVector.empty("parent", vectorAllocator)) {
writeData(count, parent);
write(parent.getChild("root"), testInFile);
}
Expand Down
10 changes: 5 additions & 5 deletions java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

Field field = new Field(
"testField", true,
new ArrowType.Int(8, true),
"testField",
new FieldType(true, new ArrowType.Int(8, true), null, null),
Collections.<Field>emptyList());
NullableTinyIntVector vector =
new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
NullableIntVector writeVector =
new NullableIntVector(
"varchar",
new FieldType(true, MinorType.INT.getType(), writeEncoding),
new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
allocator);
NullableVarCharVector writeDictionaryVector =
new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
NullableVarCharVector writeDictionaryVector =
new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
ListVector writeVector = new ListVector("list", allocator, null, null)) {
ListVector writeVector = ListVector.empty("list", allocator)) {

// data being written:
// [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.getMutator().setValueCount(2);

writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
writeVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(writeVector);
listWriter.startList();
Expand Down
4 changes: 2 additions & 2 deletions java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
Integer.MAX_VALUE);
MapVector parent = new MapVector("parent", vectorAllocator, null)) {
MapVector parent = MapVector.empty("parent", vectorAllocator)) {
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
Integer.MAX_VALUE);
MapVector parent = new MapVector("parent", vectorAllocator, null)) {
MapVector parent = MapVector.empty("parent", vectorAllocator)) {
writeData(count, parent);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
Expand Down
3 changes: 1 addition & 2 deletions java/vector/src/main/codegen/templates/NullReader.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
return type;
}


@Override
public Field getField() {
return new Field("", true, new Null(), null);
return new Field("", FieldType.nullable(new Null()), null);
}

public void copyAsValue(MapWriter writer) {}
Expand Down
4 changes: 2 additions & 2 deletions java/vector/src/main/codegen/templates/NullableValueVectors.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
private final int scale;

public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
}
<#else>
public ${className}(String name, BufferAllocator allocator) {
this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
}
</#if>

Expand Down
6 changes: 3 additions & 3 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
this.name = name;
this.allocator = allocator;
this.internalMap = new MapVector("internal", allocator, callBack);
this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
this.typeVector = new UInt1Vector("types", allocator);
this.callBack = callBack;
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
}

private FieldType fieldType(MinorType type) {
return new FieldType(true, type.getType(), null);
return FieldType.nullable(type.getType());
}

private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
typeIds[childFields.size()] = v.getMinorType().ordinal();
childFields.add(v.getField());
}
return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

@Override
public Field getField() {
return new Field(name, true, new Null(), null);
return new Field(name, FieldType.nullable(new Null()), null);
}

@Override
Expand Down
44 changes: 27 additions & 17 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
return new FixedSizeListVector(name, allocator, fieldType, null);
}

private FieldVector vector;
private final BitVector bits;
private final int listSize;
private final DictionaryEncoding dictionary;
private final FieldType fieldType;
private final List<BufferBacked> innerVectors;

private UnionFixedSizeListReader reader;

private Mutator mutator = new Mutator();
private Accessor accessor = new Accessor();

// deprecated, use FieldType or static constructor instead
@Deprecated
public FixedSizeListVector(String name,
BufferAllocator allocator,
int listSize,
DictionaryEncoding dictionary,
CallBack schemaChangeCallback) {
this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
}

public FixedSizeListVector(String name,
BufferAllocator allocator,
FieldType fieldType,
CallBack schemaChangeCallback) {
super(name, allocator);
Preconditions.checkArgument(listSize > 0, "list size must be positive");
this.bits = new BitVector("$bits$", allocator);
this.vector = ZeroVector.INSTANCE;
this.listSize = listSize;
this.dictionary = dictionary;
this.fieldType = fieldType;
this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
Preconditions.checkArgument(listSize > 0, "list size must be positive");
this.innerVectors = Collections.singletonList((BufferBacked) bits);
this.reader = new UnionFixedSizeListReader(this);
}

@Override
public Field getField() {
List<Field> children = ImmutableList.of(getDataVector().getField());
return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
return new Field(name, fieldType, children);
}

@Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
throw new IllegalArgumentException("Lists have only one child. Found: " + children);
}
Field field = children.get(0);
FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
if (!addOrGetVector.isCreated()) {
throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
}
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
TransferPair pairs[] = new TransferPair[2];

public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
this(new FixedSizeListVector(name, allocator, fieldType, callBack));
}

public TransferImpl(FixedSizeListVector to) {
this.to = to;
Field field = vector.getField();
FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
to.addOrGetVector(type);
to.addOrGetVector(vector.getField().getFieldType());
pairs[0] = bits.makeTransferPair(to.bits);
pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
}
Expand Down
34 changes: 23 additions & 11 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

public static ListVector empty(String name, BufferAllocator allocator) {
return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
}

final UInt4Vector offsets;
final BitVector bits;
private final List<BufferBacked> innerVectors;
private Mutator mutator = new Mutator();
private Accessor accessor = new Accessor();
private UnionListReader reader;
private CallBack callBack;
private final DictionaryEncoding dictionary;
private final FieldType fieldType;

// deprecated, use FieldType or static constructor instead
@Deprecated
public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
this(name, allocator, null, callBack);
this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
}

// deprecated, use FieldType or static constructor instead
@Deprecated
public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
}

public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
super(name, allocator, callBack);
this.bits = new BitVector("$bits$", allocator);
this.offsets = getOffsetVector();
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
this.reader = new UnionListReader(this);
this.dictionary = dictionary;
this.fieldType = checkNotNull(fieldType);
this.callBack = callBack;
}

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
TransferPair pairs[] = new TransferPair[3];

public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
this(new ListVector(name, allocator, dictionary, callBack));
this(new ListVector(name, allocator, fieldType, callBack));
}

public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

@Override
public Field getField() {
return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
ImmutableList.of(getDataVector().getField()));
return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
}

@Override
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-482 [Java] Exposing custom field metadata#611

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

elahrvivaz
wants to merge
1
commit into

from

 Closed


ARROW-482 [Java] Exposing custom field metadata
#611
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


2 changes: 1 addition & 1 deletion
java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -99,7 +99,7 @@ static void writeInput(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 write(parent.getChild("root"), testInFile);
 }
Expand Down

10 changes: 5 additions & 5 deletions
java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -137,8 +137,8 @@ public void basicTest() throws InterruptedException, IOException {
 BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);

 Field field = new Field(
 "testField", true,
 new ArrowType.Int(8, true),
 "testField",
 new FieldType(true, new ArrowType.Int(8, true), null, null),
 Collections.<Field>emptyList());
 NullableTinyIntVector vector =
 new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
Expand All@@ -161,7 +161,7 @@ public void testFlatDictionary() throws IOException {
 NullableIntVector writeVector =
 new NullableIntVector(
 "varchar",
 new FieldType(true, MinorType.INT.getType(), writeEncoding),
 new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
 allocator);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector(
Expand DownExpand Up@@ -237,7 +237,7 @@ public void testNestedDictionary() throws IOException {
 try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
 NullableVarCharVector writeDictionaryVector =
 new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
 ListVector writeVector = new ListVector("list", allocator, null, null)) {
 ListVector writeVector = ListVector.empty("list", allocator)) {

 // data being written:
 // [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
Expand All@@ -247,7 +247,7 @@ public void testNestedDictionary() throws IOException {
 writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
 writeDictionaryVector.getMutator().setValueCount(2);

 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
 writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
 writeVector.allocateNew();
 UnionListWriter listWriter = new UnionListWriter(writeVector);
 listWriter.startList();
Expand Down

4 changes: 2 additions & 2 deletions
java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -76,7 +76,7 @@ static void writeInputFloat(File testInFile, BufferAllocator allocator, double..
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
 Float8Writer floatWriter = rootWriter.float8("float");
Expand All@@ -95,7 +95,7 @@ static void writeInput2(File testInFile, BufferAllocator allocator) throws
 try (
 BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
 Integer.MAX_VALUE);
 MapVector parent = new MapVector("parent", vectorAllocator, null)) {
 MapVector parent = MapVector.empty("parent", vectorAllocator)) {
 writeData(count, parent);
 ComplexWriter writer = new ComplexWriterImpl("root", parent);
 MapWriter rootWriter = writer.rootAsMap();
Expand Down

3 changes: 1 addition & 2 deletions
java/vector/src/main/codegen/templates/NullReader.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -55,10 +55,9 @@ public MinorType getMinorType() {
 return type;
 }


 @Override
 public Field getField() {
 return new Field("", true, new Null(), null);
 return new Field("", FieldType.nullable(new Null()), null);
 }

 public void copyAsValue(MapWriter writer) {}
Expand Down

4 changes: 2 additions & 2 deletions
java/vector/src/main/codegen/templates/NullableValueVectors.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 private final int scale;

 public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
 this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
 this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
 }
 <#else>
 public ${className}(String name, BufferAllocator allocator) {
 this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
 this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
 }
 </#if>

Expand Down

6 changes: 3 additions & 3 deletions
java/vector/src/main/codegen/templates/UnionVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
 public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
 this.name = name;
 this.allocator = allocator;
 this.internalMap = new MapVector("internal", allocator, callBack);
 this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
 this.typeVector = new UInt1Vector("types", allocator);
 this.callBack = callBack;
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
Expand DownExpand Up@@ -125,7 +125,7 @@ private String fieldName(MinorType type) {
 }

 private FieldType fieldType(MinorType type) {
 return new FieldType(true, type.getType(), null);
 return FieldType.nullable(type.getType());
 }

 private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
Expand DownExpand Up@@ -250,7 +250,7 @@ public Field getField() {
 typeIds[childFields.size()] = v.getMinorType().ordinal();
 childFields.add(v.getField());
 }
 return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
 return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
 }

 @Override
Expand Down

3 changes: 2 additions & 1 deletion
java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -29,6 +29,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;

Expand DownExpand Up@@ -98,7 +99,7 @@ public void clear() { }

 @Override
 public Field getField() {
 return new Field(name, true, new Null(), null);
 return new Field(name, FieldType.nullable(new Null()), null);
 }

 @Override
Expand Down

44 changes: 27 additions & 17 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,11 @@
import java.util.List;
import java.util.Objects;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -48,44 +53,52 @@
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {

 public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
 FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
 return new FixedSizeListVector(name, allocator, fieldType, null);
 }

 private FieldVector vector;
 private final BitVector bits;
 private final int listSize;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;
 private final List<BufferBacked> innerVectors;

 private UnionFixedSizeListReader reader;

 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 int listSize,
 DictionaryEncoding dictionary,
 CallBack schemaChangeCallback) {
 this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
 }

 public FixedSizeListVector(String name,
 BufferAllocator allocator,
 FieldType fieldType,
 CallBack schemaChangeCallback) {
 super(name, allocator);
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.bits = new BitVector("$bits$", allocator);
 this.vector = ZeroVector.INSTANCE;
 this.listSize = listSize;
 this.dictionary = dictionary;
 this.fieldType = fieldType;
 this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
 Preconditions.checkArgument(listSize > 0, "list size must be positive");
 this.innerVectors = Collections.singletonList((BufferBacked) bits);
 this.reader = new UnionFixedSizeListReader(this);
 }

 @Override
 public Field getField() {
 List<Field> children = ImmutableList.of(getDataVector().getField());
 return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
 return new Field(name, fieldType, children);
 }

 @Override
Expand All@@ -103,8 +116,7 @@ public void initializeChildrenFromFields(List<Field> children) {
 throw new IllegalArgumentException("Lists have only one child. Found: " + children);
 }
 Field field = children.get(0);
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
 AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
 if (!addOrGetVector.isCreated()) {
 throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
 }
Expand DownExpand Up@@ -348,14 +360,12 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[2];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
 this(new FixedSizeListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(FixedSizeListVector to) {
 this.to = to;
 Field field = vector.getField();
 FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
 to.addOrGetVector(type);
 to.addOrGetVector(vector.getField().getFieldType());
 pairs[0] = bits.makeTransferPair(to.bits);
 pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
 }
Expand Down

34 changes: 23 additions & 11 deletions
java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -18,12 +18,17 @@
 ******************************************************************************/
package org.apache.arrow.vector.complex;

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
Expand All@@ -41,40 +46,48 @@
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ObjectArrays;

import io.netty.buffer.ArrowBuf;

public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {

 public static ListVector empty(String name, BufferAllocator allocator) {
 return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
 }

 final UInt4Vector offsets;
 final BitVector bits;
 private final List<BufferBacked> innerVectors;
 private Mutator mutator = new Mutator();
 private Accessor accessor = new Accessor();
 private UnionListReader reader;
 private CallBack callBack;
 private final DictionaryEncoding dictionary;
 private final FieldType fieldType;

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
 this(name, allocator, null, callBack);
 this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
 }

 // deprecated, use FieldType or static constructor instead
 @Deprecated
 public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
 this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
 }

 public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
 super(name, allocator, callBack);
 this.bits = new BitVector("$bits$", allocator);
 this.offsets = getOffsetVector();
 this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
 this.reader = new UnionListReader(this);
 this.dictionary = dictionary;
 this.fieldType = checkNotNull(fieldType);
 this.callBack = callBack;
 }

Expand DownExpand Up@@ -168,7 +181,7 @@ private class TransferImpl implements TransferPair {
 TransferPair pairs[] = new TransferPair[3];

 public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
 this(new ListVector(name, allocator, dictionary, callBack));
 this(new ListVector(name, allocator, fieldType, callBack));
 }

 public TransferImpl(ListVector to) {
Expand DownExpand Up@@ -266,8 +279,7 @@ public int getBufferSize() {

 @Override
 public Field getField() {
 return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
 ImmutableList.of(getDataVector().getField()));
 return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
 }

 @Override
Expand Down
Loading