ARROW-815 [Java] Exposing reAlloc for ValueVector

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}

java/vector/src/main/codegen/templates/NullableValueVectors.java

@@ -240,6 +240,12 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
  <#if type.major == "VarLen">
  @Override
  public void allocateNew(int totalBytes, int valueCount) {

java/vector/src/main/codegen/templates/UnionVector.java

@@ -214,6 +214,12 @@ public boolean allocateNewSafe() {
  return safe;
  }
 
+ @Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
  @Override
  public void setInitialCapacity(int numRecords) {
  }

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
  */
  boolean allocateNewSafe();
 
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
  BufferAllocator getAllocator();
 
  /**

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -142,6 +142,9 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {}
+
  @Override
  public BufferAllocator getAllocator() {
  throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -81,6 +81,13 @@ public boolean allocateNewSafe() {
  return true;
  }
 
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
  /**
  * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
  * {@link org.apache.arrow.vector.ValueVector}.

java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java

@@ -20,6 +20,10 @@
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
  public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public boolean allocateNewSafe() {
  return success;
  }
 
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
 
  @Override
  public UInt4Vector getOffsetVector() {

java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java

@@ -124,6 +124,12 @@ public void allocateNew() throws OutOfMemoryException {
  bits.allocateNewSafe();
  }
 
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
  copyFrom(inIndex, outIndex, from);
  }

java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java

@@ -197,6 +197,13 @@ public boolean allocateNewSafe() {
  bits.zeroVector();
  return success;
  }
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
  public final class Accessor extends MapVector.Accessor {
  final BitVector.Accessor bAccessor = bits.getAccessor();
 

java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java

@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}