ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all \u003cpre\u003e\u003ccode\u003e blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks"); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + '
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix"); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length \u003e 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms"); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links"); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + '
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default"); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers"); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + '
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode"); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })();
Skip to content

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal#4933

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tianchen92 wants to merge 13 commits into from
Closed

ARROW-6022: [Java] Support equals API in ValueVector to compare two vectors equal #4933

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6bc3f68
ARROW-6022: [Java] Support equals API in ValueVector to compare two v…
Jul 24, 2019
10dca2c
fix
Jul 24, 2019
e58c158
refactor Dictionary#equals
Jul 25, 2019
0026882
use MinorType and check isSet
Jul 29, 2019
3c9f066
fix
Jul 29, 2019
1d95c9c
fix Decimal equals
Jul 29, 2019
b942794
use ArrowType for equal
Jul 31, 2019
c7081c2
check list validity bit
Aug 5, 2019
0dfa943
compare struct child names and add UT
Aug 5, 2019
694d9f6
make equals to visitor mode
Aug 8, 2019
226a20f
refactor
Aug 9, 2019
a5d22fd
fix variable names
Aug 9, 2019
7e20f79
remove CompareUtility
Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,8 +18,10 @@
import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.ReferenceManager;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.UnionMode;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;

Expand All@@ -36,6 +38,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -665,16 +668,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}
UnionVector that = (UnionVector) to;
ValueVector leftVector = getVector(index);
ValueVector rightVector = that.getVector(toIndex);
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

if (leftVector.getClass() != rightVector.getClass()) {
return false;
}
return leftVector.equals(index, rightVector, toIndex);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
27 changes: 15 additions & 12 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,6 +26,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -71,6 +72,10 @@ public BaseFixedWidthVector(Field field, final BufferAllocator allocator, final
}


public int getTypeWidth() {
return typeWidth;
}

@Override
public String getName() {
return field.getName();
Expand DownExpand Up@@ -870,20 +875,18 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseFixedWidthVector that = (BaseFixedWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

int leftStart = typeWidth * index;
int leftEnd = typeWidth * (index + 1);

int rightStart = typeWidth * toIndex;
int rightEnd = typeWidth * (toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why we have equals that checks a range of values and accpet a RangeEqualsVisitor? That seems redundant

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Equals(int index, ValueVector vector, int toIndex) compares single value and RangeEqualsVisitor compares a range of values.

@BryanCutlerBryanCutlerAug 9, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I must have missed the discussion when equals was added to compare a single value. It doesn't seem like that would be very useful to me, was there a specific use case to support this?

@tianchen92tianchen92Aug 10, 2019•
edited
Loading

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we did a refactor for dictionary encoding few weeks ago, hashCode/equals API are used for DictionaryHashTable (https://github.com/apache/arrow/blob/master/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryHashTable.java#L138) to avoid memory copy introduced by previous implementation.
PR is here #4846.

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree with @BryanCutler - this causes duplication of code/tests. Is there a reason to not use the RangeEquals API (with range = 1) instead ?

I'm fine if you want to do this in a different PR.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pravindra Ah, I have already made it reuse RangeEquals API(range ==1).

@pravindrapravindraAug 12, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I meant can we remove the equals() API from ValueVector, and it's callers be modified to instead use the RangeEquals() API ? I think keeping the minimal interface in ValueVector is ideal.

@tianchen92tianchen92Aug 12, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, right, could be done in a different PR.

return visitor.visit(this);
}
}
24 changes: 11 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,7 @@
import org.apache.arrow.memory.util.ArrowBufPointer;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
Expand DownExpand Up@@ -1369,20 +1370,17 @@ public boolean equals(int index, ValueVector to, int toIndex) {
if (to == null) {
return false;
}
if (this.getClass() != to.getClass()) {
return false;
}

BaseVariableWidthVector that = (BaseVariableWidthVector) to;
Preconditions.checkArgument(index >= 0 && index < valueCount,
"index %s out of range[0, %s]:", index, valueCount - 1);
Preconditions.checkArgument(toIndex >= 0 && toIndex < to.getValueCount(),
"index %s out of range[0, %s]:", index, to.getValueCount() - 1);

final int leftStart = getStartOffset(index);
final int leftEnd = getStartOffset(index + 1);

final int rightStart = that.getStartOffset(toIndex);
final int rightEnd = that.getStartOffset(toIndex + 1);
RangeEqualsVisitor visitor = new RangeEqualsVisitor(to, index, toIndex, 1);
return this.accept(visitor);
}

int ret = ByteFunctionHelpers.equal(this.getDataBuffer(), leftStart, leftEnd,
that.getDataBuffer(), rightStart, rightEnd);
return ret == 1;
@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(this);
}
}
1 change: 0 additions & 1 deletion java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -489,7 +489,6 @@ public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
set(index, isSet, start, buffer);
}


/*----------------------------------------------------------------*
| |
| vector transfer |
Expand Down
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
Expand DownExpand Up@@ -256,4 +257,9 @@ public Iterator<ValueVector> iterator() {
public BufferAllocator getAllocator() {
return underlyingVector.getAllocator();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {
return visitor.visit(getUnderlyingVector());
}
}
8 changes: 8 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
Expand DownExpand Up@@ -272,4 +273,11 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
* @param from source vector
*/
void copyFromSafe(int fromIndex, int thisIndex, ValueVector from);

/**
* Compare range values in this vector and vector in visitor.
* @param visitor visitor which holds the vector to compare.
* @return true if equals, otherwise false.
*/
boolean accept(RangeEqualsVisitor visitor);
Comment thread
pravindra marked this conversation as resolved.
}
6 changes: 6 additions & 0 deletions java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,6 +25,7 @@

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.impl.NullReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
Expand DownExpand Up@@ -264,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) {
public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) {
throw new UnsupportedOperationException();
}

@Override
public boolean accept(RangeEqualsVisitor visitor) {

@BryanCutlerBryanCutlerAug 8, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a unit test to check if ZeroVector == ZeroVector?

@tianchen92tianchen92Aug 9, 2019

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, done.

return true;
}
}
Loading