ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
pitrou wants to merge 2 commits into from
Closed

ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9672ca4
ARROW-6000: [Python] Add support for LargeString and LargeBinary types
pitrou Jul 30, 2019
0bb1b7f
Fix Take on LargeBinary / LargeString data
pitrou Jul 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions cpp/src/arrow/array/builder_binary.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
}

// Cannot make this a static attribute because of linking issues
static constexpr int64_t memory_limit() {
return std::numeric_limits<offset_type>::max() - 1;
}

protected:
TypedBufferBuilder<offset_type> offsets_builder_;
TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
const int64_t num_bytes = value_data_builder_.length();
offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
}

// Cannot make this a static attribute because of linking issues
static constexpr int64_t memory_limit() {
return std::numeric_limits<offset_type>::max() - 1;
}
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
public:
using TypeClass = FixedSizeBinaryType;

FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
/// This view becomes invalid on the next modifying operation.
util::string_view GetView(int64_t i) const;

static constexpr int64_t memory_limit() {
return std::numeric_limits<int64_t>::max() - 1;
}

protected:
int32_t byte_width_;
BufferBuilder byte_builder_;
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/array/builder_decimal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
public:
using TypeClass = Decimal128Type;

explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/array/builder_primitive.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
public:
using TypeClass = T;
using value_type = typename T::c_type;
using ArrayType = typename TypeTraits<T>::ArrayType;
using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
public:
using TypeClass = BooleanType;
using value_type = bool;

explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/array/builder_time.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
public:
using TypeClass = DayTimeIntervalType;
using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down
22 changes: 16 additions & 6 deletions cpp/src/arrow/compute/kernels/filter-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
}
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
protected:
std::shared_ptr<DataType> value_type() {
return TypeTraits<TypeClass>::type_singleton();
}

void AssertFilter(const std::string& values, const std::string& filter,
const std::string& expected) {
TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
}
void AssertFilterDictionary(const std::string& dictionary_values,
const std::string& dictionary_filter,
const std::string& filter,
const std::string& expected_filter) {
auto dict = ArrayFromJSON(utf8(), dictionary_values);
auto type = dictionary(int8(), utf8());
auto dict = ArrayFromJSON(value_type(), dictionary_values);
auto type = dictionary(int8(), value_type());
std::shared_ptr<Array> values, actual, expected;
ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
}
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
auto dict = R"(["a", "b", "c", "d", "e"])";
this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down
20 changes: 10 additions & 10 deletions cpp/src/arrow/compute/kernels/take-internal.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
!std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
builder->UnsafeAppend(std::forward<Scalar>(value));
return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
builder->UnsafeAppend(value);
return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
Builder* builder, util::string_view value) {
RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
builder->UnsafeAppend(value);
return Status::OK();
Expand Down
31 changes: 21 additions & 10 deletions cpp/src/arrow/compute/kernels/take-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
}
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
protected:
using StringTypes =
::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
public:
std::shared_ptr<DataType> value_type() {
return TypeTraits<TypeClass>::type_singleton();
}

void AssertTake(const std::string& values, const std::string& indices,
const std::string& expected) {
TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
}
void AssertTakeDictionary(const std::string& dictionary_values,
const std::string& dictionary_indices,
const std::string& indices,
const std::string& expected_indices) {
auto dict = ArrayFromJSON(utf8(), dictionary_values);
auto type = dictionary(int8(), utf8());
auto dict = ArrayFromJSON(value_type(), dictionary_values);
auto type = dictionary(int8(), value_type());
std::shared_ptr<Array> values, actual, expected;
ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
}
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

std::shared_ptr<DataType> type = this->value_type();
std::shared_ptr<Array> arr;
ASSERT_RAISES(IndexError,
this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
"[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
auto dict = R"(["a", "b", "c", "d", "e"])";
this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down
2 changes: 2 additions & 0 deletions cpp/src/arrow/python/helpers.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
GET_PRIMITIVE_TYPE(DOUBLE, float64);
GET_PRIMITIVE_TYPE(BINARY, binary);
GET_PRIMITIVE_TYPE(STRING, utf8);
GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
default:
return nullptr;
}
Expand Down
6 changes: 6 additions & 0 deletions cpp/src/arrow/python/helpers.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
// size is assumed to be positive
*out = static_cast<int64_t>(size);
return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down
66 changes: 41 additions & 25 deletions cpp/src/arrow/python/python_to_arrow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
AppendFunc&& append_func) {
int32_t length = -1;
RETURN_NOT_OK(internal::CastSize(view.size, &length));
DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
bool* is_full) {
if (view.size > BuilderType::memory_limit()) {
return Status::Invalid("string too large for datatype");
}
DCHECK_GE(view.size, 0);
// Did we reach the builder size limit?
if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
BuilderType::memory_limit())) {
*is_full = true;
return Status::OK();
}
RETURN_NOT_OK(append_func(view.bytes, length));
RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
*is_full = false;
return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
PyBytesView view;
RETURN_NOT_OK(view.FromString(obj));
return AppendPyString(builder, view, is_full,
[&builder](const char* bytes, int32_t length) {
return builder->Append(bytes, length);
});
return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
PyBytesView view;
RETURN_NOT_OK(view.FromString(obj));
return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
return internal::InvalidValue(obj, ss.str());
}

return AppendPyString(
builder, view, is_full,
[&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
: public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
public:
StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
}
}

return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
[this](const char* bytes, int32_t length) {
return this->typed_builder_->Append(bytes, length);
});
return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
}

Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
// We should have bailed out earlier
DCHECK(!STRICT);

using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

for (size_t i = 0; i < out->size(); ++i) {
auto binary_data = (*out)[i]->data()->Copy();
binary_data->type = ::arrow::binary();
(*out)[i] = std::make_shared<BinaryArray>(binary_data);
binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
(*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
}
}
return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
NUMERIC_CONVERTER(DOUBLE, DoubleType);
SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
case Type::STRING:
if (strict_conversions) {
*out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
*out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
} else {
*out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
}
break;
case Type::LARGE_STRING:
if (strict_conversions) {
*out =
std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
} else {
*out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
*out =
std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
}
break;
case Type::TIME32: {
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

pitrou
wants to merge
2
commits into

from

 Closed


ARROW-6000: [Python] Add support for LargeString and LargeBinary types #4927
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
9672ca4

ARROW-6000: [Python] Add support for LargeString and LargeBinary types


pitrou Jul 30, 2019
0bb1b7f

Fix Take on LargeBinary / LargeString data


pitrou Jul 31, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


16 changes: 11 additions & 5 deletions
cpp/src/arrow/array/builder_binary.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -294,6 +294,11 @@ class BaseBinaryBuilder : public ArrayBuilder {
 return util::string_view(reinterpret_cast<const char*>(value_data), value_length);
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }

 protected:
 TypedBufferBuilder<offset_type> offsets_builder_;
 TypedBufferBuilder<uint8_t> value_data_builder_;
Expand All@@ -315,11 +320,6 @@ class BaseBinaryBuilder : public ArrayBuilder {
 const int64_t num_bytes = value_data_builder_.length();
 offsets_builder_.UnsafeAppend(static_cast<offset_type>(num_bytes));
 }

 // Cannot make this a static attribute because of linking issues
 static constexpr int64_t memory_limit() {
 return std::numeric_limits<offset_type>::max() - 1;
 }
};

/// \class BinaryBuilder
Expand DownExpand Up@@ -387,6 +387,8 @@ class ARROW_EXPORT LargeStringBuilder : public LargeBinaryBuilder {

class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 public:
 using TypeClass = FixedSizeBinaryType;

 FixedSizeBinaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand DownExpand Up@@ -471,6 +473,10 @@ class ARROW_EXPORT FixedSizeBinaryBuilder : public ArrayBuilder {
 /// This view becomes invalid on the next modifying operation.
 util::string_view GetView(int64_t i) const;

 static constexpr int64_t memory_limit() {
 return std::numeric_limits<int64_t>::max() - 1;
 }

 protected:
 int32_t byte_width_;
 BufferBuilder byte_builder_;
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/array/builder_decimal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -28,6 +28,8 @@ class Decimal128;

class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
 public:
 using TypeClass = Decimal128Type;

 explicit Decimal128Builder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/array/builder_primitive.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -58,6 +58,7 @@ class ARROW_EXPORT NullBuilder : public ArrayBuilder {
template <typename T>
class NumericBuilder : public ArrayBuilder {
 public:
 using TypeClass = T;
 using value_type = typename T::c_type;
 using ArrayType = typename TypeTraits<T>::ArrayType;
 using ArrayBuilder::ArrayBuilder;
Expand DownExpand Up@@ -265,7 +266,9 @@ using DoubleBuilder = NumericBuilder<DoubleType>;

class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
 public:
 using TypeClass = BooleanType;
 using value_type = bool;

 explicit BooleanBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT);

 explicit BooleanBuilder(const std::shared_ptr<DataType>& type, MemoryPool* pool);
Expand Down

1 change: 1 addition & 0 deletions
cpp/src/arrow/array/builder_time.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -34,6 +34,7 @@ namespace arrow {

class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder {
 public:
 using TypeClass = DayTimeIntervalType;
 using DayMilliseconds = DayTimeIntervalType::DayMilliseconds;

 explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT)
Expand Down

22 changes: 16 additions & 6 deletions
cpp/src/arrow/compute/kernels/filter-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -285,18 +285,26 @@ TYPED_TEST(TestFilterKernelWithNumeric, ScalarInRangeAndFilterRandomNumeric) {
 }
}

class TestFilterKernelWithString : public TestFilterKernel<StringType> {
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestFilterKernelWithString : public TestFilterKernel<TypeClass> {
 protected:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertFilter(const std::string& values, const std::string& filter,
 const std::string& expected) {
 TestFilterKernel<StringType>::AssertFilter(utf8(), values, filter, expected);
 TestFilterKernel<TypeClass>::AssertFilter(value_type(), values, filter, expected);
 }
 void AssertFilterDictionary(const std::string& dictionary_values,
 const std::string& dictionary_filter,
 const std::string& filter,
 const std::string& expected_filter) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_filter),
 dict, &values));
Expand All@@ -307,13 +315,15 @@ class TestFilterKernelWithString : public TestFilterKernel<StringType> {
 }
};

TEST_F(TestFilterKernelWithString, FilterString) {
TYPED_TEST_CASE(TestFilterKernelWithString, StringTypes);

TYPED_TEST(TestFilterKernelWithString, FilterString) {
 this->AssertFilter(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"([null, "b", "c"])", "[0, 1, 0]", R"(["b"])");
 this->AssertFilter(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b"])");
}

TEST_F(TestFilterKernelWithString, FilterDictionary) {
TYPED_TEST(TestFilterKernelWithString, FilterDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertFilterDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[4]");
 this->AssertFilterDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[4]");
Expand Down

20 changes: 10 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-internal.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,11 +20,13 @@
#include <algorithm>
#include <limits>
#include <memory>
#include <type_traits>
#include <utility>
#include <vector>

#include "arrow/builder.h"
#include "arrow/compute/context.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"
Expand All@@ -37,21 +39,19 @@ namespace compute {
using internal::checked_cast;
using internal::checked_pointer_cast;

// For non-binary builders, use regular value append
template <typename Builder, typename Scalar>
static Status UnsafeAppend(Builder* builder, Scalar&& value) {
static typename std::enable_if<
 !std::is_base_of<BaseBinaryType, typename Builder::TypeClass>::value, Status>::type
UnsafeAppend(Builder* builder, Scalar&& value) {
 builder->UnsafeAppend(std::forward<Scalar>(value));
 return Status::OK();
}

// Use BinaryBuilder::UnsafeAppend, but reserve byte storage first
static Status UnsafeAppend(BinaryBuilder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
}

// Use StringBuilder::UnsafeAppend, but reserve character storage first
static Status UnsafeAppend(StringBuilder* builder, util::string_view value) {
// For binary builders, need to reserve byte storage first
template <typename Builder>
static enable_if_base_binary<typename Builder::TypeClass, Status> UnsafeAppend(
 Builder* builder, util::string_view value) {
 RETURN_NOT_OK(builder->ReserveData(static_cast<int64_t>(value.size())));
 builder->UnsafeAppend(value);
 return Status::OK();
Expand Down

31 changes: 21 additions & 10 deletions
cpp/src/arrow/compute/kernels/take-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,18 +179,26 @@ TYPED_TEST(TestTakeKernelWithNumeric, TakeRandomNumeric) {
 }
}

class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 protected:
using StringTypes =
 ::testing::Types<BinaryType, StringType, LargeBinaryType, LargeStringType>;

template <typename TypeClass>
class TestTakeKernelWithString : public TestTakeKernel<TypeClass> {
 public:
 std::shared_ptr<DataType> value_type() {
 return TypeTraits<TypeClass>::type_singleton();
 }

 void AssertTake(const std::string& values, const std::string& indices,
 const std::string& expected) {
 TestTakeKernel<StringType>::AssertTake(utf8(), values, indices, expected);
 TestTakeKernel<TypeClass>::AssertTake(value_type(), values, indices, expected);
 }
 void AssertTakeDictionary(const std::string& dictionary_values,
 const std::string& dictionary_indices,
 const std::string& indices,
 const std::string& expected_indices) {
 auto dict = ArrayFromJSON(utf8(), dictionary_values);
 auto type = dictionary(int8(), utf8());
 auto dict = ArrayFromJSON(value_type(), dictionary_values);
 auto type = dictionary(int8(), value_type());
 std::shared_ptr<Array> values, actual, expected;
 ASSERT_OK(DictionaryArray::FromArrays(type, ArrayFromJSON(int8(), dictionary_indices),
 dict, &values));
Expand All@@ -201,19 +209,22 @@ class TestTakeKernelWithString : public TestTakeKernel<StringType> {
 }
};

TEST_F(TestTakeKernelWithString, TakeString) {
TYPED_TEST_CASE(TestTakeKernelWithString, StringTypes);

TYPED_TEST(TestTakeKernelWithString, TakeString) {
 this->AssertTake(R"(["a", "b", "c"])", "[0, 1, 0]", R"(["a", "b", "a"])");
 this->AssertTake(R"([null, "b", "c"])", "[0, 1, 0]", "[null, \"b\", null]");
 this->AssertTake(R"(["a", "b", "c"])", "[null, 1, 0]", R"([null, "b", "a"])");

 std::shared_ptr<DataType> type = this->value_type();
 std::shared_ptr<Array> arr;
 ASSERT_RAISES(IndexError,
 this->Take(utf8(), R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(utf8(), R"(["a", "b", null, "ddd", "ee"])",
int64(), "[2, 5]", &arr));
 this->Take(type, R"(["a", "b", "c"])", int8(), "[0, 9, 0]", &arr));
 ASSERT_RAISES(IndexError, this->Take(type, R"(["a", "b", null, "ddd", "ee"])", int64(),
 "[2, 5]", &arr));
}

TEST_F(TestTakeKernelWithString, TakeDictionary) {
TYPED_TEST(TestTakeKernelWithString, TakeDictionary) {
 auto dict = R"(["a", "b", "c", "d", "e"])";
 this->AssertTakeDictionary(dict, "[3, 4, 2]", "[0, 1, 0]", "[3, 4, 3]");
 this->AssertTakeDictionary(dict, "[null, 4, 2]", "[0, 1, 0]", "[null, 4, null]");
Expand Down

2 changes: 2 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -62,6 +62,8 @@ std::shared_ptr<DataType> GetPrimitiveType(Type::type type) {
 GET_PRIMITIVE_TYPE(DOUBLE, float64);
 GET_PRIMITIVE_TYPE(BINARY, binary);
 GET_PRIMITIVE_TYPE(STRING, utf8);
 GET_PRIMITIVE_TYPE(LARGE_BINARY, large_binary);
 GET_PRIMITIVE_TYPE(LARGE_STRING, large_utf8);
 default:
 return nullptr;
 }
Expand Down

6 changes: 6 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -119,6 +119,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 return Status::OK();
}

inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NULLPTR) {
 // size is assumed to be positive
 *out = static_cast<int64_t>(size);
 return Status::OK();
}

// \brief Print the Python object's __str__ form along with the passed error
// message
ARROW_PYTHON_EXPORT
Expand Down

66 changes: 41 additions & 25 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -429,29 +429,34 @@ class TimestampConverter : public TypedConverter<TimestampType, TimestampConvert

namespace detail {

template <typename BuilderType, typename AppendFunc>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view, bool* is_full,
 AppendFunc&& append_func) {
 int32_t length = -1;
 RETURN_NOT_OK(internal::CastSize(view.size, &length));
 DCHECK_GE(length, 0);
template <typename BuilderType>
inline Status AppendPyString(BuilderType* builder, const PyBytesView& view,
 bool* is_full) {
 if (view.size > BuilderType::memory_limit()) {
 return Status::Invalid("string too large for datatype");
 }
 DCHECK_GE(view.size, 0);
 // Did we reach the builder size limit?
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + length > kBinaryMemoryLimit)) {
 if (ARROW_PREDICT_FALSE(builder->value_data_length() + view.size >
 BuilderType::memory_limit())) {
 *is_full = true;
 return Status::OK();
 }
 RETURN_NOT_OK(append_func(view.bytes, length));
 RETURN_NOT_OK(builder->Append(::arrow::util::string_view(view.bytes, view.size)));
 *is_full = false;
 return Status::OK();
}

inline Status BuilderAppend(BinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full,
 [&builder](const char* bytes, int32_t length) {
 return builder->Append(bytes, length);
 });
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(LargeBinaryBuilder* builder, PyObject* obj, bool* is_full) {
 PyBytesView view;
 RETURN_NOT_OK(view.FromString(obj));
 return AppendPyString(builder, view, is_full);
}

inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
Expand All@@ -466,9 +471,7 @@ inline Status BuilderAppend(FixedSizeBinaryBuilder* builder, PyObject* obj,
 return internal::InvalidValue(obj, ss.str());
 }

 return AppendPyString(
 builder, view, is_full,
 [&builder](const char* bytes, int32_t length) { return builder->Append(bytes); });
 return AppendPyString(builder, view, is_full);
}

} // namespace detail
Expand DownExpand Up@@ -496,12 +499,15 @@ class BinaryLikeConverter : public TypedConverter<Type, BinaryLikeConverter<Type

class BytesConverter : public BinaryLikeConverter<BinaryType> {};

class LargeBytesConverter : public BinaryLikeConverter<LargeBinaryType> {};

class FixedWidthBytesConverter : public BinaryLikeConverter<FixedSizeBinaryType> {};

// For String/UTF8, if strict_conversions enabled, we reject any non-UTF8,
// otherwise we allow but return results as BinaryArray
template <bool STRICT>
class StringConverter : public TypedConverter<StringType, StringConverter<STRICT>> {
template <typename TypeClass, bool STRICT>
class StringConverter
 : public TypedConverter<TypeClass, StringConverter<TypeClass, STRICT>> {
 public:
 StringConverter() : binary_count_(0) {}

Expand All@@ -526,10 +532,7 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 }
 }

 return detail::AppendPyString(this->typed_builder_, string_view_, is_full,
 [this](const char* bytes, int32_t length) {
 return this->typed_builder_->Append(bytes, length);
 });
 return detail::AppendPyString(this->typed_builder_, string_view_, is_full);
 }

 Status AppendItem(PyObject* obj) {
Expand All@@ -556,10 +559,13 @@ class StringConverter : public TypedConverter<StringType, StringConverter<STRICT
 // We should have bailed out earlier
 DCHECK(!STRICT);

 using EquivalentBinaryType = typename TypeClass::EquivalentBinaryType;
 using EquivalentBinaryArray = typename TypeTraits<EquivalentBinaryType>::ArrayType;

 for (size_t i = 0; i < out->size(); ++i) {
 auto binary_data = (*out)[i]->data()->Copy();
 binary_data->type = ::arrow::binary();
 (*out)[i] = std::make_shared<BinaryArray>(binary_data);
 binary_data->type = TypeTraits<EquivalentBinaryType>::type_singleton();
 (*out)[i] = std::make_shared<EquivalentBinaryArray>(binary_data);
 }
 }
 return Status::OK();
Expand DownExpand Up@@ -871,14 +877,24 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 NUMERIC_CONVERTER(DOUBLE, DoubleType);
 SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
 SIMPLE_CONVERTER_CASE(BINARY, BytesConverter);
 SIMPLE_CONVERTER_CASE(LARGE_BINARY, LargeBytesConverter);
 SIMPLE_CONVERTER_CASE(FIXED_SIZE_BINARY, FixedWidthBytesConverter);
 SIMPLE_CONVERTER_CASE(DATE32, Date32Converter);
 SIMPLE_CONVERTER_CASE(DATE64, Date64Converter);
 case Type::STRING:
 if (strict_conversions) {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<true>());
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<StringType, false>());
 }
 break;
 case Type::LARGE_STRING:
 if (strict_conversions) {
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, true>());
 } else {
 *out = std::unique_ptr<SeqConverter>(new StringConverter<false>());
 *out =
 std::unique_ptr<SeqConverter>(new StringConverter<LargeStringType, false>());
 }
 break;
 case Type::TIME32: {
Expand Down
Loading