ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 3 commits into from
Closed

ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices #4956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c2e40ed
Implement 32-bit-only variant of DictionaryBuilder, use common base c…
wesm Jul 26, 2019
8603ded
Change Parquet to use 32-bit dictionary builder
wesm Jul 27, 2019
5210b30
Make implementation templates internal
wesm Jul 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions cpp/src/arrow/array-dict-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
using c_type = typename TypeParam::c_type;
auto type = std::make_shared<TypeParam>();
auto dict_type = dictionary(int32(), type);

Dictionary32Builder<TypeParam> builder;

ASSERT_OK(builder.Append(static_cast<c_type>(1)));
ASSERT_OK(builder.Append(static_cast<c_type>(2)));
ASSERT_OK(builder.Append(static_cast<c_type>(1)));
ASSERT_OK(builder.Append(static_cast<c_type>(2)));
std::shared_ptr<Array> result;
FinishAndCheckPadding(&builder, &result);

// Build expected data for the initial dictionary
auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
// Build the dictionary Array
StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
auto index_type = TypeTraits<IndexType>::type_singleton();

auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
ASSERT_OK(builder.InsertMemoValues(*ex_dict));

// Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
// Type mismatch
ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
for (int i = 0; i < 2; ++i) {
ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
std::shared_ptr<Array> result;
ASSERT_OK(builder.Finish(&result));

auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
auto dtype = dictionary(int8(), utf8());
auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
auto dtype = dictionary(index_type, utf8());
DictionaryArray expected(dtype, ex_indices, ex_dict);
ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
// Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down
168 changes: 117 additions & 51 deletions cpp/src/arrow/array/builder_dict.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
public:
using Scalar = typename internal::DictionaryScalar<T>::type;
using Scalar = typename DictionaryScalar<T>::type;

// WARNING: the type given below is the value type, not the DictionaryType.
// The DictionaryType is instantiated on the Finish() call.
template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
const std::shared_ptr<DataType>&>::type type,
MemoryPool* pool = default_memory_pool())
: ArrayBuilder(type, pool),
memo_table_(new internal::DictionaryMemoTable(type)),
memo_table_(new DictionaryMemoTable(type)),
delta_offset_(0),
byte_width_(-1),
values_builder_(pool) {}

template <typename T1 = T>
explicit DictionaryBuilder(
explicit DictionaryBuilderBase(
typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
const std::shared_ptr<DataType>&>::type type,
MemoryPool* pool = default_memory_pool())
: ArrayBuilder(type, pool),
memo_table_(new internal::DictionaryMemoTable(type)),
memo_table_(new DictionaryMemoTable(type)),
delta_offset_(0),
byte_width_(static_cast<const T1&>(*type).byte_width()),
values_builder_(pool) {}

template <typename T1 = T>
explicit DictionaryBuilder(
explicit DictionaryBuilderBase(
typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
default_memory_pool())
: DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
: DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
: ArrayBuilder(dictionary->type(), pool),
memo_table_(new internal::DictionaryMemoTable(dictionary)),
memo_table_(new DictionaryMemoTable(dictionary)),
delta_offset_(0),
byte_width_(-1),
values_builder_(pool) {}

~DictionaryBuilder() override = default;
~DictionaryBuilderBase() override = default;

/// \brief Append a scalar value
Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
return memo_table_->InsertValues(values);
}

/// \brief Append dictionary indices directly without modifying memo
///
/// NOTE: Experimental API
Status AppendIndices(const int64_t* values, int64_t length,
const uint8_t* valid_bytes = NULLPTR) {
int64_t null_count_before = values_builder_.null_count();
ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
length_ += length;
null_count_ += values_builder_.null_count() - null_count_before;
return Status::OK();
}

/// \brief Append a whole dense array to the builder
template <typename T1 = T>
Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
void Reset() override {
ArrayBuilder::Reset();
values_builder_.Reset();
memo_table_.reset(new internal::DictionaryMemoTable(type_));
memo_table_.reset(new DictionaryMemoTable(type_));
delta_offset_ = 0;
}

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
bool is_building_delta() { return delta_offset_ > 0; }

protected:
std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
std::unique_ptr<DictionaryMemoTable> memo_table_;

int32_t delta_offset_;
// Only used for FixedSizeBinaryType
int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
: ArrayBuilder(type, pool), values_builder_(pool) {}
explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
: ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
: ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

/// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
public:
using DictionaryBuilder::Append;
using DictionaryBuilder::AppendIndices;
using DictionaryBuilder::DictionaryBuilder;
using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
using BASE::BASE;

/// \brief Append dictionary indices directly without modifying memo
///
/// NOTE: Experimental API
Status AppendIndices(const int64_t* values, int64_t length,
const uint8_t* valid_bytes = NULLPTR) {
int64_t null_count_before = this->values_builder_.null_count();
ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
this->length_ += length;
this->null_count_ += this->values_builder_.null_count() - null_count_before;
return Status::OK();
}
};

BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
public:
using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
using BASE::BASE;

/// \brief Append dictionary indices directly without modifying memo
///
/// NOTE: Experimental API
Status AppendIndices(const int32_t* values, int64_t length,
const uint8_t* valid_bytes = NULLPTR) {
int64_t null_count_before = this->values_builder_.null_count();
ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
this->length_ += length;
this->null_count_ += this->values_builder_.null_count() - null_count_before;
return Status::OK();
}
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
public:
using BASE = DictionaryBuilder<T>;
using BASE::Append;
using BASE::AppendIndices;
using BASE::BASE;

BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

Status Append(const uint8_t* value, int32_t length) {
return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
}
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
public:
using DictionaryBuilder::Append;
using DictionaryBuilder::AppendIndices;
using DictionaryBuilder::DictionaryBuilder;
using BASE = Dictionary32Builder<T>;
using BASE::Append;
using BASE::AppendIndices;
using BASE::BASE;

StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
BinaryDictionary32BuilderImpl()
: BinaryDictionary32BuilderImpl(default_memory_pool()) {}

Status Append(const uint8_t* value, int32_t length) {
return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
}
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
using BASE = BinaryDictionaryBuilderImpl<StringType>;
using BASE::BASE;
};

class BinaryDictionary32Builder
: public internal::BinaryDictionary32BuilderImpl<BinaryType> {
using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
using BASE::BASE;
};

class StringDictionary32Builder
: public internal::BinaryDictionary32BuilderImpl<StringType> {
using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
using BASE::BASE;
};

} // namespace arrow
16 changes: 9 additions & 7 deletions cpp/src/parquet/arrow/arrow-reader-writer-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
public:
static constexpr int kNumRowGroups = 10;

void SetUp() override {
GenerateData(GetParam());

// Write 4 row groups; each row group will have a different dictionary
ASSERT_NO_FATAL_FAILURE(
WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
default_arrow_writer_properties(), &buffer_));

properties_ = default_arrow_reader_properties();
}

void GenerateData(double null_probability) {
constexpr int num_unique = 100;
constexpr int repeat = 100;
constexpr int num_unique = 1000;
constexpr int repeat = 50;
constexpr int64_t min_length = 2;
constexpr int64_t max_length = 100;
::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
::arrow::StringDictionaryBuilder builder(default_memory_pool());
::arrow::StringDictionary32Builder builder(default_memory_pool());
const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
ASSERT_OK(builder.AppendArray(string_array));
ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
properties_.set_read_dictionary(0, true);

std::vector<std::shared_ptr<Array>> chunks(4);
const int64_t chunk_size = expected_dense_->num_rows() / 4;
for (int i = 0; i < 4; ++i) {
std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
for (int i = 0; i < kNumRowGroups; ++i) {
AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
}
auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices#4956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
3
commits into

from

 Closed


ARROW-6042: [C++][Parquet] Add Dictionary32Builder that always returns 32-bit dictionary indices
#4956
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


3 commits


Select commit
Hold shift + click to select a range
c2e40ed

Implement 32-bit-only variant of DictionaryBuilder, use common base c…


wesm Jul 26, 2019
8603ded

Change Parquet to use 32-bit dictionary builder


wesm Jul 27, 2019
5210b30

Make implementation templates internal


wesm Jul 27, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


40 changes: 35 additions & 5 deletions
cpp/src/arrow/array-dict-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -282,6 +282,26 @@ TYPED_TEST(TestDictionaryBuilder, DoubleDeltaDictionary) {
 ASSERT_TRUE(expected_delta2.Equals(result_delta2));
}

TYPED_TEST(TestDictionaryBuilder, Dictionary32_BasicPrimitive) {
 using c_type = typename TypeParam::c_type;
 auto type = std::make_shared<TypeParam>();
 auto dict_type = dictionary(int32(), type);

 Dictionary32Builder<TypeParam> builder;

 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 ASSERT_OK(builder.Append(static_cast<c_type>(1)));
 ASSERT_OK(builder.Append(static_cast<c_type>(2)));
 std::shared_ptr<Array> result;
 FinishAndCheckPadding(&builder, &result);

 // Build expected data for the initial dictionary
 auto ex_dict1 = ArrayFromJSON(type, "[1, 2]");
 DictionaryArray expected(dict_type, ArrayFromJSON(int32(), "[0, 1, 0, 1]"), ex_dict1);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, Basic) {
 // Build the dictionary Array
 StringDictionaryBuilder builder;
Expand All@@ -301,11 +321,14 @@ TEST(TestStringDictionaryBuilder, Basic) {
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
template <typename BuilderType, typename IndexType, typename AppendCType>
void TestStringDictionaryAppendIndices() {
 auto index_type = TypeTraits<IndexType>::type_singleton();

 auto ex_dict = ArrayFromJSON(utf8(), R"(["c", "a", "b", "d"])");
 auto invalid_dict = ArrayFromJSON(binary(), R"(["e", "f"])");

StringDictionaryBuilder builder;
BuilderType builder;
 ASSERT_OK(builder.InsertMemoValues(*ex_dict));

 // Inserting again should have no effect
Expand All@@ -314,7 +337,7 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Type mismatch
 ASSERT_RAISES(Invalid, builder.InsertMemoValues(*invalid_dict));

 std::vector<int64_t> raw_indices = {0, 1, 2, -1, 3};
 std::vector<AppendCType> raw_indices = {0, 1, 2, -1, 3};
 std::vector<uint8_t> is_valid = {1, 1, 1, 0, 1};
 for (int i = 0; i < 2; ++i) {
 ASSERT_OK(builder.AppendIndices(
Expand All@@ -326,12 +349,19 @@ TEST(TestStringDictionaryBuilder, AppendIndices) {
 std::shared_ptr<Array> result;
 ASSERT_OK(builder.Finish(&result));

 auto ex_indices = ArrayFromJSON(int8(), R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(int8(), utf8());
 auto ex_indices = ArrayFromJSON(index_type, R"([0, 1, 2, null, 3, 0, 1, 2, null, 3])");
 auto dtype = dictionary(index_type, utf8());
 DictionaryArray expected(dtype, ex_indices, ex_dict);
 ASSERT_TRUE(expected.Equals(result));
}

TEST(TestStringDictionaryBuilder, AppendIndices) {
 // Currently AdaptiveIntBuilder only accepts int64_t in bulk appends
 TestStringDictionaryAppendIndices<StringDictionaryBuilder, Int8Type, int64_t>();

 TestStringDictionaryAppendIndices<StringDictionary32Builder, Int32Type, int32_t>();
}

TEST(TestStringDictionaryBuilder, ArrayInit) {
 auto dict_array = ArrayFromJSON(utf8(), R"(["test", "test2"])");
 auto int_array = ArrayFromJSON(int8(), "[0, 1, 0]");
Expand Down

168 changes: 117 additions & 51 deletions
cpp/src/arrow/array/builder_dict.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -20,8 +20,9 @@
#include <algorithm>
#include <memory>

#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_adaptive.h" // IWYU pragma: export
#include "arrow/array/builder_base.h" // IWYU pragma: export
#include "arrow/array/builder_primitive.h" // IWYU pragma: export

#include "arrow/array.h"

Expand DownExpand Up@@ -84,8 +85,6 @@ class ARROW_EXPORT DictionaryMemoTable {
 std::unique_ptr<DictionaryMemoTableImpl> impl_;
};

} // namespace internal

/// \brief Array builder for created encoded DictionaryArray from
/// dense array
///
Expand All@@ -95,50 +94,50 @@ class ARROW_EXPORT DictionaryMemoTable {
/// build a delta dictionary when new terms occur.
///
/// data
template <typename T>
class DictionaryBuilder : public ArrayBuilder {
template <typename BuilderType, typename T>
class DictionaryBuilderBase : public ArrayBuilder {
 public:
 using Scalar = typename internal::DictionaryScalar<T>::type;
 using Scalar = typename DictionaryScalar<T>::type;

 // WARNING: the type given below is the value type, not the DictionaryType.
 // The DictionaryType is instantiated on the Finish() call.
 template <typename T1 = T>
DictionaryBuilder(
DictionaryBuilderBase(
 typename std::enable_if<!std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T1>::value,
 const std::shared_ptr<DataType>&>::type type,
 MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool),
 memo_table_(new internal::DictionaryMemoTable(type)),
 memo_table_(new DictionaryMemoTable(type)),
 delta_offset_(0),
 byte_width_(static_cast<const T1&>(*type).byte_width()),
 values_builder_(pool) {}

 template <typename T1 = T>
 explicit DictionaryBuilder(
 explicit DictionaryBuilderBase(
 typename std::enable_if<TypeTraits<T1>::is_parameter_free, MemoryPool*>::type pool =
 default_memory_pool())
 : DictionaryBuilder<T1>(TypeTraits<T1>::type_singleton(), pool) {}
 : DictionaryBuilderBase<BuilderType, T1>(TypeTraits<T1>::type_singleton(), pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool),
 memo_table_(new internal::DictionaryMemoTable(dictionary)),
 memo_table_(new DictionaryMemoTable(dictionary)),
 delta_offset_(0),
 byte_width_(-1),
 values_builder_(pool) {}

 ~DictionaryBuilder() override = default;
 ~DictionaryBuilderBase() override = default;

 /// \brief Append a scalar value
 Status Append(const Scalar& value) {
Expand DownExpand Up@@ -189,18 +188,6 @@ class DictionaryBuilder : public ArrayBuilder {
 return memo_table_->InsertValues(values);
 }

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = values_builder_.null_count();
 ARROW_RETURN_NOT_OK(values_builder_.AppendValues(values, length, valid_bytes));
 length_ += length;
 null_count_ += values_builder_.null_count() - null_count_before;
 return Status::OK();
 }

 /// \brief Append a whole dense array to the builder
 template <typename T1 = T>
 Status AppendArray(
Expand DownExpand Up@@ -242,7 +229,7 @@ class DictionaryBuilder : public ArrayBuilder {
 void Reset() override {
 ArrayBuilder::Reset();
 values_builder_.Reset();
 memo_table_.reset(new internal::DictionaryMemoTable(type_));
 memo_table_.reset(new DictionaryMemoTable(type_));
 delta_offset_ = 0;
 }

Expand DownExpand Up@@ -291,26 +278,27 @@ class DictionaryBuilder : public ArrayBuilder {
 bool is_building_delta() { return delta_offset_ > 0; }

 protected:
 std::unique_ptr<internal::DictionaryMemoTable> memo_table_;
 std::unique_ptr<DictionaryMemoTable> memo_table_;

 int32_t delta_offset_;
 // Only used for FixedSizeBinaryType
 int32_t byte_width_;

AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

template <>
class DictionaryBuilder<NullType> : public ArrayBuilder {
template <typename BuilderType>
class DictionaryBuilderBase<BuilderType, NullType> : public ArrayBuilder {
 public:
DictionaryBuilder(const std::shared_ptr<DataType>& type,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<DataType>& type,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(type, pool), values_builder_(pool) {}
 explicit DictionaryBuilder(MemoryPool* pool = default_memory_pool())

 explicit DictionaryBuilderBase(MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(null(), pool), values_builder_(pool) {}

DictionaryBuilder(const std::shared_ptr<Array>& dictionary,
 MemoryPool* pool = default_memory_pool())
DictionaryBuilderBase(const std::shared_ptr<Array>& dictionary,
MemoryPool* pool = default_memory_pool())
 : ArrayBuilder(dictionary->type(), pool), values_builder_(pool) {}

 /// \brief Append a scalar null value
Expand DownExpand Up@@ -362,16 +350,68 @@ class DictionaryBuilder<NullType> : public ArrayBuilder {
 Status Finish(std::shared_ptr<DictionaryArray>* out) { return FinishTyped(out); }

 protected:
AdaptiveIntBuilder values_builder_;
BuilderType values_builder_;
};

class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType> {
} // namespace internal

/// \brief A DictionaryArray builder that uses AdaptiveIntBuilder to return the
/// smallest index size that can accommodate the dictionary indices
template <typename T>
class DictionaryBuilder : public internal::DictionaryBuilderBase<AdaptiveIntBuilder, T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = internal::DictionaryBuilderBase<AdaptiveIntBuilder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int64_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

 BinaryDictionaryBuilder() : BinaryDictionaryBuilder(default_memory_pool()) {}
/// \brief A DictionaryArray builder that always returns int32 dictionary
/// indices so that data cast to dictionary form will have a consistent index
/// type, e.g. for creating a ChunkedArray
template <typename T>
class Dictionary32Builder : public internal::DictionaryBuilderBase<Int32Builder, T> {
 public:
 using BASE = internal::DictionaryBuilderBase<Int32Builder, T>;
 using BASE::BASE;

 /// \brief Append dictionary indices directly without modifying memo
 ///
 /// NOTE: Experimental API
 Status AppendIndices(const int32_t* values, int64_t length,
 const uint8_t* valid_bytes = NULLPTR) {
 int64_t null_count_before = this->values_builder_.null_count();
 ARROW_RETURN_NOT_OK(this->values_builder_.AppendValues(values, length, valid_bytes));
 this->length_ += length;
 this->null_count_ += this->values_builder_.null_count() - null_count_before;
 return Status::OK();
 }
};

// ----------------------------------------------------------------------
// Binary / Unicode builders with slightly expanded APIs

namespace internal {

template <typename T>
class BinaryDictionaryBuilderImpl : public DictionaryBuilder<T> {
 public:
 using BASE = DictionaryBuilder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 BinaryDictionaryBuilderImpl() : BinaryDictionaryBuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -382,14 +422,16 @@ class ARROW_EXPORT BinaryDictionaryBuilder : public DictionaryBuilder<BinaryType
 }
};

/// \brief Dictionary array builder with convenience methods for strings
class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType> {
template <typename T>
class BinaryDictionary32BuilderImpl : public Dictionary32Builder<T> {
 public:
 using DictionaryBuilder::Append;
 using DictionaryBuilder::AppendIndices;
 using DictionaryBuilder::DictionaryBuilder;
 using BASE = Dictionary32Builder<T>;
 using BASE::Append;
 using BASE::AppendIndices;
 using BASE::BASE;

 StringDictionaryBuilder() : StringDictionaryBuilder(default_memory_pool()) {}
 BinaryDictionary32BuilderImpl()
 : BinaryDictionary32BuilderImpl(default_memory_pool()) {}

 Status Append(const uint8_t* value, int32_t length) {
 return Append(reinterpret_cast<const char*>(value), length);
Expand All@@ -400,4 +442,28 @@ class ARROW_EXPORT StringDictionaryBuilder : public DictionaryBuilder<StringType
 }
};

} // namespace internal

class BinaryDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionaryBuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionaryBuilder : public internal::BinaryDictionaryBuilderImpl<StringType> {
 using BASE = BinaryDictionaryBuilderImpl<StringType>;
 using BASE::BASE;
};

class BinaryDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<BinaryType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<BinaryType>;
 using BASE::BASE;
};

class StringDictionary32Builder
 : public internal::BinaryDictionary32BuilderImpl<StringType> {
 using BASE = internal::BinaryDictionary32BuilderImpl<StringType>;
 using BASE::BASE;
};

} // namespace arrow

16 changes: 9 additions & 7 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -2736,20 +2736,22 @@ TEST(TestArrowWriterAdHoc, SchemaMismatch) {

class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
 public:
 static constexpr int kNumRowGroups = 10;

 void SetUp() override {
 GenerateData(GetParam());

 // Write 4 row groups; each row group will have a different dictionary
 ASSERT_NO_FATAL_FAILURE(
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / 4,
 WriteTableToBuffer(expected_dense_, expected_dense_->num_rows() / kNumRowGroups,
 default_arrow_writer_properties(), &buffer_));

 properties_ = default_arrow_reader_properties();
 }

 void GenerateData(double null_probability) {
 constexpr int num_unique = 100;
 constexpr int repeat = 100;
 constexpr int num_unique = 1000;
 constexpr int repeat = 50;
 constexpr int64_t min_length = 2;
 constexpr int64_t max_length = 100;
 ::arrow::random::RandomArrayGenerator rag(0);
Expand DownExpand Up@@ -2781,7 +2783,7 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {
};

void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
 ::arrow::StringDictionaryBuilder builder(default_memory_pool());
 ::arrow::StringDictionary32Builder builder(default_memory_pool());
 const auto& string_array = static_cast<const ::arrow::StringArray&>(arr);
 ASSERT_OK(builder.AppendArray(string_array));
 ASSERT_OK(builder.Finish(out));
Expand All@@ -2790,9 +2792,9 @@ void AsDictionaryEncoded(const Array& arr, std::shared_ptr<Array>* out) {
TEST_P(TestArrowReadDictionary, ReadWholeFileDict) {
 properties_.set_read_dictionary(0, true);

 std::vector<std::shared_ptr<Array>> chunks(4);
 const int64_t chunk_size = expected_dense_->num_rows() / 4;
 for (int i = 0; i < 4; ++i) {
 std::vector<std::shared_ptr<Array>> chunks(kNumRowGroups);
 const int64_t chunk_size = expected_dense_->num_rows() / kNumRowGroups;
 for (int i = 0; i < kNumRowGroups; ++i) {
 AsDictionaryEncoded(*dense_values_->Slice(chunk_size * i, chunk_size), &chunks[i]);
 }
 auto ex_table = MakeSimpleTable(std::make_shared<ChunkedArray>(chunks),
Expand Down
Loading