Uh oh!
There was an error while loading. Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork 4.3k
GH-34322: [C++][Parquet] Encoding Microbench for ByteArray#34323
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
wjones127
merged 14 commits into
apache:main
from
mapleFU:parquet/benchmark-delta-length-byteMar 7, 2023
Uh oh!
There was an error while loading. Please reload this page.
Merged
Changes from all commits
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
966e877
Parquet: Introduce basic benchmark
mapleFU 2818762
Merge branch 'main' into parquet/benchmark-delta-length-byte
mapleFU 5320eec
tiny benchmark spaced
mapleFU 86290dd
add stats
mapleFU 213d4d0
Merge branch 'main' into parquet/benchmark-delta-length-byte
mapleFU cac0875
[ADD] adding range for ByteArray benchmark
mapleFU 2679cd3
Merge branch 'main' into parquet/benchmark-delta-length-byte
mapleFU e0e78cd
add ndv and Dict encoding for ByteArray
mapleFU 1bd350d
add parameter name
mapleFU 63445d7
fix lint
mapleFU 0882b93
change benchmark args
mapleFU be77f9d
update: solve comments
mapleFU 71b0d9a
Merge branch 'main' into parquet/benchmark-delta-length-byte
mapleFU 803de33
address comment
mapleFU File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading. Please reload this page.
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -569,6 +569,133 @@ BENCHMARK(BM_DeltaBitPackingDecode_Int64_Narrow)->Range(MIN_RANGE, MAX_RANGE); | ||
| BENCHMARK(BM_DeltaBitPackingDecode_Int32_Wide)->Range(MIN_RANGE, MAX_RANGE); | ||
| BENCHMARK(BM_DeltaBitPackingDecode_Int64_Wide)->Range(MIN_RANGE, MAX_RANGE); | ||
| static void ByteArrayCustomArguments(benchmark::internal::Benchmark* b) { | ||
| b->ArgsProduct({{8, 64, 1024}, {512, 2048}}) | ||
| ->ArgNames({"max-string-length", "batch-size"}); | ||
| } | ||
| void EncodingByteArrayBenchmark(benchmark::State& state, Encoding::type encoding) { | ||
| ::arrow::random::RandomArrayGenerator rag(0); | ||
| // Using arrow generator to generate random data. | ||
| int32_t max_length = static_cast<int32_t>(state.range(0)); | ||
| int32_t array_size = static_cast<int32_t>(state.range(1)); | ||
| auto array = | ||
| rag.String(/* size */ array_size, /* min_length */ 0, /* max_length */ max_length, | ||
| /* null_probability */ 0); | ||
| const auto array_actual = | ||
| ::arrow::internal::checked_pointer_cast<::arrow::StringArray>(array); | ||
| auto encoder = MakeTypedEncoder<ByteArrayType>(encoding); | ||
| std::vector<ByteArray> values; | ||
| for (int i = 0; i < array_actual->length(); ++i) { | ||
| values.emplace_back(array_actual->GetView(i)); | ||
| } | ||
| for (auto _ : state) { | ||
| encoder->Put(values.data(), static_cast<int>(values.size())); | ||
| encoder->FlushValues(); | ||
| } | ||
| state.SetItemsProcessed(state.iterations() * array_actual->length()); | ||
| state.SetBytesProcessed(state.iterations() * (array_actual->value_data()->size() + | ||
| array_actual->value_offsets()->size())); | ||
| } | ||
| static void BM_DeltaLengthEncodingByteArray(benchmark::State& state) { | ||
| EncodingByteArrayBenchmark(state, Encoding::DELTA_LENGTH_BYTE_ARRAY); | ||
| } | ||
| static void BM_PlainEncodingByteArray(benchmark::State& state) { | ||
| EncodingByteArrayBenchmark(state, Encoding::PLAIN); | ||
| } | ||
| void DecodingByteArrayBenchmark(benchmark::State& state, Encoding::type encoding) { | ||
| ::arrow::random::RandomArrayGenerator rag(0); | ||
| int32_t max_length = static_cast<int32_t>(state.range(0)); | ||
| int32_t array_size = static_cast<int32_t>(state.range(1)); | ||
| // Using arrow to write, because we just benchmark decoding here. | ||
| auto array = | ||
| rag.String(/* size */ array_size, /* min_length */ 0, /* max_length */ max_length, | ||
| /* null_probability */ 0); | ||
| const auto array_actual = | ||
| ::arrow::internal::checked_pointer_cast<::arrow::StringArray>(array); | ||
| auto encoder = MakeTypedEncoder<ByteArrayType>(encoding); | ||
| encoder->Put(*array); | ||
| std::shared_ptr<Buffer> buf = encoder->FlushValues(); | ||
| std::vector<ByteArray> values; | ||
| values.resize(array->length()); | ||
| for (auto _ : state) { | ||
| auto decoder = MakeTypedDecoder<ByteArrayType>(encoding); | ||
| decoder->SetData(static_cast<int>(array->length()), buf->data(), | ||
| static_cast<int>(buf->size())); | ||
| decoder->Decode(values.data(), static_cast<int>(values.size())); | ||
| ::benchmark::DoNotOptimize(values); | ||
| } | ||
| state.SetItemsProcessed(state.iterations() * array->length()); | ||
mapleFU marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| state.SetBytesProcessed(state.iterations() * (array_actual->value_data()->size() + | ||
| array_actual->value_offsets()->size())); | ||
| } | ||
| static void BM_PlainDecodingByteArray(benchmark::State& state) { | ||
| DecodingByteArrayBenchmark(state, Encoding::PLAIN); | ||
| } | ||
| static void BM_DeltaLengthDecodingByteArray(benchmark::State& state) { | ||
| DecodingByteArrayBenchmark(state, Encoding::DELTA_LENGTH_BYTE_ARRAY); | ||
| } | ||
| BENCHMARK(BM_PlainEncodingByteArray)->Apply(ByteArrayCustomArguments); | ||
| BENCHMARK(BM_DeltaLengthEncodingByteArray)->Apply(ByteArrayCustomArguments); | ||
| BENCHMARK(BM_PlainDecodingByteArray)->Apply(ByteArrayCustomArguments); | ||
| BENCHMARK(BM_DeltaLengthDecodingByteArray)->Apply(ByteArrayCustomArguments); | ||
| static void BM_DecodingByteArraySpaced(benchmark::State& state, Encoding::type encoding) { | ||
| const double null_percent = 0.02; | ||
| auto rand = ::arrow::random::RandomArrayGenerator(0); | ||
| int32_t max_length = static_cast<int32_t>(state.range(0)); | ||
| int32_t num_values = static_cast<int32_t>(state.range(1)); | ||
| const auto array = rand.String(num_values, /* min_length */ 0, | ||
| /* max_length */ max_length, null_percent); | ||
| const auto valid_bits = array->null_bitmap_data(); | ||
| const int null_count = static_cast<int>(array->null_count()); | ||
| const auto array_actual = | ||
| ::arrow::internal::checked_pointer_cast<::arrow::StringArray>(array); | ||
| std::vector<ByteArray> byte_arrays; | ||
| byte_arrays.reserve(array_actual->length()); | ||
| for (int i = 0; i < array_actual->length(); ++i) { | ||
| byte_arrays.emplace_back(array_actual->GetView(i)); | ||
| } | ||
| auto encoder = MakeTypedEncoder<ByteArrayType>(encoding); | ||
| encoder->PutSpaced(byte_arrays.data(), num_values, valid_bits, 0); | ||
| std::shared_ptr<Buffer> buf = encoder->FlushValues(); | ||
| auto decoder = MakeTypedDecoder<ByteArrayType>(encoding); | ||
| std::vector<uint8_t> decode_values(num_values * sizeof(ByteArray)); | ||
| auto decode_buf = reinterpret_cast<ByteArray*>(decode_values.data()); | ||
| for (auto _ : state) { | ||
| decoder->SetData(num_values - null_count, buf->data(), static_cast<int>(buf->size())); | ||
| decoder->DecodeSpaced(decode_buf, num_values, null_count, valid_bits, 0); | ||
| ::benchmark::DoNotOptimize(decode_buf); | ||
| } | ||
| state.counters["null_percent"] = null_percent * 100; | ||
mapleFU marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| state.SetItemsProcessed(state.iterations() * array_actual->length()); | ||
| state.SetBytesProcessed(state.iterations() * (array_actual->value_data()->size() + | ||
| array_actual->value_offsets()->size())); | ||
| } | ||
| static void BM_PlainDecodingSpacedByteArray(benchmark::State& state) { | ||
| BM_DecodingByteArraySpaced(state, Encoding::PLAIN); | ||
| } | ||
| static void BM_DeltaLengthDecodingSpacedByteArray(benchmark::State& state) { | ||
| BM_DecodingByteArraySpaced(state, Encoding::DELTA_LENGTH_BYTE_ARRAY); | ||
| } | ||
| BENCHMARK(BM_PlainDecodingSpacedByteArray)->Apply(ByteArrayCustomArguments); | ||
| BENCHMARK(BM_DeltaLengthDecodingSpacedByteArray)->Apply(ByteArrayCustomArguments); | ||
| template <typename Type> | ||
| static void DecodeDict(std::vector<typename Type::c_type>& values, | ||
| benchmark::State& state) { | ||
| @@ -634,6 +761,29 @@ static void BM_DictDecodingInt64_literals(benchmark::State& state) { | ||
| BENCHMARK(BM_DictDecodingInt64_literals)->Range(MIN_RANGE, MAX_RANGE); | ||
| static void BM_DictDecodingByteArray(benchmark::State& state) { | ||
| ::arrow::random::RandomArrayGenerator rag(0); | ||
| // Using arrow generator to generate random data. | ||
| int32_t max_length = static_cast<int32_t>(state.range(0)); | ||
| int32_t array_size = static_cast<int32_t>(state.range(1)); | ||
| auto array = | ||
| rag.String(/* size */ array_size, /* min_length */ 0, /* max_length */ max_length, | ||
| /* null_probability */ 0); | ||
| const auto array_actual = | ||
| ::arrow::internal::checked_pointer_cast<::arrow::StringArray>(array); | ||
| auto encoder = MakeDictDecoder<ByteArrayType>(); | ||
| std::vector<ByteArray> values; | ||
| for (int i = 0; i < array_actual->length(); ++i) { | ||
| values.emplace_back(array_actual->GetView(i)); | ||
| } | ||
| DecodeDict<ByteArrayType>(values, state); | ||
mapleFU marked this conversation as resolved.
Uh oh!There was an error while loading. Please reload this page. | ||
| state.SetItemsProcessed(state.iterations() * array_actual->length()); | ||
| state.SetBytesProcessed(state.iterations() * (array_actual->value_data()->size() + | ||
| array_actual->value_offsets()->size())); | ||
| } | ||
| BENCHMARK(BM_DictDecodingByteArray)->Apply(ByteArrayCustomArguments); | ||
| // ---------------------------------------------------------------------- | ||
| // Shared benchmarks for decoding using arrow builders | ||
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.