This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291

Closed
wesm wants to merge 2 commits into from
Closed

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c7589f7
Add ReadRowGroup and num_row_group methods to arrow::FileReader
wesm Apr 5, 2017
6d2b48a
Add virtual dtor
wesm Apr 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions cmake_modules/FindClangTools.cmake
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
set(CLANG_TIDY_FOUND 0)
message("clang-tidy not found")
else()
set(CLANG_TIDY_FOUND 1)
message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
set(CLANG_FORMAT_FOUND 0)
message("clang-format not found")
else()
set(CLANG_FORMAT_FOUND 1)
message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()

146 changes: 92 additions & 54 deletions src/parquet/arrow/arrow-reader-writer-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
int64_t row_group_size, std::shared_ptr<Buffer>* out) {
auto sink = std::make_shared<InMemoryOutputStream>();

ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
row_group_size, default_writer_properties()));
*out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
int64_t row_group_size, const std::vector<int>& column_subset,
std::shared_ptr<Table>* out) {
std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(
OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));

reader->set_num_threads(num_threads);

if (column_subset.size() > 0) {
ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
} else {
// Read everything
ASSERT_OK_NO_THROW(reader->ReadTable(out));
}
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
ASSERT_NE(nullptr, out->get());
}

void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
std::shared_ptr<::arrow::Table> out;
std::unique_ptr<FileReader> reader;
ReaderFromSink(&reader);
ReadTableFromFile(std::move(reader), &out);
ASSERT_EQ(1, out->num_columns());
ASSERT_EQ(values->length(), out->num_rows());

std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
ASSERT_EQ(1, chunked_array->num_chunks());
ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
}

void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
int64_t null_count, std::shared_ptr<Table>* out) {
std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
*out = MakeSimpleTable(parent_lists, nullable_parent_lists);
}

void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
this->sink_ = std::make_shared<InMemoryOutputStream>();
ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
values->length(), default_writer_properties()));
void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
std::shared_ptr<::arrow::Table> out;
std::unique_ptr<FileReader> reader;
ReaderFromSink(&reader);
ReadTableFromFile(std::move(reader), &out);
ASSERT_EQ(1, out->num_columns());
ASSERT_EQ(values->length(), out->num_rows());

std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
ASSERT_EQ(1, chunked_array->num_chunks());
ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
}

this->ReadAndCheckSingleColumnTable(values);
void CheckRoundTrip(const std::shared_ptr<Table>& table) {
std::shared_ptr<Table> result;
DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
ASSERT_TRUE(table->Equals(*result));
}

template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
std::shared_ptr<Table> table = MakeSimpleTable(values, true);
this->WriteReadAndCheckSingleColumnTable(table);
this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
std::shared_ptr<Table> table;
this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
this->WriteReadAndCheckSingleColumnTable(table);
this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
std::shared_ptr<Table> table;
this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
this->WriteReadAndCheckSingleColumnTable(table);
this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
std::shared_ptr<Table> table;
this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
this->WriteReadAndCheckSingleColumnTable(table);
this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
std::shared_ptr<Table> table;
this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
this->WriteReadAndCheckSingleColumnTable(table);
this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
std::shared_ptr<Table> table;
this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
this->WriteReadAndCheckSingleColumnTable(table);
this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
std::shared_ptr<::arrow::Column> column;
std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

std::shared_ptr<Array> values;
for (int i = 0; i < num_columns; ++i) {
std::vector<std::shared_ptr<Array>> arrays;
std::shared_ptr<Array> values;
ASSERT_OK(NullableArray<::arrow::DoubleType>(
num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
std::stringstream ss;
ss << "col" << i;
column = MakeColumn(ss.str(), values, true);

for (int j = 0; j < nchunks; ++j) {
arrays.push_back(values);
}
column = MakeColumn(ss.str(), arrays, true);

columns[i] = column;
fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
*out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
auto sink = std::make_shared<InMemoryOutputStream>();

ASSERT_OK_NO_THROW(WriteTable(
*table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
const int num_columns = 20;
const int num_rows = 1000;
const int num_threads = 4;

std::shared_ptr<Buffer> buffer = sink->GetBuffer();
std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(
OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
std::shared_ptr<Table> table;
MakeDoubleTable(num_columns, num_rows, 1, &table);

reader->set_num_threads(num_threads);
std::shared_ptr<Table> result;
DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

if (column_subset.size() > 0) {
ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
} else {
// Read everything
ASSERT_OK_NO_THROW(reader->ReadTable(out));
}
ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
const int num_columns = 20;
const int num_rows = 1000;
const int num_threads = 4;

std::shared_ptr<Table> table;
MakeDoubleTable(num_columns, num_rows, &table);
MakeDoubleTable(num_columns, num_rows, 1, &table);

std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

ASSERT_TRUE(table->Equals(*result));
std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(
OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));

ASSERT_EQ(2, reader->num_row_groups());

std::shared_ptr<Table> r1, r2;
// Read everything
ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

std::shared_ptr<Table> concatenated;
ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
const int num_threads = 4;

std::shared_ptr<Table> table;
MakeDoubleTable(num_columns, num_rows, &table);
MakeDoubleTable(num_columns, num_rows, 1, &table);

std::shared_ptr<Table> result;
std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content

This repository was archived by the owner on May 10, 2024. It is now read-only.

PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader#291
 Closed

wesm
wants to merge
2
commits into

from

 Closed


PARQUET-946: Add ReadRowGroup and num_row_group methods to arrow::FileReader
#291
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
c7589f7

Add ReadRowGroup and num_row_group methods to arrow::FileReader


wesm Apr 5, 2017
6d2b48a

Add virtual dtor


wesm Apr 6, 2017

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


19 changes: 9 additions & 10 deletions
cmake_modules/FindClangTools.cmake

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -27,34 +27,33 @@
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found

find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_TIDY_BIN
 NAMES clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
 set(CLANG_TIDY_FOUND 0)
 message("clang-tidy not found")
else()
 set(CLANG_TIDY_FOUND 1)
 message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()

find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
find_program(CLANG_FORMAT_BIN
 NAMES clang-format-3.9 clang-format-3.8 clang-format-3.7 clang-format-3.6 clang-format
 PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
 NO_DEFAULT_PATH
)

if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
 set(CLANG_FORMAT_FOUND 0)
 message("clang-format not found")
else()
 set(CLANG_FORMAT_FOUND 1)
 message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()


146 changes: 92 additions & 54 deletions
src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -197,6 +197,36 @@ using ParquetDataType = DataType<test_traits<T>::parquet_enum>;
template <typename T>
using ParquetWriter = TypedColumnWriter<ParquetDataType<T>>;

void WriteTableToBuffer(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, std::shared_ptr<Buffer>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), sink,
 row_group_size, default_writer_properties()));
 *out = sink->GetBuffer();
}

void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 int64_t row_group_size, const std::vector<int>& column_subset,
 std::shared_ptr<Table>* out) {
 std::shared_ptr<Buffer> buffer;
 WriteTableToBuffer(table, num_threads, row_group_size, &buffer);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 reader->set_num_threads(num_threads);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
}

template <typename TestType>
class TestParquetIO : public ::testing::Test {
 public:
Expand DownExpand Up@@ -248,19 +278,6 @@ class TestParquetIO : public ::testing::Test {
 ASSERT_NE(nullptr, out->get());
 }

 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 void PrepareListTable(int64_t size, bool nullable_lists, bool nullable_elements,
 int64_t null_count, std::shared_ptr<Table>* out) {
 std::shared_ptr<Array> values;
Expand DownExpand Up@@ -289,13 +306,23 @@ class TestParquetIO : public ::testing::Test {
 *out = MakeSimpleTable(parent_lists, nullable_parent_lists);
 }

 void WriteReadAndCheckSingleColumnTable(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Array> values = table->column(0)->data()->chunk(0);
 this->sink_ = std::make_shared<InMemoryOutputStream>();
 ASSERT_OK_NO_THROW(WriteTable(*table, ::arrow::default_memory_pool(), this->sink_,
 values->length(), default_writer_properties()));
 void ReadAndCheckSingleColumnTable(const std::shared_ptr<::arrow::Array>& values) {
 std::shared_ptr<::arrow::Table> out;
 std::unique_ptr<FileReader> reader;
 ReaderFromSink(&reader);
 ReadTableFromFile(std::move(reader), &out);
 ASSERT_EQ(1, out->num_columns());
 ASSERT_EQ(values->length(), out->num_rows());

 std::shared_ptr<ChunkedArray> chunked_array = out->column(0)->data();
 ASSERT_EQ(1, chunked_array->num_chunks());
 ASSERT_TRUE(values->Equals(chunked_array->chunk(0)));
 }

 this->ReadAndCheckSingleColumnTable(values);
 void CheckRoundTrip(const std::shared_ptr<Table>& table) {
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, 1, table->num_rows(), {}, &result);
 ASSERT_TRUE(table->Equals(*result));
 }

 template <typename ArrayType>
Expand DownExpand Up@@ -401,37 +428,37 @@ TYPED_TEST(TestParquetIO, SingleColumnTableOptionalReadWrite) {

 ASSERT_OK(NullableArray<TypeParam>(SMALL_SIZE, 10, kDefaultSeed, &values));
 std::shared_ptr<Table> table = MakeSimpleTable(values, true);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListNullableColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, true, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, true, false, 10, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListTable(SMALL_SIZE, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleNullableListRequiredListRequiredColumnReadWrite) {
 std::shared_ptr<Table> table;
 this->PrepareListOfListTable(SMALL_SIZE, true, false, false, 0, &table);
 this->WriteReadAndCheckSingleColumnTable(table);
 this->CheckRoundTrip(table);
}

TYPED_TEST(TestParquetIO, SingleColumnRequiredChunkedWrite) {
Expand DownExpand Up@@ -756,18 +783,24 @@ TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) {
 this->CheckSingleColumnRequiredTableRead(4);
}

void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out) {
void MakeDoubleTable(
 int num_columns, int num_rows, int nchunks, std::shared_ptr<Table>* out) {
 std::shared_ptr<::arrow::Column> column;
 std::vector<std::shared_ptr<::arrow::Column>> columns(num_columns);
 std::vector<std::shared_ptr<::arrow::Field>> fields(num_columns);

 std::shared_ptr<Array> values;
 for (int i = 0; i < num_columns; ++i) {
 std::vector<std::shared_ptr<Array>> arrays;
 std::shared_ptr<Array> values;
 ASSERT_OK(NullableArray<::arrow::DoubleType>(
 num_rows, num_rows / 10, static_cast<uint32_t>(i), &values));
 std::stringstream ss;
 ss << "col" << i;
 column = MakeColumn(ss.str(), values, true);

 for (int j = 0; j < nchunks; ++j) {
 arrays.push_back(values);
 }
 column = MakeColumn(ss.str(), arrays, true);

 columns[i] = column;
 fields[i] = column->field();
Expand All@@ -776,41 +809,46 @@ void MakeDoubleTable(int num_columns, int num_rows, std::shared_ptr<Table>* out)
 *out = std::make_shared<Table>(schema, columns);
}

void DoTableRoundtrip(const std::shared_ptr<Table>& table, int num_threads,
 const std::vector<int>& column_subset, std::shared_ptr<Table>* out) {
 auto sink = std::make_shared<InMemoryOutputStream>();

 ASSERT_OK_NO_THROW(WriteTable(
 *table, ::arrow::default_memory_pool(), sink, (table->num_rows() + 1) / 2));
TEST(TestArrowReadWrite, MultithreadedRead) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Buffer> buffer = sink->GetBuffer();
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 reader->set_num_threads(num_threads);
 std::shared_ptr<Table> result;
 DoSimpleRoundtrip(table, num_threads, table->num_rows(), {}, &result);

 if (column_subset.size() > 0) {
 ASSERT_OK_NO_THROW(reader->ReadTable(column_subset, out));
 } else {
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
 }
 ASSERT_TRUE(table->Equals(*result));
}

TEST(TestArrowReadWrite, MultithreadedRead) {
TEST(TestArrowReadWrite, ReadSingleRowGroup) {
 const int num_columns = 20;
 const int num_rows = 1000;
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
DoTableRoundtrip(table, num_threads, {}, &result);
 std::shared_ptr<Buffer> buffer;
WriteTableToBuffer(table, 1, num_rows / 2, &buffer);

 ASSERT_TRUE(table->Equals(*result));
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));

 ASSERT_EQ(2, reader->num_row_groups());

 std::shared_ptr<Table> r1, r2;
 // Read everything
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(0, &r1));
 ASSERT_OK_NO_THROW(reader->ReadRowGroup(1, &r2));

 std::shared_ptr<Table> concatenated;
 ASSERT_OK(ConcatenateTables({r1, r2}, &concatenated));

 ASSERT_TRUE(table->Equals(*concatenated));
}

TEST(TestArrowReadWrite, ReadColumnSubset) {
Expand All@@ -819,11 +857,11 @@ TEST(TestArrowReadWrite, ReadColumnSubset) {
 const int num_threads = 4;

 std::shared_ptr<Table> table;
 MakeDoubleTable(num_columns, num_rows, &table);
 MakeDoubleTable(num_columns, num_rows, 1, &table);

 std::shared_ptr<Table> result;
 std::vector<int> column_subset = {0, 4, 8, 10};
DoTableRoundtrip(table, num_threads, column_subset, &result);
DoSimpleRoundtrip(table, num_threads, table->num_rows(), column_subset, &result);

 std::vector<std::shared_ptr<::arrow::Column>> ex_columns;
 std::vector<std::shared_ptr<::arrow::Field>> ex_fields;
Expand Down
Loading