ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 7 commits into from
Closed

ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads #4999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
85f9b72
Initial threading of read_dictionary parameter, not terribly satisfying
wesm Aug 2, 2019
9d50351
Read Parquet fields directly as DictionaryArray in parquet.read_table…
wesm Aug 2, 2019
7237e69
Fix C++ and Python unit tests
wesm Aug 2, 2019
8e2b70b
Expand read_dictionary with ParquetDataset test for multiple files
wesm Aug 2, 2019
0f450d5
Clean up FileReaderBuilder. Add simle Python docs
wesm Aug 3, 2019
ee73d7b
Add missing PARQUET_EXPORT
wesm Aug 5, 2019
2ca3881
Improve docstring for read_dictionary parameter, add to ParquetDataset
wesm Aug 5, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 16 additions & 21 deletions cpp/src/parquet/arrow/arrow-reader-writer-test.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
::arrow::default_memory_pool(), &reader));
ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
::arrow::default_memory_pool(), &reader));

reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
std::shared_ptr<Buffer> buffer;
ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, out));
::arrow::default_memory_pool(), out));
}

void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
::arrow::default_memory_pool(), &reader));

ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
properties.set_batch_size(100);

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(), properties, &reader));
FileReaderBuilder builder;
ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
ASSERT_OK(builder.properties(properties)->Build(&reader));

std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
::arrow::default_memory_pool(), &reader));

int64_t num_rows_returned = 0;
ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
::arrow::default_memory_pool(), &reader));

// Read everything
std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

// Read 1 record at a time
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader));
::arrow::default_memory_pool(), &reader));

std::unique_ptr<ColumnReader> col_reader;
ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
void InitReader() {
std::shared_ptr<Buffer> buffer;
ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
ASSERT_OK_NO_THROW(
OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
::parquet::default_reader_properties(), nullptr, &reader_));
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
::arrow::default_memory_pool(), &reader_));
}

void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

void CheckReadWholeFile(const Table& expected) {
std::unique_ptr<FileReader> reader;
ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
::arrow::default_memory_pool(), properties_, &reader));

FileReaderBuilder builder;
ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
ASSERT_OK(builder.properties(properties_)->Build(&reader));

std::shared_ptr<Table> actual;
ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down
56 changes: 42 additions & 14 deletions cpp/src/parquet/arrow/reader.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
MemoryPool* pool, const ReaderProperties& props,
const std::shared_ptr<FileMetaData>& metadata,
std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
reader);
FileReaderBuilder::FileReaderBuilder()
: pool_(::arrow::default_memory_pool()),
properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
const ReaderProperties& properties,
const std::shared_ptr<FileMetaData>& metadata) {
PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
pool_ = pool;
return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
const ArrowReaderProperties& arg_properties) {
properties_ = arg_properties;
return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
MemoryPool* pool, const ReaderProperties& props,
const std::shared_ptr<FileMetaData>& metadata,
std::unique_ptr<FileReader>* reader) {
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file, props, metadata));
return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
MemoryPool* pool, const ArrowReaderProperties& properties,
std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
file, ::parquet::default_reader_properties(), nullptr));
return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down
33 changes: 27 additions & 6 deletions cpp/src/parquet/arrow/reader.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
public:
FileReaderBuilder();

::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
const ReaderProperties& properties = default_reader_properties(),
const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

ParquetFileReader* raw_reader() { return raw_reader_.get(); }

FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
::arrow::Status Build(std::unique_ptr<FileReader>* out);

private:
::arrow::MemoryPool* pool_;
ArrowReaderProperties properties_;
std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* allocator,
const ReaderProperties& properties,
const std::shared_ptr<FileMetaData>& metadata,
std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* allocator,
const ReaderProperties& properties,
const std::shared_ptr<FileMetaData>& metadata,
std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* allocator,
Expand Down
15 changes: 14 additions & 1 deletion docs/source/python/parquet.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for

@xhochyxhochyAug 5, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?

@wesmwesmAug 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types

@xhochyxhochyAug 5, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.

@xhochyxhochyAug 5, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)

@wesmwesmAug 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up

@wesmwesmAug 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140

string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down
31 changes: 16 additions & 15 deletions python/pyarrow/_parquet.pxd
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
ReaderProperties default_reader_properties()

cdef cppclass ParquetFileReader:
@staticmethod
unique_ptr[ParquetFileReader] Open(
const shared_ptr[RandomAccessFile]& file,
const ReaderProperties& props,
const shared_ptr[CFileMetaData]& metadata)

@staticmethod
unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
cdef cppclass ArrowReaderProperties:
pass
ArrowReaderProperties()
void set_read_dictionary(int column_index, c_bool read_dict)
c_bool read_dictionary()
void set_batch_size()
int64_t batch_size()

ArrowReaderProperties default_arrow_reader_properties()

CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
CMemoryPool* allocator,
const ReaderProperties& properties,
const shared_ptr[CFileMetaData]& metadata,
unique_ptr[FileReader]* reader)

cdef cppclass FileReader:
FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

void set_use_threads(c_bool use_threads)

cdef cppclass FileReaderBuilder:
FileReaderBuilder()
CStatus Open(const shared_ptr[RandomAccessFile]& file,
const ReaderProperties& properties,
const shared_ptr[CFileMetaData]& metadata)

ParquetFileReader* raw_reader()
FileReaderBuilder* memory_pool(CMemoryPool*)
FileReaderBuilder* properties(const ArrowReaderProperties&)
CStatus Build(unique_ptr[FileReader]* out)

CStatus FromParquetSchema(
const SchemaDescriptor* parquet_schema,
const ArrowReaderProperties& properties,
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads#4999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
7
commits into

from

 Closed


ARROW-3325: [Python][Parquet] Add "read_dictionary" argument to parquet.read_table, ParquetDataset to enable direct-to-DictionaryArray reads
#4999
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


7 commits


Select commit
Hold shift + click to select a range
85f9b72

Initial threading of read_dictionary parameter, not terribly satisfying


wesm Aug 2, 2019
9d50351

Read Parquet fields directly as DictionaryArray in parquet.read_table…


wesm Aug 2, 2019
7237e69

Fix C++ and Python unit tests


wesm Aug 2, 2019
8e2b70b

Expand read_dictionary with ParquetDataset test for multiple files


wesm Aug 2, 2019
0f450d5

Clean up FileReaderBuilder. Add simle Python docs


wesm Aug 3, 2019
ee73d7b

Add missing PARQUET_EXPORT


wesm Aug 5, 2019
2ca3881

Improve docstring for read_dictionary parameter, add to ParquetDataset


wesm Aug 5, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


37 changes: 16 additions & 21 deletions
cpp/src/parquet/arrow/arrow-reader-writer-test.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -383,8 +383,7 @@ void DoConfiguredRoundtrip(

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));
 ASSERT_OK_NO_THROW(reader->ReadTable(out));
}

Expand DownExpand Up@@ -421,8 +420,7 @@ void DoSimpleRoundtrip(const std::shared_ptr<Table>& table, bool use_threads,

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 reader->set_use_threads(use_threads);

Expand DownExpand Up@@ -499,8 +497,7 @@ class TestParquetIO : public ::testing::Test {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(sink_->Finish(&buffer));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, out));
 ::arrow::default_memory_pool(), out));
 }

 void ReadSingleColumnFile(std::unique_ptr<FileReader> file_reader,
Expand DownExpand Up@@ -1869,8 +1866,7 @@ TEST(TestArrowReadWrite, ReadSingleRowGroup) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 ASSERT_EQ(2, reader->num_row_groups());

Expand DownExpand Up@@ -1907,8 +1903,9 @@ TEST(TestArrowReadWrite, GetRecordBatchReader) {
 properties.set_batch_size(100);

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), properties, &reader));
 FileReaderBuilder builder;
 ASSERT_OK(builder.Open(std::make_shared<BufferReader>(buffer)));
 ASSERT_OK(builder.properties(properties)->Build(&reader));

 std::shared_ptr<::arrow::RecordBatchReader> rb_reader;
 ASSERT_OK_NO_THROW(reader->GetRecordBatchReader({0, 1}, &rb_reader));
Expand DownExpand Up@@ -1938,8 +1935,7 @@ TEST(TestArrowReadWrite, ScanContents) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 int64_t num_rows_returned = 0;
 ASSERT_OK_NO_THROW(reader->ScanContents({}, 256, &num_rows_returned));
Expand DownExpand Up@@ -1994,8 +1990,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 // Read everything
 std::shared_ptr<Table> result;
Expand All@@ -2004,8 +1999,7 @@ TEST(TestArrowReadWrite, ListLargeRecords) {

 // Read 1 record at a time
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader));
 ::arrow::default_memory_pool(), &reader));

 std::unique_ptr<ColumnReader> col_reader;
 ASSERT_OK(reader->GetColumn(0, &col_reader));
Expand DownExpand Up@@ -2216,9 +2210,8 @@ class TestNestedSchemaRead : public ::testing::TestWithParam<Repetition::type> {
 void InitReader() {
 std::shared_ptr<Buffer> buffer;
 ASSERT_OK_NO_THROW(nested_parquet_->Finish(&buffer));
 ASSERT_OK_NO_THROW(
 OpenFile(std::make_shared<BufferReader>(buffer), ::arrow::default_memory_pool(),
 ::parquet::default_reader_properties(), nullptr, &reader_));
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer),
 ::arrow::default_memory_pool(), &reader_));
 }

 void InitNewParquetFile(const std::shared_ptr<GroupNode>& schema, int num_rows) {
Expand DownExpand Up@@ -2780,8 +2773,10 @@ class TestArrowReadDictionary : public ::testing::TestWithParam<double> {

 void CheckReadWholeFile(const Table& expected) {
 std::unique_ptr<FileReader> reader;
 ASSERT_OK_NO_THROW(OpenFile(std::make_shared<BufferReader>(buffer_),
 ::arrow::default_memory_pool(), properties_, &reader));

 FileReaderBuilder builder;
 ASSERT_OK_NO_THROW(builder.Open(std::make_shared<BufferReader>(buffer_)));
 ASSERT_OK(builder.properties(properties_)->Build(&reader));

 std::shared_ptr<Table> actual;
 ASSERT_OK_NO_THROW(reader->ReadTable(&actual));
Expand Down

56 changes: 42 additions & 14 deletions
cpp/src/parquet/arrow/reader.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -814,28 +814,56 @@ Status FileReader::Make(::arrow::MemoryPool* pool,
 return Make(pool, std::move(reader), default_arrow_reader_properties(), out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 std::unique_ptr<ParquetReader> pq_reader;
 PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(file, props, metadata));
 return FileReader::Make(pool, std::move(pq_reader), default_arrow_reader_properties(),
 reader);
FileReaderBuilder::FileReaderBuilder()
 : pool_(::arrow::default_memory_pool()),
 properties_(default_arrow_reader_properties()) {}

Status FileReaderBuilder::Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata) {
 PARQUET_CATCH_NOT_OK(raw_reader_ = ParquetReader::Open(file, properties, metadata));
 return Status::OK();
}

FileReaderBuilder* FileReaderBuilder::memory_pool(::arrow::MemoryPool* pool) {
 pool_ = pool;
 return this;
}

FileReaderBuilder* FileReaderBuilder::properties(
 const ArrowReaderProperties& arg_properties) {
 properties_ = arg_properties;
 return this;
}

Status FileReaderBuilder::Build(std::unique_ptr<FileReader>* out) {
 return FileReader::Make(pool_, std::move(raw_reader_), properties_, out);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, std::unique_ptr<FileReader>* reader) {
 return OpenFile(file, pool, ::parquet::default_reader_properties(), nullptr, reader);
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 MemoryPool* pool, const ReaderProperties& props,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader) {
 // Deprecated since 0.15.0
 FileReaderBuilder builder;
 RETURN_NOT_OK(builder.Open(file, props, metadata));
 return builder.memory_pool(pool)->Build(reader);
}

Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
::arrow::MemoryPool* pool, const ArrowReaderProperties& properties,
 MemoryPool* pool, const ArrowReaderProperties& properties,
 std::unique_ptr<FileReader>* reader) {
std::unique_ptr<ParquetReader> pq_reader;
PARQUET_CATCH_NOT_OK(pq_reader = ParquetReader::Open(
 file, ::parquet::default_reader_properties(), nullptr));
 return FileReader::Make(pool, std::move(pq_reader), properties, reader);
// Deprecated since 0.15.0
FileReaderBuilder builder;
RETURN_NOT_OK(builder.Open(file));
 return builder.memory_pool(pool)->properties(properties)->Build(reader);
}

} // namespace arrow
Expand Down

33 changes: 27 additions & 6 deletions
cpp/src/parquet/arrow/reader.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -24,6 +24,7 @@
#include <vector>

#include "parquet/platform.h"
#include "parquet/properties.h"

namespace arrow {

Expand DownExpand Up@@ -286,22 +287,42 @@ class PARQUET_EXPORT ColumnReader {
 std::shared_ptr<::arrow::ChunkedArray>* out) = 0;
};

// Helper function to create a file reader from an implementation of an Arrow
// random access file
//
// metadata : separately-computed file metadata, can be nullptr
/// \brief Experimental helper class for bindings (like Python) that struggle
/// either with std::move or C++ exceptions
class PARQUET_EXPORT FileReaderBuilder {
 public:
 FileReaderBuilder();

 ::arrow::Status Open(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 const ReaderProperties& properties = default_reader_properties(),
 const std::shared_ptr<FileMetaData>& metadata = NULLPTR);

 ParquetFileReader* raw_reader() { return raw_reader_.get(); }

 FileReaderBuilder* memory_pool(::arrow::MemoryPool* pool);
 FileReaderBuilder* properties(const ArrowReaderProperties& arg_properties);
 ::arrow::Status Build(std::unique_ptr<FileReader>* out);

 private:
 ::arrow::MemoryPool* pool_;
 ArrowReaderProperties properties_;
 std::unique_ptr<ParquetFileReader> raw_reader_;
};

PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
 const ReaderProperties& properties,
 const std::shared_ptr<FileMetaData>& metadata,
 std::unique_ptr<FileReader>* reader);

ARROW_DEPRECATED("Deprecated since 0.15.0. Use FileReaderBuilder")
PARQUET_EXPORT
::arrow::Status OpenFile(const std::shared_ptr<::arrow::io::RandomAccessFile>& file,
 ::arrow::MemoryPool* allocator,
Expand Down

15 changes: 14 additions & 1 deletion
docs/source/python/parquet.rst

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -210,6 +210,19 @@ Alternatively python ``with`` syntax can also be use:
Data Type Handling
------------------

Reading types as DictionaryArray
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The ``read_dictionary`` option in ``read_table`` and ``ParquetDataset`` will
cause columns to be read as ``DictionaryArray``, which will become
``pandas.Categorical`` when converted to pandas. This option is only valid for
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the limitation intended or simply because we only have it implemented for binary columns?
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only implemented for BYTE_ARRAY columns at the moment. We could expand that but there is little benefit from a performance/memory use point of view for the primitive types
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've also used this (through pandas.Categorical) in the past on date and float types (e.g. in some datasets you can have 1000s of products that only have one of 5 prices). This often gave a 4-6x improvement in memory usage for these columns.
@xhochyxhochyAug 5, 2019

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(just dropping it here as FYI)
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I'll open a JIRA as a follow up
@wesmwesmAug 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://issues.apache.org/jira/browse/ARROW-6140
string and binary column types, and it can yield significantly lower memory use
and improved performance for columns with many repeated string values.

.. code-block:: python

 pq.read_table(table, where, read_dictionary=['binary_c0', 'stringb_c2'])

Storing timestamps
~~~~~~~~~~~~~~~~~~

Expand DownExpand Up@@ -305,7 +318,7 @@ A dataset partitioned by year and month may look like on disk:
 ...

Writing to Partitioned Datasets
------------------------------------------------
-------------------------------

You can write a partitioned dataset for any ``pyarrow`` file system that is a
file-store (e.g. local, HDFS, S3). The default behaviour when no filesystem is
Expand Down

31 changes: 16 additions & 15 deletions
python/pyarrow/_parquet.pxd

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -328,14 +328,6 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil:
 ReaderProperties default_reader_properties()

 cdef cppclass ParquetFileReader:
 @staticmethod
 unique_ptr[ParquetFileReader] Open(
 const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& props,
 const shared_ptr[CFileMetaData]& metadata)

 @staticmethod
 unique_ptr[ParquetFileReader] OpenFile(const c_string& path)
 shared_ptr[CFileMetaData] metadata()


Expand All@@ -359,16 +351,14 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:

cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
 cdef cppclass ArrowReaderProperties:
 pass
 ArrowReaderProperties()
 void set_read_dictionary(int column_index, c_bool read_dict)
 c_bool read_dictionary()
 void set_batch_size()
 int64_t batch_size()

 ArrowReaderProperties default_arrow_reader_properties()

 CStatus OpenFile(const shared_ptr[RandomAccessFile]& file,
 CMemoryPool* allocator,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata,
 unique_ptr[FileReader]* reader)

 cdef cppclass FileReader:
 FileReader(CMemoryPool* pool, unique_ptr[ParquetFileReader] reader)
 CStatus ReadColumn(int i, shared_ptr[CChunkedArray]* out)
Expand All@@ -390,6 +380,17 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:

 void set_use_threads(c_bool use_threads)

 cdef cppclass FileReaderBuilder:
 FileReaderBuilder()
 CStatus Open(const shared_ptr[RandomAccessFile]& file,
 const ReaderProperties& properties,
 const shared_ptr[CFileMetaData]& metadata)

 ParquetFileReader* raw_reader()
 FileReaderBuilder* memory_pool(CMemoryPool*)
 FileReaderBuilder* properties(const ArrowReaderProperties&)
 CStatus Build(unique_ptr[FileReader]* out)

 CStatus FromParquetSchema(
 const SchemaDescriptor* parquet_schema,
 const ArrowReaderProperties& properties,
Expand Down
Loading