ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 5 commits into from
Closed

ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types #5302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9ae4c47
Use pandas null coding consistently on List and Struct types also
wesm Sep 5, 2019
2b2ac65
Do not attempt to recode DictionaryArray in Table.to_pandas if alread…
wesm Sep 6, 2019
c5a50a4
Raise more helpful exception when trying to convert array of NaN to l…
wesm Sep 6, 2019
ee8574d
Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors
wesm Sep 6, 2019
6b88de0
Rebase and remove duplicated code
wesm Sep 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions cpp/src/arrow/python/arrow_to_pandas.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
FunctionContext ctx;
for (int i = 0; i < table->num_columns(); i++) {
std::shared_ptr<ChunkedArray> col = table->column(i);
if (col->type()->id() == Type::DICTIONARY) {
// No need to dictionary encode again. Came up in ARROW-6434,
// ARROW-6435
continue;
}
if (categorical_columns.count(table->field(i)->name())) {
if (table->field(i)->type()->id() == Type::DICTIONARY) {
// this column is already dictionary encoded
continue;
}
Datum out;
RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down
7 changes: 7 additions & 0 deletions cpp/src/arrow/python/helpers.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
std::string obj_as_str;
RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
return Status::TypeError("Could not convert ", obj_as_str, " with type ",
Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
if (PyLong_Check(obj)) {
int overflow = 0;
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/python/helpers.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down
256 changes: 139 additions & 117 deletions cpp/src/arrow/python/python_to_arrow.cc
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
case Type::TYPE: { \
if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
return value_converter_->AppendMultiple(obj, value_length); \
} \
return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
}

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
case Type::TYPE: { \
return value_converter_->AppendMultiple(obj, value_length); \
}

@wesmwesmSep 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code


template <typename TypeClass, NullCoding null_coding>
class ListConverter
: public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
null_coding> {

@wesmwesmSep 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes

public:
using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
}

template <int NUMPY_TYPE, typename Type>
Status AppendNdarrayTypedItem(PyArrayObject* arr);
Status AppendNdarrayItem(PyObject* arr);
Status AppendNdarrayTypedItem(PyArrayObject* arr) {

@wesmwesmSep 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged

using traits = internal::npy_traits<NUMPY_TYPE>;
using T = typename traits::value_type;
using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

const bool null_sentinels_possible =
// Always treat Numpy's NaT as null
NUMPY_TYPE == NPY_DATETIME ||
// Observing pandas's null sentinels
(from_pandas_ && traits::supports_nulls);

auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

// TODO(wesm): Vector append when not strided
Ndarray1DIndexer<T> values(arr);
if (null_sentinels_possible) {
for (int64_t i = 0; i < values.size(); ++i) {
if (traits::isnull(values[i])) {
RETURN_NOT_OK(child_builder->AppendNull());
} else {
RETURN_NOT_OK(child_builder->Append(values[i]));
}
}
} else {
for (int64_t i = 0; i < values.size(); ++i) {
RETURN_NOT_OK(child_builder->Append(values[i]));
}
}
return Status::OK();
}

Status AppendNdarrayItem(PyObject* obj) {
PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

if (PyArray_NDIM(arr) != 1) {
return Status::Invalid("Can only convert 1-dimensional array values");
}

const int64_t value_length = PyArray_SIZE(arr);

switch (value_type_->id()) {
LIST_SLOW_CASE(NA)
LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
LIST_SLOW_CASE(DATE32)
LIST_SLOW_CASE(DATE64)
LIST_SLOW_CASE(TIME32)
LIST_SLOW_CASE(TIME64)
LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
LIST_SLOW_CASE(BINARY)
LIST_SLOW_CASE(FIXED_SIZE_BINARY)
LIST_SLOW_CASE(STRING)
case Type::LIST: {
if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
return Status::Invalid(
"Can only convert list types from NumPy object "
"array input");
}
return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
return value_converter_->AppendSingleVirtual(item);
});
}
default: {
return Status::TypeError("Unknown list item type: ", value_type_->ToString());
}
}
}

Status AppendItem(PyObject* obj) {
RETURN_NOT_OK(this->typed_builder_->Append());
if (PyArray_Check(obj)) {
return AppendNdarrayItem(obj);
}
const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
if (ARROW_PREDICT_FALSE(list_size == -1)) {
RETURN_IF_PYERROR();
if (!PySequence_Check(obj)) {
return internal::InvalidType(obj,
"was not a sequence or recognized null"
" for conversion to list type");
}
int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
return value_converter_->AppendMultiple(obj, list_size);
}

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
using traits = internal::npy_traits<NUMPY_TYPE>;
using T = typename traits::value_type;
using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

const bool null_sentinels_possible =
// Always treat Numpy's NaT as null
NUMPY_TYPE == NPY_DATETIME ||
// Observing pandas's null sentinels
(from_pandas_ && traits::supports_nulls);

auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

// TODO(wesm): Vector append when not strided
Ndarray1DIndexer<T> values(arr);
if (null_sentinels_possible) {
for (int64_t i = 0; i < values.size(); ++i) {
if (traits::isnull(values[i])) {
RETURN_NOT_OK(child_builder->AppendNull());
} else {
RETURN_NOT_OK(child_builder->Append(values[i]));
}
}
} else {
for (int64_t i = 0; i < values.size(); ++i) {
RETURN_NOT_OK(child_builder->Append(values[i]));
}
}
return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
case Type::TYPE: { \
if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
return value_converter_->AppendMultiple(obj, value_length); \
} \
return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
}

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
case Type::TYPE: { \
return value_converter_->AppendMultiple(obj, value_length); \
}

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

if (PyArray_NDIM(arr) != 1) {
return Status::Invalid("Can only convert 1-dimensional array values");
}

const int64_t value_length = PyArray_SIZE(arr);

switch (value_type_->id()) {
LIST_SLOW_CASE(NA)
LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
LIST_SLOW_CASE(DATE32)
LIST_SLOW_CASE(DATE64)
LIST_SLOW_CASE(TIME32)
LIST_SLOW_CASE(TIME64)
LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
LIST_SLOW_CASE(BINARY)
LIST_SLOW_CASE(FIXED_SIZE_BINARY)
LIST_SLOW_CASE(STRING)
case Type::LIST: {
if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
return Status::Invalid(
"Can only convert list types from NumPy object "
"array input");
}
return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
return value_converter_->AppendSingleVirtual(item);
});
}
default: {
return Status::TypeError("Unknown list item type: ", value_type_->ToString());
}
}
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
: public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
public:
explicit StructConverter(bool from_pandas, bool strict_conversions)
: from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

Status Init(ArrayBuilder* builder) {
builder_ = builder;
typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
const auto& struct_type = checked_cast<const StructType&>(*builder->type());

num_fields_ = typed_builder_->num_fields();
num_fields_ = this->typed_builder_->num_fields();
DCHECK_EQ(num_fields_, struct_type.num_children());

field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
std::unique_ptr<SeqConverter> value_converter;
RETURN_NOT_OK(
GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
value_converters_.push_back(std::move(value_converter));

// Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
}

Status AppendItem(PyObject* obj) {
RETURN_NOT_OK(typed_builder_->Append());
RETURN_NOT_OK(this->typed_builder_->Append());
// Note heterogenous sequences are not allowed
if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
} else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
return AppendTupleItem(obj);
} else {
return Status::TypeError("Expected sequence of dicts or tuples for struct type");
return internal::InvalidType(obj,
"was not a dict, tuple, or recognized null value"
" for conversion to struct type");
}
}

// Append a missing item
Status AppendNull() {
RETURN_NOT_OK(typed_builder_->AppendNull());
RETURN_NOT_OK(this->typed_builder_->AppendNull());
// Need to also insert a missing item on all child builders
// (compare with ListConverter)
for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
switch (type->id()) {
case Type::LIST:
*out = std::unique_ptr<SeqConverter>(
new ListConverter<ListType>(from_pandas, strict_conversions));
if (from_pandas) {
*out = std::unique_ptr<SeqConverter>(
new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
from_pandas, strict_conversions));
} else {
*out = std::unique_ptr<SeqConverter>(
new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
strict_conversions));
}
return Status::OK();
case Type::LARGE_LIST:
*out = std::unique_ptr<SeqConverter>(
new ListConverter<LargeListType>(from_pandas, strict_conversions));
if (from_pandas) {
*out = std::unique_ptr<SeqConverter>(
new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
from_pandas, strict_conversions));
} else {
*out = std::unique_ptr<SeqConverter>(
new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
strict_conversions));
}
return Status::OK();
case Type::STRUCT:
*out = std::unique_ptr<SeqConverter>(
new StructConverter(from_pandas, strict_conversions));
if (from_pandas) {
*out = std::unique_ptr<SeqConverter>(
new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
strict_conversions));
} else {
*out = std::unique_ptr<SeqConverter>(
new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
}

@wesmwesmSep 5, 2019

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes

return Status::OK();
default:
break;
Expand Down
4 changes: 2 additions & 2 deletions docker-compose.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
# docker-compose build python
# docker-compose build hdfs-integration
# docker-compose run hdfs-integration
image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
links:
- hdfs-namenode:hdfs-namenode
- hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
context: .
dockerfile: integration/hdfs/Dockerfile
args:
HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
volumes: *ubuntu-volumes

# TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types#5302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Closed

wesm
wants to merge
5
commits into

from

 Closed


ARROW-6435: [Python] Use pandas null coding consistently on List and Struct types
#5302
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


5 commits


Select commit
Hold shift + click to select a range
9ae4c47

Use pandas null coding consistently on List and Struct types also


wesm Sep 5, 2019
2b2ac65

Do not attempt to recode DictionaryArray in Table.to_pandas if alread…


wesm Sep 6, 2019
c5a50a4

Raise more helpful exception when trying to convert array of NaN to l…


wesm Sep 6, 2019
ee8574d

Use Hadoop 2.9.2 since 2.6.5 is no longer available from ASF mirrors


wesm Sep 6, 2019
6b88de0

Rebase and remove duplicated code


wesm Sep 6, 2019

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


9 changes: 5 additions & 4 deletions
cpp/src/arrow/python/arrow_to_pandas.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -1951,11 +1951,12 @@ Status ConvertTableToPandas(const PandasOptions& options,
 FunctionContext ctx;
 for (int i = 0; i < table->num_columns(); i++) {
 std::shared_ptr<ChunkedArray> col = table->column(i);
 if (col->type()->id() == Type::DICTIONARY) {
 // No need to dictionary encode again. Came up in ARROW-6434,
 // ARROW-6435
 continue;
 }
 if (categorical_columns.count(table->field(i)->name())) {
 if (table->field(i)->type()->id() == Type::DICTIONARY) {
 // this column is already dictionary encoded
 continue;
 }
 Datum out;
 RETURN_NOT_OK(DictionaryEncode(&ctx, Datum(col), &out));
 std::shared_ptr<ChunkedArray> array = out.chunked_array();
Expand Down

7 changes: 7 additions & 0 deletions
cpp/src/arrow/python/helpers.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -301,6 +301,13 @@ Status InvalidValue(PyObject* obj, const std::string& why) {
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status InvalidType(PyObject* obj, const std::string& why) {
 std::string obj_as_str;
 RETURN_NOT_OK(internal::PyObject_StdStringStr(obj, &obj_as_str));
 return Status::TypeError("Could not convert ", obj_as_str, " with type ",
 Py_TYPE(obj)->tp_name, ": ", why);
}

Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
 if (PyLong_Check(obj)) {
 int overflow = 0;
Expand Down

3 changes: 3 additions & 0 deletions
cpp/src/arrow/python/helpers.h

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -130,6 +130,9 @@ inline Status CastSize(Py_ssize_t size, int64_t* out, const char* error_msg = NU
ARROW_PYTHON_EXPORT
Status InvalidValue(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status InvalidType(PyObject* obj, const std::string& why);

ARROW_PYTHON_EXPORT
Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
ARROW_PYTHON_EXPORT
Expand Down

256 changes: 139 additions & 117 deletions
cpp/src/arrow/python/python_to_arrow.cc

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -607,8 +607,26 @@ class StringConverter
// ----------------------------------------------------------------------
// Convert lists (NumPy arrays containing lists or ndarrays as values)

template <typename TypeClass>
class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>> {
// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code

template <typename TypeClass, NullCoding null_coding>
class ListConverter
 : public TypedConverter<TypeClass, ListConverter<TypeClass, null_coding>,
 null_coding> {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 public:
 using BuilderType = typename TypeTraits<TypeClass>::BuilderType;

Expand All@@ -626,18 +644,94 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 }

 template <int NUMPY_TYPE, typename Type>
 Status AppendNdarrayTypedItem(PyArrayObject* arr);
 Status AppendNdarrayItem(PyObject* arr);
 Status AppendNdarrayTypedItem(PyArrayObject* arr) {
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved code. Unchanged
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
 }

 Status AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(this->typed_builder_->Append());
 if (PyArray_Check(obj)) {
 return AppendNdarrayItem(obj);
 }
 const auto list_size = static_cast<int64_t>(PySequence_Size(obj));
 if (ARROW_PREDICT_FALSE(list_size == -1)) {
 RETURN_IF_PYERROR();
 if (!PySequence_Check(obj)) {
 return internal::InvalidType(obj,
 "was not a sequence or recognized null"
 " for conversion to list type");
 }
 int64_t list_size = static_cast<int64_t>(PySequence_Size(obj));
 return value_converter_->AppendMultiple(obj, list_size);
 }

Expand All@@ -658,116 +752,22 @@ class ListConverter : public TypedConverter<TypeClass, ListConverter<TypeClass>>
 bool strict_conversions_;
};

template <typename TypeClass>
template <int NUMPY_TYPE, typename Type>
Status ListConverter<TypeClass>::AppendNdarrayTypedItem(PyArrayObject* arr) {
 using traits = internal::npy_traits<NUMPY_TYPE>;
 using T = typename traits::value_type;
 using ValueBuilderType = typename TypeTraits<Type>::BuilderType;

 const bool null_sentinels_possible =
 // Always treat Numpy's NaT as null
 NUMPY_TYPE == NPY_DATETIME ||
 // Observing pandas's null sentinels
 (from_pandas_ && traits::supports_nulls);

 auto child_builder = checked_cast<ValueBuilderType*>(value_converter_->builder());

 // TODO(wesm): Vector append when not strided
 Ndarray1DIndexer<T> values(arr);
 if (null_sentinels_possible) {
 for (int64_t i = 0; i < values.size(); ++i) {
 if (traits::isnull(values[i])) {
 RETURN_NOT_OK(child_builder->AppendNull());
 } else {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 } else {
 for (int64_t i = 0; i < values.size(); ++i) {
 RETURN_NOT_OK(child_builder->Append(values[i]));
 }
 }
 return Status::OK();
}

// If the value type does not match the expected NumPy dtype, then fall through
// to a slower PySequence-based path
#define LIST_FAST_CASE(TYPE, NUMPY_TYPE, ArrowType) \
 case Type::TYPE: { \
 if (PyArray_DESCR(arr)->type_num != NUMPY_TYPE) { \
 return value_converter_->AppendMultiple(obj, value_length); \
 } \
 return AppendNdarrayTypedItem<NUMPY_TYPE, ArrowType>(arr); \
 }

// Use internal::VisitSequence, fast for NPY_OBJECT but slower otherwise
#define LIST_SLOW_CASE(TYPE) \
 case Type::TYPE: { \
 return value_converter_->AppendMultiple(obj, value_length); \
 }

template <typename TypeClass>
Status ListConverter<TypeClass>::AppendNdarrayItem(PyObject* obj) {
 PyArrayObject* arr = reinterpret_cast<PyArrayObject*>(obj);

 if (PyArray_NDIM(arr) != 1) {
 return Status::Invalid("Can only convert 1-dimensional array values");
 }

 const int64_t value_length = PyArray_SIZE(arr);

 switch (value_type_->id()) {
 LIST_SLOW_CASE(NA)
 LIST_FAST_CASE(UINT8, NPY_UINT8, UInt8Type)
 LIST_FAST_CASE(INT8, NPY_INT8, Int8Type)
 LIST_FAST_CASE(UINT16, NPY_UINT16, UInt16Type)
 LIST_FAST_CASE(INT16, NPY_INT16, Int16Type)
 LIST_FAST_CASE(UINT32, NPY_UINT32, UInt32Type)
 LIST_FAST_CASE(INT32, NPY_INT32, Int32Type)
 LIST_FAST_CASE(UINT64, NPY_UINT64, UInt64Type)
 LIST_FAST_CASE(INT64, NPY_INT64, Int64Type)
 LIST_SLOW_CASE(DATE32)
 LIST_SLOW_CASE(DATE64)
 LIST_SLOW_CASE(TIME32)
 LIST_SLOW_CASE(TIME64)
 LIST_FAST_CASE(TIMESTAMP, NPY_DATETIME, TimestampType)
 LIST_FAST_CASE(HALF_FLOAT, NPY_FLOAT16, HalfFloatType)
 LIST_FAST_CASE(FLOAT, NPY_FLOAT, FloatType)
 LIST_FAST_CASE(DOUBLE, NPY_DOUBLE, DoubleType)
 LIST_SLOW_CASE(BINARY)
 LIST_SLOW_CASE(FIXED_SIZE_BINARY)
 LIST_SLOW_CASE(STRING)
 case Type::LIST: {
 if (PyArray_DESCR(arr)->type_num != NPY_OBJECT) {
 return Status::Invalid(
 "Can only convert list types from NumPy object "
 "array input");
 }
 return internal::VisitSequence(obj, [this](PyObject* item, bool*) {
 return value_converter_->AppendSingleVirtual(item);
 });
 }
 default: {
 return Status::TypeError("Unknown list item type: ", value_type_->ToString());
 }
 }
}

// ----------------------------------------------------------------------
// Convert structs

class StructConverter : public TypedConverter<StructType, StructConverter> {
template <NullCoding null_coding>
class StructConverter
 : public TypedConverter<StructType, StructConverter<null_coding>, null_coding> {
 public:
 explicit StructConverter(bool from_pandas, bool strict_conversions)
 : from_pandas_(from_pandas), strict_conversions_(strict_conversions) {}

 Status Init(ArrayBuilder* builder) {
 builder_ = builder;
 typed_builder_ = checked_cast<StructBuilder*>(builder);
this->builder_ = builder;
this->typed_builder_ = checked_cast<StructBuilder*>(builder);
 const auto& struct_type = checked_cast<const StructType&>(*builder->type());

 num_fields_ = typed_builder_->num_fields();
 num_fields_ = this->typed_builder_->num_fields();
 DCHECK_EQ(num_fields_, struct_type.num_children());

 field_name_list_.reset(PyList_New(num_fields_));
Expand All@@ -781,7 +781,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 std::unique_ptr<SeqConverter> value_converter;
 RETURN_NOT_OK(
 GetConverter(field_type, from_pandas_, strict_conversions_, &value_converter));
 RETURN_NOT_OK(value_converter->Init(typed_builder_->field_builder(i)));
 RETURN_NOT_OK(value_converter->Init(this->typed_builder_->field_builder(i)));
 value_converters_.push_back(std::move(value_converter));

 // Store the field name as a PyObject, for dict matching
Expand All@@ -795,7 +795,7 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 }

 Status AppendItem(PyObject* obj) {
 RETURN_NOT_OK(typed_builder_->Append());
 RETURN_NOT_OK(this->typed_builder_->Append());
 // Note heterogenous sequences are not allowed
 if (ARROW_PREDICT_FALSE(source_kind_ == UNKNOWN)) {
 if (PyDict_Check(obj)) {
Expand All@@ -809,13 +809,15 @@ class StructConverter : public TypedConverter<StructType, StructConverter> {
 } else if (PyTuple_Check(obj) && source_kind_ == TUPLES) {
 return AppendTupleItem(obj);
 } else {
 return Status::TypeError("Expected sequence of dicts or tuples for struct type");
 return internal::InvalidType(obj,
 "was not a dict, tuple, or recognized null value"
 " for conversion to struct type");
 }
 }

 // Append a missing item
 Status AppendNull() {
 RETURN_NOT_OK(typed_builder_->AppendNull());
 RETURN_NOT_OK(this->typed_builder_->AppendNull());
 // Need to also insert a missing item on all child builders
 // (compare with ListConverter)
 for (int i = 0; i < num_fields_; i++) {
Expand DownExpand Up@@ -959,16 +961,36 @@ Status GetConverter(const std::shared_ptr<DataType>& type, bool from_pandas,
 bool strict_conversions, std::unique_ptr<SeqConverter>* out) {
 switch (type->id()) {
 case Type::LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<ListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::LARGE_LIST:
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType>(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::PANDAS_SENTINELS>(
 from_pandas, strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new ListConverter<LargeListType, NullCoding::NONE_ONLY>(from_pandas,
 strict_conversions));
 }
 return Status::OK();
 case Type::STRUCT:
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter(from_pandas, strict_conversions));
 if (from_pandas) {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::PANDAS_SENTINELS>(from_pandas,
 strict_conversions));
 } else {
 *out = std::unique_ptr<SeqConverter>(
 new StructConverter<NullCoding::NONE_ONLY>(from_pandas, strict_conversions));
 }
@wesmwesmSep 5, 2019

Copy link


Copy Markdown

MemberAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Relevant changes
 return Status::OK();
 default:
 break;
Expand Down

4 changes: 2 additions & 2 deletions
docker-compose.yml

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -595,7 +595,7 @@ services:
 # docker-compose build python
 # docker-compose build hdfs-integration
 # docker-compose run hdfs-integration
 image: arrow:hdfs-${HDFS_VERSION:-2.6.5}
 image: arrow:hdfs-${HDFS_VERSION:-2.9.2}
 links:
 - hdfs-namenode:hdfs-namenode
 - hdfs-datanode-1:hdfs-datanode-1
Expand All@@ -608,7 +608,7 @@ services:
 context: .
 dockerfile: integration/hdfs/Dockerfile
 args:
 HDFS_VERSION: ${HDFS_VERSION:-2.6.5}
 HDFS_VERSION: ${HDFS_VERSION:-2.9.2}
 volumes: *ubuntu-volumes

 # TODO(kszucs): pass dask version explicitly as a build argument
Expand Down
Loading