From 6894d6cd0471345d709e2c82f34c605a3028377f Mon Sep 17 00:00:00 2001 From: Adam Lippai Date: Fri, 21 Aug 2026 22:42:20 -0400 Subject: [PATCH 1/7] GH-42018: [Python] Add NumPy StringDType support --- .../pyarrow/src/arrow/python/numpy_convert.cc | 1 + .../src/arrow/python/numpy_to_arrow.cc | 56 +++++++++++++++++ python/pyarrow/tests/test_array.py | 63 +++++++++++++++++++ 3 files changed, 120 insertions(+) diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index 6e59835286d9..882e9e47e5b1 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -151,6 +151,7 @@ Result> NumPyDtypeToArrow(PyArray_Descr* descr) { TO_ARROW_TYPE_CASE(FLOAT64, float64); TO_ARROW_TYPE_CASE(STRING, binary); TO_ARROW_TYPE_CASE(UNICODE, utf8); + TO_ARROW_TYPE_CASE(VSTRING, utf8); case NPY_DATETIME: { auto date_dtype = reinterpret_cast(PyDataType_C_METADATA(descr)); diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 5647e895d0f7..64dfe2ac30fc 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -295,6 +296,9 @@ class NumPyConverter { template Status VisitString(T* builder); + template + Status VisitStringDType(T* builder); + Status TypeNotImplemented(std::string type_name) { return Status::NotImplemented("NumPyConverter doesn't implement <", type_name, "> conversion. "); @@ -342,6 +346,11 @@ Status NumPyConverter::Convert() { return Status::Invalid("Must pass data type for non-object arrays"); } + if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) { + return Status::TypeError( + "NumPy StringDType can only be converted to Arrow string types"); + } + // Visit the type to perform conversion return VisitTypeInline(*type_, this); } @@ -697,8 +706,55 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder } // namespace +template +Status NumPyConverter::VisitStringDType(T* builder) { + const char* data = PyArray_BYTES(arr_); + auto* allocator = + NpyString_acquire_allocator(reinterpret_cast(dtype_)); + if (allocator == nullptr) { + return Status::Invalid("Failed to acquire NumPy StringDType allocator"); + } + std::unique_ptr + allocator_guard(allocator, &NpyString_release_allocator); + + npy_static_string value = {0, nullptr}; + const auto append_value = [&](const char* item) -> Status { + const auto* packed = reinterpret_cast(item); + const int is_null = NpyString_load(allocator, packed, &value); + if (is_null == -1) { + return Status::Invalid("Failed to load NumPy StringDType value"); + } + if (is_null) { + return builder->AppendNull(); + } + return builder->Append(std::string_view(value.buf, value.size)); + }; + + if (mask_ != nullptr) { + Ndarray1DIndexer mask_values(mask_); + for (int64_t i = 0; i < length_; ++i) { + if (mask_values[i]) { + RETURN_NOT_OK(builder->AppendNull()); + } else { + RETURN_NOT_OK(append_value(data)); + } + data += stride_; + } + } else { + for (int64_t i = 0; i < length_; ++i) { + RETURN_NOT_OK(append_value(data)); + data += stride_; + } + } + return Status::OK(); +} + template Status NumPyConverter::VisitString(T* builder) { + if (dtype_->type_num == NPY_VSTRING) { + return VisitStringDType(builder); + } + auto data = reinterpret_cast(PyArray_DATA(arr_)); char numpy_byteorder = dtype_->byteorder; diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index a1e3616c9cea..8c27ef9a45ca 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -2925,6 +2925,69 @@ def test_array_from_numpy_unicode(string_type): assert arrow_arr.equals(expected) +@pytest.fixture +def numpy_string_dtype(): + dtypes = pytest.importorskip("numpy.dtypes") + return dtypes.StringDType + + +@pytest.mark.numpy +@pytest.mark.parametrize('string_type', [ + None, + pa.string(), + pa.large_string(), + pa.string_view()]) +def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type): + values = [ + "short", + "a" * 100, + "b" * 300, + "árvíztűrő tükörfúrógép 🥐 你好", + "🥐" * 200, + "", + ] + arr = np.array(values, dtype=numpy_string_dtype()) + + arrow_arr = pa.array(arr, type=string_type) + + arrow_arr.validate(full=True) + assert arrow_arr.type == (string_type or pa.string()) + assert arrow_arr.to_pylist() == arr.tolist() + + strided = np.array(list(itertools.chain.from_iterable( + zip(values, itertools.repeat("skip")))), + dtype=numpy_string_dtype())[::2] + arrow_arr = pa.array(strided, type=string_type) + arrow_arr.validate(full=True) + assert arrow_arr.to_pylist() == values + + +@pytest.mark.numpy +@pytest.mark.parametrize('na_object', [None, "__placeholder__"]) +def test_array_from_numpy_string_dtype_nulls_and_mask( + numpy_string_dtype, na_object): + arr = np.array(["some", na_object, "strings"], + dtype=numpy_string_dtype(na_object=na_object)) + + arrow_arr = pa.array(arr) + arrow_arr.validate(full=True) + assert arrow_arr.to_pylist() == ["some", None, "strings"] + + mask = np.array([False, False, True]) + arrow_arr = pa.array(arr, mask=mask) + arrow_arr.validate(full=True) + assert arrow_arr.to_pylist() == ["some", None, None] + + +@pytest.mark.numpy +def test_array_from_numpy_string_dtype_rejects_non_string_type( + numpy_string_dtype): + arr = np.array(["some", "strings"], dtype=numpy_string_dtype()) + + with pytest.raises(TypeError, match="can only be converted"): + pa.array(arr, type=pa.binary()) + + @pytest.mark.numpy def test_array_string_from_non_string(): # ARROW-5682 - when converting to string raise on non string-like dtype From 917b83684abd333bdbe6023e8933d8a70dd2e93b Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Sep 2026 11:21:16 -0600 Subject: [PATCH 2/7] GH-42018: [Python] Simplify the StringDType conversion loop --- .../src/arrow/python/numpy_to_arrow.cc | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 64dfe2ac30fc..8367563ab2d5 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -708,43 +708,34 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder template Status NumPyConverter::VisitStringDType(T* builder) { + auto* descr = reinterpret_cast(dtype_); const char* data = PyArray_BYTES(arr_); - auto* allocator = - NpyString_acquire_allocator(reinterpret_cast(dtype_)); - if (allocator == nullptr) { - return Status::Invalid("Failed to acquire NumPy StringDType allocator"); + Ndarray1DIndexer mask_values; + if (mask_ != nullptr) { + mask_values = Ndarray1DIndexer(mask_); } + + // NumPy takes the GIL while holding this lock, so never take it with the GIL held + auto* allocator = NpyString_acquire_allocator(descr); std::unique_ptr allocator_guard(allocator, &NpyString_release_allocator); npy_static_string value = {0, nullptr}; - const auto append_value = [&](const char* item) -> Status { - const auto* packed = reinterpret_cast(item); + for (int64_t i = 0; i < length_; ++i, data += stride_) { + if (mask_ != nullptr && mask_values[i]) { + RETURN_NOT_OK(builder->AppendNull()); + continue; + } + const auto* packed = reinterpret_cast(data); const int is_null = NpyString_load(allocator, packed, &value); if (is_null == -1) { return Status::Invalid("Failed to load NumPy StringDType value"); } if (is_null) { - return builder->AppendNull(); - } - return builder->Append(std::string_view(value.buf, value.size)); - }; - - if (mask_ != nullptr) { - Ndarray1DIndexer mask_values(mask_); - for (int64_t i = 0; i < length_; ++i) { - if (mask_values[i]) { - RETURN_NOT_OK(builder->AppendNull()); - } else { - RETURN_NOT_OK(append_value(data)); - } - data += stride_; - } - } else { - for (int64_t i = 0; i < length_; ++i) { - RETURN_NOT_OK(append_value(data)); - data += stride_; + RETURN_NOT_OK(builder->AppendNull()); + continue; } + RETURN_NOT_OK(builder->Append(std::string_view(value.buf, value.size))); } return Status::OK(); } @@ -752,6 +743,7 @@ Status NumPyConverter::VisitStringDType(T* builder) { template Status NumPyConverter::VisitString(T* builder) { if (dtype_->type_num == NPY_VSTRING) { + // Acquires a lock, so must stay ahead of the gil_lock below return VisitStringDType(builder); } From 3732c422623ceffbc2487afbade3aa44f3c02fd6 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Sep 2026 11:21:16 -0600 Subject: [PATCH 3/7] GH-42018: [Python] Convert StringDType nulls with a string na_object to that string --- .../src/arrow/python/numpy_to_arrow.cc | 21 +++++++++++++++++-- python/pyarrow/tests/test_array.py | 14 ++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 8367563ab2d5..f5b8a7f94e43 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -706,9 +706,22 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder } // namespace +namespace { + +std::string_view ToStringView(const npy_static_string& value) { + return value.buf == nullptr ? std::string_view() + : std::string_view(value.buf, value.size); +} + +} // namespace + template Status NumPyConverter::VisitStringDType(T* builder) { auto* descr = reinterpret_cast(dtype_); + // NumPy reports a null entry as the na_object itself when that is a string + const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na; + const std::string_view null_string = ToStringView(descr->default_string); + const char* data = PyArray_BYTES(arr_); Ndarray1DIndexer mask_values; if (mask_ != nullptr) { @@ -732,10 +745,14 @@ Status NumPyConverter::VisitStringDType(T* builder) { return Status::Invalid("Failed to load NumPy StringDType value"); } if (is_null) { - RETURN_NOT_OK(builder->AppendNull()); + if (null_is_missing) { + RETURN_NOT_OK(builder->AppendNull()); + } else { + RETURN_NOT_OK(builder->Append(null_string)); + } continue; } - RETURN_NOT_OK(builder->Append(std::string_view(value.buf, value.size))); + RETURN_NOT_OK(builder->Append(ToStringView(value))); } return Status::OK(); } diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 8c27ef9a45ca..bc925c22791b 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -2963,20 +2963,24 @@ def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type): @pytest.mark.numpy -@pytest.mark.parametrize('na_object', [None, "__placeholder__"]) -def test_array_from_numpy_string_dtype_nulls_and_mask( - numpy_string_dtype, na_object): +@pytest.mark.parametrize('na_object, expected', [ + (None, None), + (float("nan"), None), + ("__placeholder__", "__placeholder__"), +]) +def test_array_from_numpy_string_dtype_na_object( + numpy_string_dtype, na_object, expected): arr = np.array(["some", na_object, "strings"], dtype=numpy_string_dtype(na_object=na_object)) arrow_arr = pa.array(arr) arrow_arr.validate(full=True) - assert arrow_arr.to_pylist() == ["some", None, "strings"] + assert arrow_arr.to_pylist() == ["some", expected, "strings"] mask = np.array([False, False, True]) arrow_arr = pa.array(arr, mask=mask) arrow_arr.validate(full=True) - assert arrow_arr.to_pylist() == ["some", None, None] + assert arrow_arr.to_pylist() == ["some", expected, None] @pytest.mark.numpy From b8de7b08803f7bcc3cc6cea5d24c51d651efe2ce Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Sep 2026 09:25:53 -0600 Subject: [PATCH 4/7] GH-42018: [Python] Name the requested type when rejecting a StringDType conversion --- python/pyarrow/src/arrow/python/numpy_to_arrow.cc | 3 ++- python/pyarrow/tests/test_array.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index f5b8a7f94e43..86229f238fd8 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -348,7 +348,8 @@ Status NumPyConverter::Convert() { if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) { return Status::TypeError( - "NumPy StringDType can only be converted to Arrow string types"); + "NumPy StringDType can only be converted to Arrow string types, got ", + type_->ToString()); } // Visit the type to perform conversion diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index bc925c22791b..9e0de84429dd 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -2988,7 +2988,7 @@ def test_array_from_numpy_string_dtype_rejects_non_string_type( numpy_string_dtype): arr = np.array(["some", "strings"], dtype=numpy_string_dtype()) - with pytest.raises(TypeError, match="can only be converted"): + with pytest.raises(TypeError, match="can only be converted.*got binary"): pa.array(arr, type=pa.binary()) From 4a70795c07d0d2823db1bd9b9431aa6c969ff612 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Sep 2026 09:25:53 -0600 Subject: [PATCH 5/7] GH-42018: [Python] Test list inference for StringDType arrays --- python/pyarrow/tests/test_array.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 9e0de84429dd..e9d74e640e35 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -2992,6 +2992,17 @@ def test_array_from_numpy_string_dtype_rejects_non_string_type( pa.array(arr, type=pa.binary()) +@pytest.mark.numpy +def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype): + values = [["a", "bb"], ["ccc"]] + arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values] + + result = pa.array(arrays) + + assert result.type == pa.list_(pa.string()) + assert result.to_pylist() == values + + @pytest.mark.numpy def test_array_string_from_non_string(): # ARROW-5682 - when converting to string raise on non string-like dtype From a490f48f13a550a1e1ac32e18ba06d515230be34 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Sep 2026 09:25:53 -0600 Subject: [PATCH 6/7] GH-42018: [Python][Docs] Document NumPy StringDType conversion --- docs/source/python/numpy.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/source/python/numpy.rst b/docs/source/python/numpy.rst index 07a6aa803f8a..10a70159c8e4 100644 --- a/docs/source/python/numpy.rst +++ b/docs/source/python/numpy.rst @@ -51,7 +51,28 @@ factory function. ] Converting from NumPy supports a wide range of input dtypes, including -structured dtypes or strings. +structured dtypes and both fixed-width (``S`` and ``U``) and variable-width +(:class:`numpy.dtypes.StringDType`) strings. + +A ``StringDType`` array converts to :func:`~pyarrow.string` unless +:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with +``type``. Missing entries become nulls: + +.. code-block:: python + + >>> dtype = np.dtypes.StringDType(na_object=np.nan) + >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype)) + >>> arr + + [ + "some", + null, + "strings" + ] + +When the ``na_object`` is a string, NumPy treats missing entries as that string +in every operation, and so does the conversion. Pass ``mask`` to mark values as +null explicitly. Arrow to NumPy -------------- From a040f7af50eaac67d7dccb4d9853025246046220 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 3 Sep 2026 12:22:11 -0600 Subject: [PATCH 7/7] GH-42018: [Python] Clean up error message and comments --- python/pyarrow/src/arrow/python/numpy_to_arrow.cc | 10 +++++----- python/pyarrow/tests/test_array.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 86229f238fd8..d4e859d1c707 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -347,9 +347,8 @@ Status NumPyConverter::Convert() { } if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) { - return Status::TypeError( - "NumPy StringDType can only be converted to Arrow string types, got ", - type_->ToString()); + return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ", + type_->ToString()); } // Visit the type to perform conversion @@ -719,7 +718,7 @@ std::string_view ToStringView(const npy_static_string& value) { template Status NumPyConverter::VisitStringDType(T* builder) { auto* descr = reinterpret_cast(dtype_); - // NumPy reports a null entry as the na_object itself when that is a string + // Use the na_object itself when na_object is a string const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na; const std::string_view null_string = ToStringView(descr->default_string); @@ -729,7 +728,8 @@ Status NumPyConverter::VisitStringDType(T* builder) { mask_values = Ndarray1DIndexer(mask_); } - // NumPy takes the GIL while holding this lock, so never take it with the GIL held + // Acquiring the allocator lock, so do not acquire the GIL or lock other + // mutexes below or risk deadlocks auto* allocator = NpyString_acquire_allocator(descr); std::unique_ptr allocator_guard(allocator, &NpyString_release_allocator); diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index e9d74e640e35..062248d12696 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -2988,7 +2988,8 @@ def test_array_from_numpy_string_dtype_rejects_non_string_type( numpy_string_dtype): arr = np.array(["some", "strings"], dtype=numpy_string_dtype()) - with pytest.raises(TypeError, match="can only be converted.*got binary"): + msg = "Expected an Arrow string type.*got binary" + with pytest.raises(TypeError, match=msg): pa.array(arr, type=pa.binary())