GH-42018: [Python] Add NumPy StringDType to Arrow conversion

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype

docs/source/python/numpy.rst

@@ -51,7 +51,28 @@ factory function.
  ]
 
 Converting from NumPy supports a wide range of input dtypes, including
-structured dtypes or strings.
+structured dtypes and both fixed-width (``S`` and ``U``) and variable-width
+(:class:`numpy.dtypes.StringDType`) strings.
+
+A ``StringDType`` array converts to :func:`~pyarrow.string` unless
+:func:`~pyarrow.large_string` or :func:`~pyarrow.string_view` is requested with
+``type``. Missing entries become nulls:
+
+.. code-block:: python
+
+ >>> dtype = np.dtypes.StringDType(na_object=np.nan)
+ >>> arr = pa.array(np.array(["some", np.nan, "strings"], dtype=dtype))
+ >>> arr
+ <pyarrow.lib.StringArray object at ...>
+ [
+ "some",
+ null,
+ "strings"
+ ]
+
+When the ``na_object`` is a string, NumPy treats missing entries as that string
+in every operation, and so does the conversion. Pass ``mask`` to mark values as
+null explicitly.
 
 Arrow to NumPy
 --------------

python/pyarrow/src/arrow/python/numpy_convert.cc

@@ -151,6 +151,7 @@ Result<std::shared_ptr<DataType>> NumPyDtypeToArrow(PyArray_Descr* descr) {
  TO_ARROW_TYPE_CASE(FLOAT64, float64);
  TO_ARROW_TYPE_CASE(STRING, binary);
  TO_ARROW_TYPE_CASE(UNICODE, utf8);
+ TO_ARROW_TYPE_CASE(VSTRING, utf8);
  case NPY_DATETIME: {
  auto date_dtype =
  reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(PyDataType_C_METADATA(descr));

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -27,6 +27,7 @@
 #include <limits>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -295,6 +296,9 @@ class NumPyConverter {
  template <typename T>
  Status VisitString(T* builder);
 
+ template <typename T>
+ Status VisitStringDType(T* builder);
+
  Status TypeNotImplemented(std::string type_name) {
  return Status::NotImplemented("NumPyConverter doesn't implement <", type_name,
  "> conversion. ");
@@ -342,6 +346,11 @@ Status NumPyConverter::Convert() {
  return Status::Invalid("Must pass data type for non-object arrays");
  }
 
+ if (dtype_->type_num == NPY_VSTRING && !is_string_or_string_view(type_->id())) {
+ return Status::TypeError("Expected an Arrow string type for NumPy StringDType, got ",
+ type_->ToString());
+ }
+
  // Visit the type to perform conversion
  return VisitTypeInline(*type_, this);
 }
@@ -697,8 +706,65 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
 
 } // namespace
 
+namespace {
+
+std::string_view ToStringView(const npy_static_string& value) {
+ return value.buf == nullptr ? std::string_view()
+ : std::string_view(value.buf, value.size);
+}
+
+} // namespace
+
+template <typename T>
+Status NumPyConverter::VisitStringDType(T* builder) {
+ auto* descr = reinterpret_cast<PyArray_StringDTypeObject*>(dtype_);
+ // Use the na_object itself when na_object is a string
+ const bool null_is_missing = descr->na_object != nullptr && !descr->has_string_na;
+ const std::string_view null_string = ToStringView(descr->default_string);
+
+ const char* data = PyArray_BYTES(arr_);
+ Ndarray1DIndexer<uint8_t> mask_values;
+ if (mask_ != nullptr) {
+ mask_values = Ndarray1DIndexer<uint8_t>(mask_);
+ }
+
+ // Acquiring the allocator lock, so do not acquire the GIL or lock other
+ // mutexes below or risk deadlocks
+ auto* allocator = NpyString_acquire_allocator(descr);
+ std::unique_ptr<npy_string_allocator, decltype(&NpyString_release_allocator)>
+ allocator_guard(allocator, &NpyString_release_allocator);
+
+ npy_static_string value = {0, nullptr};
+ for (int64_t i = 0; i < length_; ++i, data += stride_) {
+ if (mask_ != nullptr && mask_values[i]) {
+ RETURN_NOT_OK(builder->AppendNull());
+ continue;
+ }
+ const auto* packed = reinterpret_cast<const npy_packed_static_string*>(data);
+ const int is_null = NpyString_load(allocator, packed, &value);
+ if (is_null == -1) {
+ return Status::Invalid("Failed to load NumPy StringDType value");
+ }
+ if (is_null) {
+ if (null_is_missing) {
+ RETURN_NOT_OK(builder->AppendNull());
+ } else {
+ RETURN_NOT_OK(builder->Append(null_string));
+ }
+ continue;
+ }
+ RETURN_NOT_OK(builder->Append(ToStringView(value)));
+ }
+ return Status::OK();
+}
+
 template <typename T>
 Status NumPyConverter::VisitString(T* builder) {
+ if (dtype_->type_num == NPY_VSTRING) {
+ // Acquires a lock, so must stay ahead of the gil_lock below
+ return VisitStringDType(builder);
+ }
+
  auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
 
  char numpy_byteorder = dtype_->byteorder;

python/pyarrow/tests/test_array.py

@@ -2925,6 +2925,85 @@ def test_array_from_numpy_unicode(string_type):
  assert arrow_arr.equals(expected)
 
 
+@pytest.fixture
+def numpy_string_dtype():
+ dtypes = pytest.importorskip("numpy.dtypes")
+ return dtypes.StringDType
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('string_type', [
+ None,
+ pa.string(),
+ pa.large_string(),
+ pa.string_view()])
+def test_array_from_numpy_string_dtype(numpy_string_dtype, string_type):
+ values = [
+ "short",
+ "a" * 100,
+ "b" * 300,
+ "árvíztűrő tükörfúrógép 🥐 你好",
+ "🥐" * 200,
+ "",
+ ]
+ arr = np.array(values, dtype=numpy_string_dtype())
+
+ arrow_arr = pa.array(arr, type=string_type)
+
+ arrow_arr.validate(full=True)
+ assert arrow_arr.type == (string_type or pa.string())
+ assert arrow_arr.to_pylist() == arr.tolist()
+
+ strided = np.array(list(itertools.chain.from_iterable(
+ zip(values, itertools.repeat("skip")))),
+ dtype=numpy_string_dtype())[::2]
+ arrow_arr = pa.array(strided, type=string_type)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == values
+
+
+@pytest.mark.numpy
+@pytest.mark.parametrize('na_object, expected', [
+ (None, None),
+ (float("nan"), None),
+ ("__placeholder__", "__placeholder__"),
+])
+def test_array_from_numpy_string_dtype_na_object(
+ numpy_string_dtype, na_object, expected):
+ arr = np.array(["some", na_object, "strings"],
+ dtype=numpy_string_dtype(na_object=na_object))
+
+ arrow_arr = pa.array(arr)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, "strings"]
+
+ mask = np.array([False, False, True])
+ arrow_arr = pa.array(arr, mask=mask)
+ arrow_arr.validate(full=True)
+ assert arrow_arr.to_pylist() == ["some", expected, None]
+
+
+@pytest.mark.numpy
+def test_array_from_numpy_string_dtype_rejects_non_string_type(
+ numpy_string_dtype):
+ arr = np.array(["some", "strings"], dtype=numpy_string_dtype())
+
+ msg = "Expected an Arrow string type.*got binary"
+ with pytest.raises(TypeError, match=msg):
+ pa.array(arr, type=pa.binary())
+
+
+@pytest.mark.numpy
+def test_array_from_list_of_numpy_string_dtype_arrays(numpy_string_dtype):
+ values = [["a", "bb"], ["ccc"]]
+ arrays = [np.array(v, dtype=numpy_string_dtype()) for v in values]
+
+ result = pa.array(arrays)
+
+ assert result.type == pa.list_(pa.string())
+ assert result.to_pylist() == values
+
+
 @pytest.mark.numpy
 def test_array_string_from_non_string():
  # ARROW-5682 - when converting to string raise on non string-like dtype