From b141fb31030fdf2f31df3c4d65d148f9d9e96556 Mon Sep 17 00:00:00 2001 From: gbranaa4-hue Date: Sun, 6 Sep 2026 04:31:49 -0700 Subject: [PATCH] GH-39914: [Python] Fall back to schema field type for unparseable complex dtype metadata pd.read_parquet() (and Table.to_pandas() generally) raises TypeError when a column's pandas metadata records a complex ArrowDtype (list, struct, dictionary, ...) as its 'numpy_type', e.g. "list[pyarrow]". _get_extension_dtypes() passes this string straight to pandas.api.types.pandas_dtype(), which cannot parse nested Arrow type syntax and raises TypeError -- so any DataFrame that has such a column can never round-trip through parquet with the default (numpy) backend, and this has stayed broken since 2023. GH-39914 previously fixed the case where an explicit types_mapper / dtype_backend="pyarrow" is passed, by resolving those columns earlier so this code path is skipped entirely. The default backend, and any metadata-only path with no types_mapper, still hit the crash. Fix: when pandas_dtype(dtype) raises TypeError, fall back to building the ArrowDtype directly from the schema's actual field type instead of giving up. The schema always has the precise type available regardless of whether the metadata string can be parsed back. Verified against pandas 3.0.4 + pyarrow 24.0.0: list, struct, and dictionary ArrowDtype columns now round-trip through to_pandas() and parquet with no types_mapper/dtype_backend needed, a double write/read/write/read round-trip of a struct column still works, and plain numpy dtypes plus simple extension dtypes (Int64) are unaffected. Fixes pandas-dev/pandas#53011 Co-Authored-By: Claude Sonnet 5 --- python/pyarrow/pandas_compat.py | 16 ++++++++++++++- python/pyarrow/tests/test_pandas.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 36d336d5135..cd772a7b5e7 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -902,7 +902,21 @@ def _get_extension_dtypes(table, columns_metadata, types_mapper, options, catego if name not in ext_columns and dtype not in _pandas_supported_numpy_types: # pandas_dtype is expensive, so avoid doing this for types # that are certainly numpy dtypes - pandas_dtype = _pandas_api.pandas_dtype(dtype) + try: + pandas_dtype = _pandas_api.pandas_dtype(dtype) + except TypeError: + # Complex/nested Arrow types (list, struct, dictionary, ...) + # serialize to a 'numpy_type' string (e.g. + # "list[pyarrow]") that pandas_dtype() cannot + # parse back. Fall back to building the ArrowDtype directly + # from the schema's actual field type instead of giving up + # on round-tripping the dtype entirely. + # See GH-39914 / pandas-dev/pandas#53011. + try: + field = table.schema.field(name) + except KeyError: + continue + pandas_dtype = _pandas_api.pd.ArrowDtype(field.type) if isinstance(pandas_dtype, _pandas_api.extension_dtype): if isinstance(pandas_dtype, _pandas_api.pd.StringDtype): # when the metadata indicate to use the string dtype, diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index dd20a0aa977..8f9d198f4a8 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4566,6 +4566,37 @@ def test_to_pandas_extension_dtypes_mapping_complex_type(): pd.testing.assert_frame_equal(df0, df1) +def test_to_pandas_extension_dtypes_mapping_complex_type_no_types_mapper(): + # GH-39914: without an explicit types_mapper, the pandas metadata + # embedded by from_pandas() stores a 'numpy_type' string like + # "list[pyarrow]" for complex/nested ArrowDtype columns. + # pandas_dtype() cannot parse that string back, so to_pandas() (and, by + # extension, pd.read_parquet() without dtype_backend="pyarrow") used to + # raise a TypeError instead of falling back to the schema's actual field + # type. See pandas-dev/pandas#53011. + list_type = pd.ArrowDtype(pa.list_(pa.string())) + df0 = pd.DataFrame({ + "a": pd.Series([["x"], ["x", "y"]], dtype=list_type), + }) + + table = pa.Table.from_pandas(df0) + df1 = table.to_pandas() + pd.testing.assert_frame_equal(df0, df1) + + struct_type = pd.ArrowDtype( + pa.struct([pa.field("bar", pa.bool_()), pa.field("baz", pa.float32())]) + ) + df2 = pd.DataFrame({ + "a": pd.Series( + [{"bar": True, "baz": 1.0}, {"bar": False, "baz": None}], + dtype=struct_type, + ), + }) + table2 = pa.Table.from_pandas(df2) + df3 = table2.to_pandas() + pd.testing.assert_frame_equal(df2, df3) + + def test_array_to_pandas(): for arr in [pd.period_range("2012-01-01", periods=3, freq="D").array, pd.interval_range(1, 4).array]: