Skip to content

BUG: Fix GroupBy aggregate coersion of outputs inconsistency for pyarrow dtypes #61640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v3.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -829,6 +829,7 @@ Groupby/resample/rolling
- Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` were not keeping the index name when the index had :class:`ArrowDtype` timestamp dtype (:issue:`61222`)
- Bug in :meth:`DataFrame.resample` changing index type to :class:`MultiIndex` when the dataframe is empty and using an upsample method (:issue:`55572`)
- Bug in :meth:`DataFrameGroupBy.agg` that raises ``AttributeError`` when there is dictionary input and duplicated columns, instead of returning a DataFrame with the aggregation of all duplicate columns. (:issue:`55041`)
- Bug in :meth:`DataFrameGroupBy.agg` when applied to columns with :class:`ArrowDtype`, where pandas attempted to cast the result back to the original dtype (:issue:`61636`)
- Bug in :meth:`DataFrameGroupBy.agg` where applying a user-defined function to an empty DataFrame returned a Series instead of an empty DataFrame. (:issue:`61503`)
- Bug in :meth:`DataFrameGroupBy.apply` and :meth:`SeriesGroupBy.apply` for empty data frame with ``group_keys=False`` still creating output index using group keys. (:issue:`60471`)
- Bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
Expand Down
15 changes: 15 additions & 0 deletions pandas/core/arrays/arrow/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ def floordiv_compat(
ArrayLike,
AxisInt,
Dtype,
DtypeObj,
FillnaOptions,
InterpolateOptions,
Iterator,
Expand Down Expand Up @@ -313,6 +314,20 @@ def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
)
self._dtype = ArrowDtype(self._pa_array.type)

@classmethod
def _from_scalars(cls, scalars, dtype: DtypeObj) -> Self:
inferred_dtype = lib.infer_dtype(scalars, skipna=True)
try:
pa_array = cls._from_sequence(scalars, dtype=dtype)
except pa.ArrowNotImplementedError as err:
# _from_scalars should only raise ValueError or TypeError.
raise ValueError from err

same_dtype = lib.infer_dtype(pa_array, skipna=True) == inferred_dtype
if not same_dtype:
raise ValueError
return pa_array

@classmethod
def _from_sequence(
cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
Expand Down
7 changes: 7 additions & 0 deletions pandas/core/arrays/string_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
from pandas._typing import (
ArrayLike,
Dtype,
DtypeObj,
NpDtype,
Self,
npt,
Expand Down Expand Up @@ -180,6 +181,12 @@ def __len__(self) -> int:
"""
return len(self._pa_array)

@classmethod
def _from_scalars(cls, scalars, dtype: DtypeObj) -> Self:
if lib.infer_dtype(scalars, skipna=True) not in ["string", "empty"]:
raise ValueError
return cls._from_sequence(scalars, dtype=dtype)

@classmethod
def _from_sequence(
cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
Expand Down
21 changes: 21 additions & 0 deletions pandas/tests/extension/test_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3257,6 +3257,27 @@ def test_groupby_count_return_arrow_dtype(data_missing):
tm.assert_frame_equal(result, expected)


@pytest.mark.parametrize(
"func, func_dtype",
[
[lambda x: x.to_dict(), "object"],
[lambda x: 1, "int64"],
[lambda x: "s", ArrowDtype(pa.string())],
],
)
def test_groupby_aggregate_coersion(func, func_dtype):
# GH 61636
df = pd.DataFrame(
{
"b": pd.array([0, 1]),
"c": pd.array(["X", "Y"], dtype=ArrowDtype(pa.string())),
},
index=pd.Index(["A", "B"], name="a"),
)
result = df.groupby("b").agg(func)
assert result["c"].dtype == func_dtype


def test_fixed_size_list():
# GH#55000
ser = pd.Series(
Expand Down
Loading