Skip to content

ENH: Allow ArrowDtype(pa.string()) to be compatable with str methods #50325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 30 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
e3780b0
start adding string methods
mroeschke Dec 7, 2022
2c8f389
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 8, 2022
fa83634
Finish adding methods
mroeschke Dec 9, 2022
ffe33b6
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 13, 2022
f242d81
Add some tests that are failing
mroeschke Dec 14, 2022
fd22cba
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 14, 2022
def31cc
More test adjustment
mroeschke Dec 15, 2022
a4e315f
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 16, 2022
a7adc92
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 17, 2022
e6f5ceb
All tests passing
mroeschke Dec 17, 2022
63ebc09
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 17, 2022
adb70b8
Add whatsnew
mroeschke Dec 17, 2022
0af8223
Remove commented out code
mroeschke Dec 17, 2022
5b22ccd
Add whatsnew note
mroeschke Dec 17, 2022
f4e6987
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 19, 2022
05964ee
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 19, 2022
19ae7b5
Fix bug
mroeschke Dec 19, 2022
b97cc9a
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 20, 2022
40fbe97
Fix methods for inherited ArrowStringARray
mroeschke Dec 20, 2022
edb4258
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 21, 2022
6c00095
Fix some library compat
mroeschke Dec 21, 2022
7e419f7
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 23, 2022
078e387
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Dec 28, 2022
ffee70a
Fix one typing
mroeschke Dec 28, 2022
7c57a8a
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Jan 6, 2023
b0ec827
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Jan 16, 2023
fcabdb6
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Jan 19, 2023
6261d57
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Jan 20, 2023
5a4253b
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Jan 23, 2023
1acedec
Merge remote-tracking branch 'upstream/main' into enh/arrowdtype_str_…
mroeschke Jan 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v2.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,7 @@ Other enhancements
- Added :meth:`Index.infer_objects` analogous to :meth:`Series.infer_objects` (:issue:`50034`)
- Added ``copy`` parameter to :meth:`Series.infer_objects` and :meth:`DataFrame.infer_objects`, passing ``False`` will avoid making copies for series or columns that are already non-object or where no better dtype can be inferred (:issue:`50096`)
- :meth:`DataFrame.plot.hist` now recognizes ``xlabel`` and ``ylabel`` arguments (:issue:`49793`)
- Added support for ``str`` accessor methods when using ``pd.ArrowDtype(pyarrow.string())`` (:issue:`50325`)
- :meth:`Series.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`48304`)
- :meth:`Series.dropna` and :meth:`DataFrame.dropna` has gained ``ignore_index`` keyword to reset index (:issue:`31725`)
- Improved error message in :func:`to_datetime` for non-ISO8601 formats, informing users about the position of the first error (:issue:`50361`)
Expand Down
122 changes: 122 additions & 0 deletions pandas/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
Index,
MultiIndex,
)
from pandas.core.strings.accessor import StringMethods

try:
import pyarrow as pa
Expand Down Expand Up @@ -1947,3 +1948,124 @@ def warsaw(request):
tzinfo for Europe/Warsaw using pytz, dateutil, or zoneinfo.
"""
return request.param


_any_string_method = [
("cat", (), {"sep": ","}),
("cat", (Series(list("zyx")),), {"sep": ",", "join": "left"}),
("center", (10,), {}),
("contains", ("a",), {}),
("count", ("a",), {}),
("decode", ("UTF-8",), {}),
("encode", ("UTF-8",), {}),
("endswith", ("a",), {}),
("endswith", ("a",), {"na": True}),
("endswith", ("a",), {"na": False}),
("extract", ("([a-z]*)",), {"expand": False}),
("extract", ("([a-z]*)",), {"expand": True}),
("extractall", ("([a-z]*)",), {}),
("find", ("a",), {}),
("findall", ("a",), {}),
("get", (0,), {}),
# because "index" (and "rindex") fail intentionally
# if the string is not found, search only for empty string
("index", ("",), {}),
("join", (",",), {}),
("ljust", (10,), {}),
("match", ("a",), {}),
("fullmatch", ("a",), {}),
("normalize", ("NFC",), {}),
("pad", (10,), {}),
("partition", (" ",), {"expand": False}),
("partition", (" ",), {"expand": True}),
("repeat", (3,), {}),
("replace", ("a", "z"), {}),
("rfind", ("a",), {}),
("rindex", ("",), {}),
("rjust", (10,), {}),
("rpartition", (" ",), {"expand": False}),
("rpartition", (" ",), {"expand": True}),
("slice", (0, 1), {}),
("slice_replace", (0, 1, "z"), {}),
("split", (" ",), {"expand": False}),
# ("split", (" ",), {"expand": True}),
("startswith", ("a",), {}),
("startswith", ("a",), {"na": True}),
("startswith", ("a",), {"na": False}),
("removeprefix", ("a",), {}),
("removesuffix", ("a",), {}),
# translating unicode points of "a" to "d"
("translate", ({97: 100},), {}),
("wrap", (2,), {}),
("zfill", (10,), {}),
] + list(
zip(
[
# methods without positional arguments: zip with empty tuple and empty dict
"capitalize",
"cat",
"get_dummies",
"isalnum",
"isalpha",
"isdecimal",
"isdigit",
"islower",
"isnumeric",
"isspace",
"istitle",
"isupper",
"len",
"lower",
"lstrip",
"partition",
"rpartition",
"rsplit",
"rstrip",
"slice",
"slice_replace",
"split",
"strip",
"swapcase",
"title",
"upper",
"casefold",
],
[()] * 100,
[{}] * 100,
)
)
ids, _, _ = zip(*_any_string_method) # use method name as fixture-id
missing_methods = {f for f in dir(StringMethods) if not f.startswith("_")} - set(ids)

# test that the above list captures all methods of StringMethods
assert not missing_methods


@pytest.fixture(params=_any_string_method, ids=ids)
def any_string_method(request):
"""
Fixture for all public methods of `StringMethods`

This fixture returns a tuple of the method name and sample arguments
necessary to call the method.

Returns
-------
method_name : str
The name of the method in `StringMethods`
args : tuple
Sample values for the positional arguments
kwargs : dict
Sample values for the keyword arguments

Examples
--------
>>> def test_something(any_string_method):
... s = Series(['a', 'b', np.nan, 'd'])
...
... method_name, args, kwargs = any_string_method
... method = getattr(s.str, method_name)
... # will not raise
... method(*args, **kwargs)
"""
return request.param
Loading