Skip to content

Commit fa01870

Browse files
authored
BUG: ArrowDtype.numpy_dtype returning ns units for non-nano timestamp and duration types (#51800)
* BUG: ArrowDtype.numpy_dtype returning ns units for non-nano timestamp and duration types * fix comment * whatsnew * reference arrow issue tracker
1 parent 852aa75 commit fa01870

File tree

3 files changed

+14
-6
lines changed

3 files changed

+14
-6
lines changed

doc/source/whatsnew/v2.1.0.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ Numeric
145145

146146
Conversion
147147
^^^^^^^^^^
148-
-
148+
- Bug in :meth:`ArrowDtype.numpy_dtype` returning nanosecond units for non-nanosecond ``pyarrow.timestamp`` and ``pyarrow.duration`` types (:issue:`51800`)
149149
-
150150

151151
Strings

pandas/core/arrays/arrow/dtype.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,18 @@ def name(self) -> str: # type: ignore[override]
148148
@cache_readonly
149149
def numpy_dtype(self) -> np.dtype:
150150
"""Return an instance of the related numpy dtype"""
151+
if pa.types.is_timestamp(self.pyarrow_dtype):
152+
# pa.timestamp(unit).to_pandas_dtype() returns ns units
153+
# regardless of the pyarrow timestamp units.
154+
# This can be removed if/when pyarrow addresses it:
155+
# https://github.com/apache/arrow/issues/34462
156+
return np.dtype(f"datetime64[{self.pyarrow_dtype.unit}]")
157+
if pa.types.is_duration(self.pyarrow_dtype):
158+
# pa.duration(unit).to_pandas_dtype() returns ns units
159+
# regardless of the pyarrow duration units
160+
# This can be removed if/when pyarrow addresses it:
161+
# https://github.com/apache/arrow/issues/34462
162+
return np.dtype(f"timedelta64[{self.pyarrow_dtype.unit}]")
151163
if pa.types.is_string(self.pyarrow_dtype):
152164
# pa.string().to_pandas_dtype() = object which we don't want
153165
return np.dtype(str)

pandas/tests/extension/test_arrow.py

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -640,11 +640,7 @@ def test_get_common_dtype(self, dtype, request):
640640
if (
641641
pa.types.is_date(pa_dtype)
642642
or pa.types.is_time(pa_dtype)
643-
or (
644-
pa.types.is_timestamp(pa_dtype)
645-
and (pa_dtype.unit != "ns" or pa_dtype.tz is not None)
646-
)
647-
or (pa.types.is_duration(pa_dtype) and pa_dtype.unit != "ns")
643+
or (pa.types.is_timestamp(pa_dtype) and pa_dtype.tz is not None)
648644
or pa.types.is_binary(pa_dtype)
649645
or pa.types.is_decimal(pa_dtype)
650646
):

0 commit comments

Comments
 (0)