Merge remote-tracking branch 'upstream/main' into perf/ri/reindex_return

Author: mroeschke

ci/code_checks.sh

@@ -162,9 +162,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.Series.dt.qyear\
         pandas.Series.dt.unit\
         pandas.Series.empty\
-        pandas.Timedelta.microseconds\
-        pandas.Timedelta.unit\
-        pandas.Timedelta.value\
         pandas.Timestamp.day\
         pandas.Timestamp.fold\
         pandas.Timestamp.hour\
@@ -876,11 +873,9 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.plotting.parallel_coordinates\
         pandas.plotting.radviz\
         pandas.plotting.table\
-        pandas.read_feather\
         pandas.read_orc\
         pandas.read_sas\
         pandas.read_spss\
-        pandas.read_sql_query\
         pandas.read_stata\
         pandas.set_eng_float_format\
         pandas.timedelta_range\

doc/source/development/maintaining.rst

@@ -326,34 +326,6 @@ a milestone before tagging, you can request the bot to backport it with:
    @Meeseeksdev backport <branch>
 
 
-.. _maintaining.asv-machine:
-
-Benchmark machine
------------------
-
-The team currently owns dedicated hardware for hosting a website for pandas' ASV performance benchmark. The results
-are published to https://asv-runner.github.io/asv-collection/pandas/
-
-Configuration
-`````````````
-
-The machine can be configured with the `Ansible <http://docs.ansible.com/ansible/latest/index.html>`_ playbook in https://github.com/tomaugspurger/asv-runner.
-
-Publishing
-``````````
-
-The results are published to another GitHub repository, https://github.com/tomaugspurger/asv-collection.
-Finally, we have a cron job on our docs server to pull from https://github.com/tomaugspurger/asv-collection, to serve them from ``/speed``.
-Ask Tom or Joris for access to the webserver.
-
-Debugging
-`````````
-
-The benchmarks are scheduled by Airflow. It has a dashboard for viewing and debugging the results. You'll need to setup an SSH tunnel to view them
-
-    ssh -L 8080:localhost:8080 [email protected]
-
-
 .. _maintaining.release:
 
 Release process

doc/source/whatsnew/v3.0.0.rst

@@ -244,6 +244,7 @@ Removal of prior version deprecations/changes
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
+- :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57542`)
 - Performance improvement in :class:`DataFrame` when ``data`` is a ``dict`` and ``columns`` is specified (:issue:`24368`)
 - Performance improvement in :meth:`DataFrame.join` for sorted but non-unique indexes (:issue:`56941`)
 - Performance improvement in :meth:`DataFrame.join` when left and/or right are non-unique and ``how`` is ``"left"``, ``"right"``, or ``"inner"`` (:issue:`56817`)
@@ -252,12 +253,12 @@ Performance improvements
 - Performance improvement in :meth:`Index.join` by propagating cached attributes in cases where the result matches one of the inputs (:issue:`57023`)
 - Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`)
 - Performance improvement in :meth:`MultiIndex.equals` for equal length indexes (:issue:`56990`)
+- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
 - Performance improvement in :meth:`RangeIndex.append` when appending the same index (:issue:`57252`)
 - Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`)
 - Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`)
-- Performance improvement in indexing operations for string dtypes (:issue:`56997`)
-- :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`?``)
 - Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`)
+- Performance improvement in indexing operations for string dtypes (:issue:`56997`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.bug_fixes:
@@ -266,6 +267,7 @@ Bug fixes
 ~~~~~~~~~
 - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
 - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
+- Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`)
 - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 
 Categorical
@@ -325,6 +327,7 @@ MultiIndex
 
 I/O
 ^^^
+- Bug in :meth:`DataFrame.to_excel` when writing empty :class:`DataFrame` with :class:`MultiIndex` on both axes (:issue:`57696`)
 -
 -
 

pandas/_libs/tslib.pyx

@@ -598,7 +598,8 @@ cpdef array_to_datetime(
         is_same_offsets = len(out_tzoffset_vals) == 1
         if not is_same_offsets:
             raise ValueError(
-                "cannot parse datetimes with mixed time zones unless `utc=True`"
+                "Mixed timezones detected. Pass utc=True in to_datetime "
+                "or tz='UTC' in DatetimeIndex to convert to a common timezone."
             )
         elif state.found_naive or state.found_other:
             # e.g. test_to_datetime_mixed_awareness_mixed_types
@@ -610,7 +611,7 @@ cpdef array_to_datetime(
             if not tz_compare(tz_out, tz_out2):
                 # e.g. test_to_datetime_mixed_tzs_mixed_types
                 raise ValueError(
-                    "Mixed timezones detected. pass utc=True in to_datetime "
+                    "Mixed timezones detected. Pass utc=True in to_datetime "
                     "or tz='UTC' in DatetimeIndex to convert to a common timezone."
                 )
             # e.g. test_to_datetime_mixed_types_matching_tzs

pandas/_libs/tslibs/dtypes.pxd

@@ -11,7 +11,6 @@ cpdef int64_t periods_per_second(NPY_DATETIMEUNIT reso) except? -1
 cdef NPY_DATETIMEUNIT get_supported_reso(NPY_DATETIMEUNIT reso)
 cdef bint is_supported_unit(NPY_DATETIMEUNIT reso)
 
-cpdef freq_to_period_freqstr(freq_n, freq_name)
 cdef dict c_OFFSET_TO_PERIOD_FREQSTR
 cdef dict c_OFFSET_DEPR_FREQSTR
 cdef dict c_REVERSE_OFFSET_DEPR_FREQSTR

pandas/_libs/tslibs/dtypes.pyi

@@ -7,7 +7,6 @@ OFFSET_TO_PERIOD_FREQSTR: dict[str, str]
 def periods_per_day(reso: int = ...) -> int: ...
 def periods_per_second(reso: int) -> int: ...
 def abbrev_to_npy_unit(abbrev: str | None) -> int: ...
-def freq_to_period_freqstr(freq_n: int, freq_name: str) -> str: ...
 
 class PeriodDtypeBase:
     _dtype_code: int  # PeriodDtypeCode

pandas/_libs/tslibs/dtypes.pyx

@@ -334,15 +334,6 @@ cdef dict c_REVERSE_OFFSET_DEPR_FREQSTR = {
     v: k for k, v in c_OFFSET_DEPR_FREQSTR.items()
 }
 
-cpdef freq_to_period_freqstr(freq_n, freq_name):
-    if freq_n == 1:
-        freqstr = f"""{c_OFFSET_TO_PERIOD_FREQSTR.get(
-            freq_name, freq_name)}"""
-    else:
-        freqstr = f"""{freq_n}{c_OFFSET_TO_PERIOD_FREQSTR.get(
-            freq_name, freq_name)}"""
-    return freqstr
-
 # Map deprecated resolution abbreviations to correct resolution abbreviations
 cdef dict c_DEPR_ABBREVS = {
     "A": "Y",

pandas/_libs/tslibs/offsets.pyx

@@ -4698,7 +4698,7 @@ _lite_rule_alias = {
     "ns": "ns",
 }
 
-_dont_uppercase = _dont_uppercase = {"h", "bh", "cbh", "MS", "ms", "s"}
+_dont_uppercase = {"h", "bh", "cbh", "MS", "ms", "s"}
 
 
 INVALID_FREQ_ERR_MSG = "Invalid frequency: {0}"

pandas/_libs/tslibs/period.pyx

@@ -38,10 +38,7 @@ from libc.time cimport (
     tm,
 )
 
-from pandas._libs.tslibs.dtypes cimport (
-    c_OFFSET_TO_PERIOD_FREQSTR,
-    freq_to_period_freqstr,
-)
+from pandas._libs.tslibs.dtypes cimport c_OFFSET_TO_PERIOD_FREQSTR
 
 from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
 
@@ -97,9 +94,6 @@ from pandas._libs.tslibs.dtypes cimport (
     attrname_to_abbrevs,
     freq_group_code_to_npy_unit,
 )
-
-from pandas._libs.tslibs.dtypes import freq_to_period_freqstr
-
 from pandas._libs.tslibs.parsing cimport quarter_to_myear
 
 from pandas._libs.tslibs.parsing import parse_datetime_string_with_reso
@@ -1554,7 +1548,7 @@ def extract_ordinals(ndarray values, freq) -> np.ndarray:
         # if we don't raise here, we'll segfault later!
         raise TypeError("extract_ordinals values must be object-dtype")
 
-    freqstr = freq_to_period_freqstr(freq.n, freq.name)
+    freqstr = PeriodDtypeBase(freq._period_dtype_code, freq.n)._freqstr
 
     for i in range(n):
         # Analogous to: p = values[i]
@@ -1722,8 +1716,15 @@ cdef class PeriodMixin:
             condition = self.freq != other
 
         if condition:
-            freqstr = freq_to_period_freqstr(self.freq.n, self.freq.name)
-            other_freqstr = freq_to_period_freqstr(other.n, other.name)
+            freqstr = PeriodDtypeBase(
+                self.freq._period_dtype_code, self.freq.n
+            )._freqstr
+            if hasattr(other, "_period_dtype_code"):
+                other_freqstr = PeriodDtypeBase(
+                    other._period_dtype_code, other.n
+                )._freqstr
+            else:
+                other_freqstr = other.freqstr
             msg = DIFFERENT_FREQ.format(
                 cls=type(self).__name__,
                 own_freq=freqstr,
@@ -2479,7 +2480,7 @@ cdef class _Period(PeriodMixin):
         >>> pd.Period('2020-01', 'D').freqstr
         'D'
         """
-        freqstr = freq_to_period_freqstr(self.freq.n, self.freq.name)
+        freqstr = PeriodDtypeBase(self.freq._period_dtype_code, self.freq.n)._freqstr
         return freqstr
 
     def __repr__(self) -> str:

pandas/_libs/tslibs/strptime.pyx

@@ -503,7 +503,8 @@ def array_strptime(
         is_same_offsets = len(out_tzoffset_vals) == 1
         if not is_same_offsets or (state.found_naive or state.found_other):
             raise ValueError(
-                "cannot parse datetimes with mixed time zones unless `utc=True`"
+                "Mixed timezones detected. Pass utc=True in to_datetime "
+                "or tz='UTC' in DatetimeIndex to convert to a common timezone."
             )
         elif tz_out is not None:
             # GH#55693
@@ -512,7 +513,8 @@ def array_strptime(
             if not tz_compare(tz_out, tz_out2):
                 # e.g. test_to_datetime_mixed_offsets_with_utc_false_removed
                 raise ValueError(
-                    "cannot parse datetimes with mixed time zones unless `utc=True`"
+                    "Mixed timezones detected. Pass utc=True in to_datetime "
+                    "or tz='UTC' in DatetimeIndex to convert to a common timezone."
                 )
             # e.g. test_guess_datetime_format_with_parseable_formats
         else:
@@ -523,7 +525,8 @@ def array_strptime(
         if tz_out and (state.found_other or state.found_naive_str):
             # found_other indicates a tz-naive int, float, dt64, or date
             raise ValueError(
-                "cannot parse datetimes with mixed time zones unless `utc=True`"
+                "Mixed timezones detected. Pass utc=True in to_datetime "
+                "or tz='UTC' in DatetimeIndex to convert to a common timezone."
             )
 
     if infer_reso:

pandas/_libs/tslibs/timedeltas.pyx

@@ -1038,6 +1038,25 @@ cdef class _Timedelta(timedelta):
 
     @property
     def value(self):
+        """
+        Return the value of Timedelta object in nanoseconds.
+
+        Return the total seconds, milliseconds and microseconds
+        of the timedelta as nanoseconds.
+
+        Returns
+        -------
+        int
+
+        See Also
+        --------
+        Timedelta.unit : Return the unit of Timedelta object.
+
+        Examples
+        --------
+        >>> pd.Timedelta(1, "us").value
+        1000
+        """
         try:
             return convert_reso(self._value, self._creso, NPY_FR_ns, False)
         except OverflowError:
@@ -1120,6 +1139,37 @@ cdef class _Timedelta(timedelta):
     def microseconds(self) -> int:  # TODO(cython3): make cdef property
         # NB: using the python C-API PyDateTime_DELTA_GET_MICROSECONDS will fail
         #  (or be incorrect)
+        """
+        Return the number of microseconds (n), where 0 <= n < 1 millisecond.
+
+        Timedelta.microseconds = milliseconds * 1000 + microseconds.
+
+        Returns
+        -------
+        int
+            Number of microseconds.
+
+        See Also
+        --------
+        Timedelta.components : Return all attributes with assigned values
+            (i.e. days, hours, minutes, seconds, milliseconds, microseconds,
+            nanoseconds).
+
+        Examples
+        --------
+        **Using string input**
+
+        >>> td = pd.Timedelta('1 days 2 min 3 us')
+
+        >>> td.microseconds
+        3
+
+        **Using integer input**
+
+        >>> td = pd.Timedelta(42, unit='us')
+        >>> td.microseconds
+        42
+        """
         self._ensure_components()
         return self._ms * 1000 + self._us
 
@@ -1141,6 +1191,26 @@ cdef class _Timedelta(timedelta):
 
     @property
     def unit(self) -> str:
+        """
+        Return the unit of Timedelta object.
+
+        The unit of Timedelta object is nanosecond, i.e., 'ns' by default.
+
+        Returns
+        -------
+        str
+
+        See Also
+        --------
+        Timedelta.value : Return the value of Timedelta object in nanoseconds.
+        Timedelta.as_unit : Convert the underlying int64 representation to
+            the given unit.
+
+        Examples
+        --------
+        >>> td = pd.Timedelta(42, unit='us')
+        'ns'
+        """
         return npy_unit_to_abbrev(self._creso)
 
     def __hash__(_Timedelta self):

pandas/core/arrays/period.py

@@ -37,7 +37,6 @@
 from pandas._libs.tslibs.dtypes import (
     FreqGroup,
     PeriodDtypeBase,
-    freq_to_period_freqstr,
 )
 from pandas._libs.tslibs.fields import isleapyear_arr
 from pandas._libs.tslibs.offsets import (
@@ -325,7 +324,7 @@ def _from_datetime64(cls, data, freq, tz=None) -> Self:
         PeriodArray[freq]
         """
         if isinstance(freq, BaseOffset):
-            freq = freq_to_period_freqstr(freq.n, freq.name)
+            freq = PeriodDtype(freq)._freqstr
         data, freq = dt64arr_to_periodarr(data, freq, tz)
         dtype = PeriodDtype(freq)
         return cls(data, dtype=dtype)
@@ -399,7 +398,7 @@ def freq(self) -> BaseOffset:
 
     @property
     def freqstr(self) -> str:
-        return freq_to_period_freqstr(self.freq.n, self.freq.name)
+        return PeriodDtype(self.freq)._freqstr
 
     def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
         if dtype == "i8":
@@ -1002,13 +1001,17 @@ def raise_on_incompatible(left, right) -> IncompatibleFrequency:
     if isinstance(right, (np.ndarray, ABCTimedeltaArray)) or right is None:
         other_freq = None
     elif isinstance(right, BaseOffset):
-        other_freq = freq_to_period_freqstr(right.n, right.name)
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore", r"PeriodDtype\[B\] is deprecated", category=FutureWarning
+            )
+            other_freq = PeriodDtype(right)._freqstr
     elif isinstance(right, (ABCPeriodIndex, PeriodArray, Period)):
         other_freq = right.freqstr
     else:
         other_freq = delta_to_tick(Timedelta(right)).freqstr
 
-    own_freq = freq_to_period_freqstr(left.freq.n, left.freq.name)
+    own_freq = PeriodDtype(left.freq)._freqstr
     msg = DIFFERENT_FREQ.format(
         cls=type(left).__name__, own_freq=own_freq, other_freq=other_freq
     )