BUG: read_hdf bad filtering in case of categorical string columns

.github/ISSUE_TEMPLATE/documentation_improvement.md

@@ -9,7 +9,7 @@ labels: "Docs, Needs Triage"
 
 #### Location of the documentation
 
-[this should provide the location of the documentation, e.g. "pandas.read_csv" or the URL of the documentation, e.g. "https://dev.pandas.io/docs/reference/api/pandas.read_csv.html"]
+[this should provide the location of the documentation, e.g. "pandas.read_csv" or the URL of the documentation, e.g. "https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html"]
 
 **Note**: You can check the latest versions of the docs on `master` [here](https://pandas.pydata.org/docs/dev/).
 

.pre-commit-config.yaml

@@ -192,6 +192,6 @@ repos:
         files: ^pandas/
         exclude: ^pandas/tests/
 -   repo: https://github.com/MarcoGorelli/no-string-hints
-    rev: v0.1.5
+    rev: v0.1.6
     hooks:
     -   id: no-string-hints

README.md

@@ -1,5 +1,5 @@
 <div align="center">
-  <img src="https://dev.pandas.io/static/img/pandas.svg"><br>
+  <img src="https://pandas.pydata.org/static/img/pandas.svg"><br>
 </div>
 
 -----------------

asv_bench/benchmarks/indexing.py

@@ -243,6 +243,20 @@ def time_loc_list(self, monotonic):
         monotonic.loc[80000:]
 
 
+class DatetimeIndexIndexing:
+    def setup(self):
+        dti = date_range("2016-01-01", periods=10000, tz="US/Pacific")
+        dti2 = dti.tz_convert("UTC")
+        self.dti = dti
+        self.dti2 = dti2
+
+    def time_get_indexer_mismatched_tz(self):
+        # reached via e.g.
+        #  ser = Series(range(len(dti)), index=dti)
+        #  ser[dti2]
+        self.dti.get_indexer(self.dti2)
+
+
 class CategoricalIndexIndexing:
 
     params = ["monotonic_incr", "monotonic_decr", "non_monotonic"]

doc/source/whatsnew/v0.25.0.rst

@@ -829,7 +829,7 @@ If installed, we now require:
 | pytest (dev)    | 4.0.2           |          |
 +-----------------+-----------------+----------+
 
-For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
 

doc/source/whatsnew/v1.0.0.rst

@@ -27,7 +27,7 @@ version releases. Briefly,
 
 See :ref:`policies.version` for more.
 
-.. _2019 Pandas User Survey: http://dev.pandas.io/pandas-blog/2019-pandas-user-survey.html
+.. _2019 Pandas User Survey: https://pandas.pydata.org/community/blog/2019-user-survey.html
 .. _SemVer: https://semver.org
 
 {{ header }}
@@ -702,7 +702,7 @@ If installed, we now require:
 | pytest (dev)    | 4.0.2           |          |         |
 +-----------------+-----------------+----------+---------+
 
-For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
 

doc/source/whatsnew/v1.1.0.rst

@@ -751,7 +751,7 @@ If installed, we now require:
 | pytest (dev)    | 4.0.2           |          |         |
 +-----------------+-----------------+----------+---------+
 
-For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
 

doc/source/whatsnew/v1.2.0.rst

@@ -448,7 +448,7 @@ If installed, we now require:
 | mypy (dev)      | 0.782           |          |    X    |
 +-----------------+-----------------+----------+---------+
 
-For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
 

doc/source/whatsnew/v1.3.0.rst

@@ -120,7 +120,7 @@ If installed, we now require:
 | mypy (dev)      | 0.790           |          |    X    |
 +-----------------+-----------------+----------+---------+
 
-For `optional libraries <https://dev.pandas.io/docs/install.html#dependencies>`_ the general recommendation is to use the latest version.
+For `optional libraries <https://pandas.pydata.org/docs/getting_started/install.html>`_ the general recommendation is to use the latest version.
 The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
 Optional libraries below the lowest tested version may still work, but are not considered supported.
 
@@ -207,6 +207,7 @@ Performance improvements
 - Performance improvement in :meth:`Series.mean` for nullable data types (:issue:`34814`)
 - Performance improvement in :meth:`Series.isin` for nullable data types (:issue:`38340`)
 - Performance improvement in :meth:`DataFrame.corr` for method=kendall (:issue:`28329`)
+- Performance improvement in :meth:`core.window.Rolling.corr` and :meth:`core.window.Rolling.cov` (:issue:`39388`)
 
 .. ---------------------------------------------------------------------------
 
@@ -287,7 +288,9 @@ Indexing
 - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` with empty :class:`DataFrame` and specified columns for string indexer and non empty :class:`DataFrame` to set (:issue:`38831`)
 - Bug in :meth:`DataFrame.loc.__setitem__` raising ValueError when expanding unique column for :class:`DataFrame` with duplicate columns (:issue:`38521`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
+- Bug in :meth:`DataFrame.__setitem__` not raising ``ValueError`` when right hand side is a :class:`DataFrame` with wrong number of columns (:issue:`38604`)
 - Bug in :meth:`DataFrame.loc` dropping levels of :class:`MultiIndex` when :class:`DataFrame` used as input has only one row (:issue:`10521`)
+- Bug in :meth:`DataFrame.__getitem__` and :meth:`Series.__getitem__` always raising ``KeyError`` when slicing with existing strings an :class:`Index` with milliseconds (:issue:`33589`)
 - Bug in setting ``timedelta64`` values into numeric :class:`Series` failing to cast to object dtype (:issue:`39086`)
 - Bug in setting :class:`Interval` values into a :class:`Series` or :class:`DataFrame` with mismatched :class:`IntervalDtype` incorrectly casting the new values to the existing dtype (:issue:`39120`)
 - Bug in incorrectly raising in :meth:`Index.insert`, when setting a new column that cannot be held in the existing ``frame.columns``, or in :meth:`Series.reset_index` or :meth:`DataFrame.reset_index` instead of casting to a compatible dtype (:issue:`39068`)
@@ -327,10 +330,11 @@ I/O
 - Bug in :func:`read_csv` not switching ``true_values`` and ``false_values`` for nullable ``boolean`` dtype (:issue:`34655`)
 - Bug in :func:`read_json` when ``orient="split"`` does not maintain numeric string index (:issue:`28556`)
 - :meth:`read_sql` returned an empty generator if ``chunksize`` was no-zero and the query returned no results. Now returns a generator with a single empty dataframe (:issue:`34411`)
+- Bug in :func:`read_hdf` returning unexpected records when filtering on categorical string columns using ``where`` parameter (:issue:`39189`)
 
 Period
 ^^^^^^
-- Comparisons of :class:`Period` objects or :class:`Index`, :class:`Series`, or :class:`DataFrame` with mismatched ``PeriodDtype`` now behave like other mismatched-type comparisons, returning ``False`` for equals, ``True`` for not-equal, and raising ``TypeError`` for inequality checks (:issue:`??`)
+- Comparisons of :class:`Period` objects or :class:`Index`, :class:`Series`, or :class:`DataFrame` with mismatched ``PeriodDtype`` now behave like other mismatched-type comparisons, returning ``False`` for equals, ``True`` for not-equal, and raising ``TypeError`` for inequality checks (:issue:`39274`)
 -
 -
 
@@ -343,14 +347,15 @@ Plotting
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
-
+- Bug in :meth:`DataFrameGroupBy.agg` and :meth:`SeriesGroupBy.agg` with :class:`PeriodDtype` columns incorrectly casting results too aggressively (:issue:`38254`)
 - Bug in :meth:`SeriesGroupBy.value_counts` where unobserved categories in a grouped categorical series were not tallied (:issue:`38672`)
 - Bug in :meth:`SeriesGroupBy.value_counts` where error was raised on an empty series (:issue:`39172`)
 - Bug in :meth:`.GroupBy.indices` would contain non-existent indices when null values were present in the groupby keys (:issue:`9304`)
 - Fixed bug in :meth:`DataFrameGroupBy.sum` and :meth:`SeriesGroupBy.sum` causing loss of precision through using Kahan summation (:issue:`38778`)
 - Fixed bug in :meth:`DataFrameGroupBy.cumsum`, :meth:`SeriesGroupBy.cumsum`, :meth:`DataFrameGroupBy.mean` and :meth:`SeriesGroupBy.mean` causing loss of precision through using Kahan summation (:issue:`38934`)
 - Bug in :meth:`.Resampler.aggregate` and :meth:`DataFrame.transform` raising ``TypeError`` instead of ``SpecificationError`` when missing keys had mixed dtypes (:issue:`39025`)
 - Bug in :meth:`.DataFrameGroupBy.idxmin` and :meth:`.DataFrameGroupBy.idxmax` with ``ExtensionDtype`` columns (:issue:`38733`)
+- Bug in :meth:`Series.resample` would raise when the index was a :class:`PeriodIndex` consisting of ``NaT`` (:issue:`39227`)
 
 Reshaping
 ^^^^^^^^^
@@ -380,7 +385,8 @@ ExtensionArray
 Other
 ^^^^^
 - Bug in :class:`Index` constructor sometimes silently ignorning a specified ``dtype`` (:issue:`38879`)
--
+- Bug in constructing a :class:`Series` from a list and a :class:`PandasDtype` (:issue:`39357`)
+- Bug in :class:`Styler` which caused CSS to duplicate on multiple renders. (:issue:`39395`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/_libs/tslibs/period.pyx

@@ -1402,6 +1402,28 @@ cdef accessor _get_accessor_func(str field):
     return NULL
 
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def from_ordinals(const int64_t[:] values, freq):
+    cdef:
+        Py_ssize_t i, n = len(values)
+        int64_t[:] result = np.empty(len(values), dtype="i8")
+        int64_t val
+
+    freq = to_offset(freq)
+    if not isinstance(freq, BaseOffset):
+        raise ValueError("freq not specified and cannot be inferred")
+
+    for i in range(n):
+        val = values[i]
+        if val == NPY_NAT:
+            result[i] = NPY_NAT
+        else:
+            result[i] = Period(val, freq=freq).ordinal
+
+    return result.base
+
+
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def extract_ordinals(ndarray[object] values, freq):
@@ -1419,6 +1441,8 @@ def extract_ordinals(ndarray[object] values, freq):
 
         if is_null_datetimelike(p):
             ordinals[i] = NPY_NAT
+        elif util.is_integer_object(p):
+            raise TypeError(p)
         else:
             try:
                 ordinals[i] = p.ordinal

pandas/_testing/__init__.py

@@ -88,6 +88,7 @@
     assert_timedelta_array_equal,
     raise_assert_detail,
 )
+from pandas._testing.compat import get_dtype  # noqa:F401
 from pandas._testing.contexts import (  # noqa:F401
     RNGContext,
     decompress_file,

pandas/_testing/compat.py

@@ -0,0 +1,13 @@
+"""
+Helpers for sharing tests between DataFrame/Series
+"""
+
+from pandas import DataFrame
+
+
+def get_dtype(obj):
+    if isinstance(obj, DataFrame):
+        # Note: we are assuming only one column
+        return obj.dtypes.iat[0]
+    else:
+        return obj.dtype

pandas/core/algorithms.py

@@ -47,6 +47,7 @@
     needs_i8_conversion,
     pandas_dtype,
 )
+from pandas.core.dtypes.dtypes import PandasDtype
 from pandas.core.dtypes.generic import (
     ABCDatetimeArray,
     ABCExtensionArray,
@@ -1929,7 +1930,6 @@ def diff(arr, n: int, axis: int = 0, stacklevel=3):
     -------
     shifted
     """
-    from pandas.core.arrays import PandasDtype
 
     n = int(n)
     na = np.nan
@@ -1942,7 +1942,7 @@ def diff(arr, n: int, axis: int = 0, stacklevel=3):
 
     if isinstance(dtype, PandasDtype):
         # PandasArray cannot necessarily hold shifted versions of itself.
-        arr = np.asarray(arr)
+        arr = arr.to_numpy()
         dtype = arr.dtype
 
     if is_extension_array_dtype(dtype):

pandas/core/arrays/__init__.py

@@ -10,7 +10,7 @@
 from pandas.core.arrays.integer import IntegerArray
 from pandas.core.arrays.interval import IntervalArray
 from pandas.core.arrays.masked import BaseMaskedArray
-from pandas.core.arrays.numpy_ import PandasArray, PandasDtype
+from pandas.core.arrays.numpy_ import PandasArray
 from pandas.core.arrays.period import PeriodArray, period_array
 from pandas.core.arrays.sparse import SparseArray
 from pandas.core.arrays.string_ import StringArray
@@ -28,7 +28,6 @@
     "IntegerArray",
     "IntervalArray",
     "PandasArray",
-    "PandasDtype",
     "PeriodArray",
     "period_array",
     "SparseArray",

pandas/core/arrays/_mixins.py

@@ -17,7 +17,7 @@
 from pandas.core.dtypes.missing import array_equivalent
 
 from pandas.core import missing
-from pandas.core.algorithms import take, unique
+from pandas.core.algorithms import take, unique, value_counts
 from pandas.core.array_algos.transforms import shift
 from pandas.core.arrays.base import ExtensionArray
 from pandas.core.construction import extract_array
@@ -367,3 +367,37 @@ def where(
     def delete(self: NDArrayBackedExtensionArrayT, loc) -> NDArrayBackedExtensionArrayT:
         res_values = np.delete(self._ndarray, loc)
         return self._from_backing_data(res_values)
+
+    # ------------------------------------------------------------------------
+    # Additional array methods
+    #  These are not part of the EA API, but we implement them because
+    #  pandas assumes they're there.
+
+    def value_counts(self, dropna: bool = False):
+        """
+        Return a Series containing counts of unique values.
+
+        Parameters
+        ----------
+        dropna : bool, default False
+            Don't include counts of NA values.
+
+        Returns
+        -------
+        Series
+        """
+        if self.ndim != 1:
+            raise NotImplementedError
+
+        from pandas import Index, Series
+
+        if dropna:
+            values = self[~self.isna()]._ndarray
+        else:
+            values = self._ndarray
+
+        result = value_counts(values, sort=False, dropna=dropna)
+
+        index_arr = self._from_backing_data(np.asarray(result.index._data))
+        index = Index(index_arr, name=result.index.name)
+        return Series(result._values, index=index, name=result.name)

pandas/core/arrays/base.py

@@ -997,12 +997,12 @@ def repeat(self, repeats, axis=None):
     # ------------------------------------------------------------------------
 
     def take(
-        self,
+        self: ExtensionArrayT,
         indices: Sequence[int],
         *,
         allow_fill: bool = False,
         fill_value: Any = None,
-    ) -> ExtensionArray:
+    ) -> ExtensionArrayT:
         """
         Take elements from an array.
 
@@ -1261,6 +1261,13 @@ def _reduce(self, name: str, *, skipna: bool = True, **kwargs):
     def __hash__(self):
         raise TypeError(f"unhashable type: {repr(type(self).__name__)}")
 
+    # ------------------------------------------------------------------------
+    # Non-Optimized Default Methods
+
+    def delete(self: ExtensionArrayT, loc) -> ExtensionArrayT:
+        indexer = np.delete(np.arange(len(self)), loc)
+        return self.take(indexer)
+
 
 class ExtensionOpsMixin:
     """

pandas/core/arrays/datetimelike.py

@@ -63,7 +63,7 @@
 from pandas.core.dtypes.missing import is_valid_nat_for_dtype, isna
 
 from pandas.core import nanops, ops
-from pandas.core.algorithms import checked_add_with_arr, isin, unique1d, value_counts
+from pandas.core.algorithms import checked_add_with_arr, isin, unique1d
 from pandas.core.arraylike import OpsMixin
 from pandas.core.arrays._mixins import NDArrayBackedExtensionArray, ravel_compat
 import pandas.core.common as com
@@ -691,37 +691,6 @@ def _unbox(
     #  These are not part of the EA API, but we implement them because
     #  pandas assumes they're there.
 
-    def value_counts(self, dropna: bool = False):
-        """
-        Return a Series containing counts of unique values.
-
-        Parameters
-        ----------
-        dropna : bool, default True
-            Don't include counts of NaT values.
-
-        Returns
-        -------
-        Series
-        """
-        if self.ndim != 1:
-            raise NotImplementedError
-
-        from pandas import Index, Series
-
-        if dropna:
-            values = self[~self.isna()]._ndarray
-        else:
-            values = self._ndarray
-
-        cls = type(self)
-
-        result = value_counts(values, sort=False, dropna=dropna)
-        index = Index(
-            cls(result.index.view("i8"), dtype=self.dtype), name=result.index.name
-        )
-        return Series(result._values, index=index, name=result.name)
-
     @ravel_compat
     def map(self, mapper):
         # TODO(GH-23179): Add ExtensionArray.map

pandas/core/arrays/interval.py

@@ -1413,6 +1413,15 @@ def to_tuples(self, na_tuple=True):
 
     # ---------------------------------------------------------------------
 
+    def delete(self: IntervalArrayT, loc) -> IntervalArrayT:
+        if isinstance(self._left, np.ndarray):
+            new_left = np.delete(self._left, loc)
+            new_right = np.delete(self._right, loc)
+        else:
+            new_left = self._left.delete(loc)
+            new_right = self._right.delete(loc)
+        return self._shallow_copy(left=new_left, right=new_right)
+
     @Appender(_extension_array_shared_docs["repeat"] % _shared_docs_kwargs)
     def repeat(self, repeats, axis=None):
         nv.validate_repeat((), {"axis": axis})