Merge remote-tracking branch 'upstream/master' into depr_str_cat_LLoLL

Author: h-vetinari

.gitignore

@@ -62,6 +62,8 @@ dist
 coverage.xml
 coverage_html_report
 *.pytest_cache
+# hypothesis test database
+.hypothesis/
 
 # OS generated files #
 ######################

ci/appveyor-27.yaml

@@ -28,3 +28,4 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0

ci/appveyor-36.yaml

@@ -25,3 +25,4 @@ dependencies:
   - cython>=0.28.2
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0

ci/check_imports.py

@@ -9,6 +9,7 @@
     'html5lib',
     'ipython',
     'jinja2'
+    'hypothesis',
     'lxml',
     'numexpr',
     'openpyxl',

ci/circle-27-compat.yaml

@@ -7,7 +7,7 @@ dependencies:
   - cython=0.28.2
   - jinja2=2.8
   - numexpr=2.4.4 # we test that we correctly don't use an unsupported numexpr
-  - numpy=1.9.2
+  - numpy=1.9.3
   - openpyxl
   - psycopg2
   - pytables=3.2.2
@@ -26,3 +26,4 @@ dependencies:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1
     - pymysql==0.6.0
+    - hypothesis>=3.58.0

ci/circle-35-ascii.yaml

@@ -11,3 +11,5 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pip:
+    - hypothesis>=3.58.0

ci/circle-36-locale.yaml

@@ -31,3 +31,5 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - pip:
+    - hypothesis>=3.58.0

ci/circle-36-locale_slow.yaml

@@ -32,3 +32,5 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - pip:
+    - hypothesis>=3.58.0

ci/doctests.sh

@@ -28,7 +28,7 @@ if [ "$DOCTEST" ]; then
     fi
 
     pytest --doctest-modules -v pandas/core/series.py \
-        -k"-agg -map -nlargest -nonzero -nsmallest -reindex -searchsorted -to_dict"
+        -k"-nlargest -nonzero -nsmallest -reindex -searchsorted -to_dict"
 
     if [ $? -ne "0" ]; then
         RET=1

ci/environment-dev.yaml

@@ -7,6 +7,7 @@ dependencies:
   - NumPy
   - flake8
   - flake8-comprehensions
+  - hypothesis>=3.58.0
   - moto
   - pytest>=3.6
   - python-dateutil>=2.5.0

ci/requirements_dev.txt

@@ -4,6 +4,7 @@ Cython>=0.28.2
 NumPy
 flake8
 flake8-comprehensions
+hypothesis>=3.58.0
 moto
 pytest>=3.6
 python-dateutil>=2.5.0

ci/travis-27-locale.yaml

@@ -7,7 +7,7 @@ dependencies:
   - cython=0.28.2
   - lxml
   - matplotlib=1.4.3
-  - numpy=1.9.2
+  - numpy=1.9.3
   - openpyxl=2.4.0
   - python-dateutil
   - python-blosc
@@ -22,6 +22,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/travis-27.yaml

@@ -45,6 +45,7 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0
   - pip:
     - backports.lzma
     - cpplint

ci/travis-35-osx.yaml

@@ -25,3 +25,4 @@ dependencies:
   - pytest-xdist
   - pip:
     - python-dateutil==2.5.3
+    - hypothesis>=3.58.0

ci/travis-36-doc.yaml

@@ -10,6 +10,7 @@ dependencies:
   - fastparquet
   - feather-format
   - html5lib
+  - hypothesis>=3.58.0
   - ipykernel
   - ipython
   - ipywidgets

ci/travis-36-numpydev.yaml

@@ -8,6 +8,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"

ci/travis-36-slow.yaml

@@ -28,3 +28,4 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0

ci/travis-36.yaml

@@ -41,6 +41,7 @@ dependencies:
   - pytest-xdist
   - pytest-cov
   - moto
+  - hypothesis>=3.58.0
   - pip:
     - brotlipy
     - coverage

ci/travis-37.yaml

@@ -12,3 +12,4 @@ dependencies:
   - pytz
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0

doc/source/contributing.rst

@@ -820,6 +820,46 @@ Tests that we have ``parametrized`` are now accessible via the test name, for ex
    test_cool_feature.py::test_series[int8] PASSED
 
 
+.. _using-hypothesis:
+
+Using ``hypothesis``
+~~~~~~~~~~~~~~~~~~~~
+
+Hypothesis is a library for property-based testing.  Instead of explicitly
+parametrizing a test, you can describe *all* valid inputs and let Hypothesis
+try to find a failing input.  Even better, no matter how many random examples
+it tries, Hypothesis always reports a single minimal counterexample to your
+assertions - often an example that you would never have thought to test.
+
+See `Getting Started with Hypothesis <https://hypothesis.works/articles/getting-started-with-hypothesis/>`_
+for more of an introduction, then `refer to the Hypothesis documentation
+for details <https://hypothesis.readthedocs.io/en/latest/index.html>`_.
+
+.. code-block:: python
+
+    import json
+    from hypothesis import given, strategies as st
+
+    any_json_value = st.deferred(lambda: st.one_of(
+        st.none(), st.booleans(), st.floats(allow_nan=False), st.text(),
+        st.lists(any_json_value), st.dictionaries(st.text(), any_json_value)
+    ))
+
+    @given(value=any_json_value)
+    def test_json_roundtrip(value):
+        result = json.loads(json.dumps(value))
+        assert value == result
+
+This test shows off several useful features of Hypothesis, as well as
+demonstrating a good use-case: checking properties that should hold over
+a large or complicated domain of inputs.
+
+To keep the Pandas test suite running quickly, parametrized tests are
+preferred if the inputs or logic are simple, with Hypothesis tests reserved
+for cases with complex logic or where there are too many combinations of
+options or subtle interactions to test (or think of!) all of them.
+
+
 Running the test suite
 ----------------------
 

doc/source/install.rst

@@ -202,7 +202,8 @@ pandas is equipped with an exhaustive set of unit tests, covering about 97% of
 the code base as of this writing. To run it on your machine to verify that
 everything is working (and that you have all of the dependencies, soft and hard,
 installed), make sure you have `pytest
-<http://docs.pytest.org/en/latest/>`__ >= 3.6 and run:
+<http://docs.pytest.org/en/latest/>`__ >= 3.6 and `Hypothesis
+<https://hypothesis.readthedocs.io/>`__ >= 3.58, then run:
 
 ::
 

doc/source/whatsnew/v0.24.0.txt

@@ -239,7 +239,7 @@ without timezone localization. This is inconsistent from parsing the same
 datetime string with :class:`Timestamp` which would preserve the UTC
 offset in the ``tz`` attribute. Now, :func:`to_datetime` preserves the UTC
 offset in the ``tz`` attribute when all the datetime strings have the same
-UTC offset (:issue:`17697`, :issue:`11736`)
+UTC offset (:issue:`17697`, :issue:`11736`, :issue:`22457`)
 
 *Previous Behavior*:
 
@@ -447,6 +447,7 @@ ExtensionType Changes
 - Added ``ExtensionDtype._is_numeric`` for controlling whether an extension dtype is considered numeric (:issue:`22290`).
 - The ``ExtensionArray`` constructor, ``_from_sequence`` now take the keyword arg ``copy=False`` (:issue:`21185`)
 - Bug in :meth:`Series.get` for ``Series`` using ``ExtensionArray`` and integer index (:issue:`21257`)
+- :meth:`~Series.shift` now dispatches to :meth:`ExtensionArray.shift` (:issue:`22386`)
 - :meth:`Series.combine()` works correctly with :class:`~pandas.api.extensions.ExtensionArray` inside of :class:`Series` (:issue:`20825`)
 - :meth:`Series.combine()` with scalar argument now works for any function type (:issue:`21248`)
 - :meth:`Series.astype` and :meth:`DataFrame.astype` now dispatch to :meth:`ExtensionArray.astype` (:issue:`21185:`).
@@ -583,12 +584,14 @@ Datetimelike
 - Bug in :class:`DataFrame` comparisons against ``Timestamp``-like objects failing to raise ``TypeError`` for inequality checks with mismatched types (:issue:`8932`,:issue:`22163`)
 - Bug in :class:`DataFrame` with mixed dtypes including ``datetime64[ns]`` incorrectly raising ``TypeError`` on equality comparisons (:issue:`13128`,:issue:`22163`)
 - Bug in :meth:`DataFrame.eq` comparison against ``NaT`` incorrectly returning ``True`` or ``NaN`` (:issue:`15697`,:issue:`22163`)
-- Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`)
--
 
 Timedelta
 ^^^^^^^^^
-
+- Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype division by ``Timedelta``-like scalar incorrectly returning ``timedelta64[ns]`` dtype instead of ``float64`` dtype (:issue:`20088`,:issue:`22163`)
+- Bug in adding a :class:`Index` with object dtype to a :class:`Series` with ``timedelta64[ns]`` dtype incorrectly raising (:issue:`22390`)
+- Bug in multiplying a :class:`Series` with numeric dtype against a ``timedelta`` object (:issue:`22390`)
+- Bug in :class:`Series` with numeric dtype when adding or subtracting an an array or ``Series`` with ``timedelta64`` dtype (:issue:`22390`)
+- Bug in :class:`Index` with numeric dtype when multiplying or dividing an array with dtype ``timedelta64`` (:issue:`22390`)
 -
 -
 -
@@ -633,6 +636,7 @@ Numeric
   a ``TypeError`` was wrongly raised. For all three methods such calculation are now done correctly. (:issue:`16679`).
 - Bug in :class:`Series` comparison against datetime-like scalars and arrays (:issue:`22074`)
 - Bug in :class:`DataFrame` multiplication between boolean dtype and integer returning ``object`` dtype instead of integer dtype (:issue:`22047`,:issue:`22163`)
+- Bug in :meth:`DataFrame.apply` where, when supplied with a string argument and additional positional or keyword arguments (e.g. ``df.apply('sum', min_count=1)``), a ``TypeError`` was wrongly raised (:issue:`22376`)
 -
 
 Strings
@@ -721,12 +725,13 @@ Reshaping
 - Bug in :func:`get_dummies` with Unicode attributes in Python 2 (:issue:`22084`)
 - Bug in :meth:`DataFrame.replace` raises ``RecursionError`` when replacing empty lists (:issue:`22083`)
 - Bug in :meth:`Series.replace` and meth:`DataFrame.replace` when dict is used as the `to_replace` value and one key in the dict is is another key's value, the results were inconsistent between using integer key and using string key (:issue:`20656`)
--
+- Bug in :meth:`DataFrame.drop_duplicates` for empty ``DataFrame`` which incorrectly raises an error (:issue:`20516`)
 
 Build Changes
 ^^^^^^^^^^^^^
 
 - Building pandas for development now requires ``cython >= 0.28.2`` (:issue:`21688`)
+- Testing pandas now requires ``hypothesis>=3.58`` (:issue:22280).  You can find `the Hypothesis docs here <https://hypothesis.readthedocs.io/en/latest/index.html>`_, and a pandas-specific introduction :ref:`in the contributing guide <using-hypothesis>` .
 -
 
 Other

pandas/conftest.py

@@ -450,3 +450,37 @@ def mock():
         return importlib.import_module("unittest.mock")
     else:
         return pytest.importorskip("mock")
+
+
+# ----------------------------------------------------------------
+# Global setup for tests using Hypothesis
+
+from hypothesis import strategies as st
+
+# Registering these strategies makes them globally available via st.from_type,
+# which is use for offsets in tests/tseries/offsets/test_offsets_properties.py
+for name in 'MonthBegin MonthEnd BMonthBegin BMonthEnd'.split():
+    cls = getattr(pd.tseries.offsets, name)
+    st.register_type_strategy(cls, st.builds(
+        cls,
+        n=st.integers(-99, 99),
+        normalize=st.booleans(),
+    ))
+
+for name in 'YearBegin YearEnd BYearBegin BYearEnd'.split():
+    cls = getattr(pd.tseries.offsets, name)
+    st.register_type_strategy(cls, st.builds(
+        cls,
+        n=st.integers(-5, 5),
+        normalize=st.booleans(),
+        month=st.integers(min_value=1, max_value=12),
+    ))
+
+for name in 'QuarterBegin QuarterEnd BQuarterBegin BQuarterEnd'.split():
+    cls = getattr(pd.tseries.offsets, name)
+    st.register_type_strategy(cls, st.builds(
+        cls,
+        n=st.integers(-24, 24),
+        normalize=st.booleans(),
+        startingMonth=st.integers(min_value=1, max_value=12)
+    ))

pandas/core/apply.py

@@ -71,7 +71,9 @@ def __init__(self, obj, func, broadcast, raw, reduce, result_type,
         self.result_type = result_type
 
         # curry if needed
-        if kwds or args and not isinstance(func, np.ufunc):
+        if ((kwds or args) and
+                not isinstance(func, (np.ufunc, compat.string_types))):
+
             def f(x):
                 return func(x, *args, **kwds)
         else:

pandas/core/arrays/base.py

@@ -59,6 +59,10 @@ class ExtensionArray(object):
     * factorize / _values_for_factorize
     * argsort / _values_for_argsort
 
+    The remaining methods implemented on this class should be performant,
+    as they only compose abstract methods. Still, a more efficient
+    implementation may be available, and these methods can be overridden.
+
     This class does not inherit from 'abc.ABCMeta' for performance reasons.
     Methods and properties required by the interface raise
     ``pandas.errors.AbstractMethodError`` and no ``register`` method is
@@ -400,6 +404,40 @@ def dropna(self):
 
         return self[~self.isna()]
 
+    def shift(self, periods=1):
+        # type: (int) -> ExtensionArray
+        """
+        Shift values by desired number.
+
+        Newly introduced missing values are filled with
+        ``self.dtype.na_value``.
+
+        .. versionadded:: 0.24.0
+
+        Parameters
+        ----------
+        periods : int, default 1
+            The number of periods to shift. Negative values are allowed
+            for shifting backwards.
+
+        Returns
+        -------
+        shifted : ExtensionArray
+        """
+        # Note: this implementation assumes that `self.dtype.na_value` can be
+        # stored in an instance of your ExtensionArray with `self.dtype`.
+        if periods == 0:
+            return self.copy()
+        empty = self._from_sequence([self.dtype.na_value] * abs(periods),
+                                    dtype=self.dtype)
+        if periods > 0:
+            a = empty
+            b = self[:-periods]
+        else:
+            a = self[abs(periods):]
+            b = empty
+        return self._concat_same_type([a, b])
+
     def unique(self):
         """Compute the ExtensionArray of unique values.