merge with master

Author: arw2019

.travis.yml

@@ -58,12 +58,6 @@ matrix:
       services:
         - mysql
         - postgresql
-
-    - env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-      services:
-        - mysql
-        - postgresql
   allow_failures:
     - arch: arm64
       env:

asv_bench/benchmarks/rolling.py

@@ -91,11 +91,18 @@ class EWMMethods:
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5
         arr = (100 * np.random.random(N)).astype(dtype)
+        times = pd.date_range("1900", periods=N, freq="23s")
         self.ewm = getattr(pd, constructor)(arr).ewm(halflife=window)
+        self.ewm_times = getattr(pd, constructor)(arr).ewm(
+            halflife="1 Day", times=times
+        )
 
     def time_ewm(self, constructor, window, dtype, method):
         getattr(self.ewm, method)()
 
+    def time_ewm_times(self, constructor, window, dtype, method):
+        self.ewm.mean()
+
 
 class VariableWindowMethods(Methods):
     params = (

asv_bench/benchmarks/tslibs/resolution.py

@@ -0,0 +1,50 @@
+"""
+ipython analogue:
+
+tr = TimeResolution()
+mi = pd.MultiIndex.from_product(tr.params[:-1] + ([str(x) for x in tr.params[-1]],))
+df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"])
+
+for unit in tr.params[0]:
+    for size in tr.params[1]:
+        for tz in tr.params[2]:
+            tr.setup(unit, size, tz)
+            key = (unit, size, str(tz))
+            print(key)
+
+            val = %timeit -o tr.time_get_resolution(unit, size, tz)
+
+            df.loc[key] = (val.average, val.stdev)
+
+"""
+from datetime import timedelta, timezone
+
+from dateutil.tz import gettz, tzlocal
+import numpy as np
+import pytz
+
+from pandas._libs.tslibs.resolution import get_resolution
+
+
+class TimeResolution:
+    params = (
+        ["D", "h", "m", "s", "us", "ns"],
+        [1, 100, 10 ** 4, 10 ** 6],
+        [
+            None,
+            timezone.utc,
+            timezone(timedelta(minutes=60)),
+            pytz.timezone("US/Pacific"),
+            gettz("Asia/Tokyo"),
+            tzlocal(),
+        ],
+    )
+    param_names = ["unit", "size", "tz"]
+
+    def setup(self, unit, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        arr = arr.view(f"M8[{unit}]").astype("M8[ns]").view("i8")
+        self.i8data = arr
+
+    def time_get_resolution(self, unit, size, tz):
+        get_resolution(self.i8data, tz)

asv_bench/benchmarks/tslibs/tslib.py

@@ -0,0 +1,55 @@
+"""
+ipython analogue:
+
+tr = TimeIntsToPydatetime()
+mi = pd.MultiIndex.from_product(
+    tr.params[:-1] + ([str(x) for x in tr.params[-1]],)
+)
+df = pd.DataFrame(np.nan, index=mi, columns=["mean", "stdev"])
+for box in tr.params[0]:
+    for size in tr.params[1]:
+        for tz in tr.params[2]:
+            tr.setup(box, size, tz)
+            key = (box, size, str(tz))
+            print(key)
+            val = %timeit -o tr.time_ints_to_pydatetime(box, size, tz)
+            df.loc[key] = (val.average, val.stdev)
+"""
+from datetime import timedelta, timezone
+
+from dateutil.tz import gettz, tzlocal
+import numpy as np
+import pytz
+
+from pandas._libs.tslib import ints_to_pydatetime
+
+_tzs = [
+    None,
+    timezone.utc,
+    timezone(timedelta(minutes=60)),
+    pytz.timezone("US/Pacific"),
+    gettz("Asia/Tokyo"),
+    tzlocal(),
+]
+_sizes = [0, 1, 100, 10 ** 4, 10 ** 6]
+
+
+class TimeIntsToPydatetime:
+    params = (
+        ["time", "date", "datetime", "timestamp"],
+        _sizes,
+        _tzs,
+    )
+    param_names = ["box", "size", "tz"]
+    # TODO: fold? freq?
+
+    def setup(self, box, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_ints_to_pydatetime(self, box, size, tz):
+        if box == "date":
+            # ints_to_pydatetime does not allow non-None tz with date;
+            #  this will mean doing some duplicate benchmarks
+            tz = None
+        ints_to_pydatetime(self.i8data, tz, box=box)

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -0,0 +1,30 @@
+import numpy as np
+from pytz import UTC
+
+from pandas._libs.tslibs.tzconversion import tz_convert, tz_localize_to_utc
+
+from .tslib import _sizes, _tzs
+
+
+class TimeTZConvert:
+    params = (
+        _sizes,
+        [x for x in _tzs if x is not None],
+    )
+    param_names = ["size", "tz"]
+
+    def setup(self, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_tz_convert_from_utc(self, size, tz):
+        # effectively:
+        #  dti = DatetimeIndex(self.i8data, tz=tz)
+        #  dti.tz_localize(None)
+        tz_convert(self.i8data, UTC, tz)
+
+    def time_tz_localize_to_utc(self, size, tz):
+        # effectively:
+        #  dti = DatetimeIndex(self.i8data)
+        #  dti.tz_localize(tz, ambiguous="NaT", nonexistent="NaT")
+        tz_localize_to_utc(self.i8data, tz, ambiguous="NaT", nonexistent="NaT")

ci/azure/posix.yml

@@ -30,6 +30,11 @@ jobs:
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans"
 
+        py36_slow:
+          ENV_FILE: ci/deps/azure-36-slow.yaml
+          CONDA_PY: "36"
+          PATTERN: "slow"
+
         py36_locale:
           ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"

ci/deps/travis-36-slow.yaml renamed to ci/deps/azure-36-slow.yaml

@@ -166,5 +166,4 @@ if [[ -n ${SQL:0} ]]; then
 else
    echo "not using dbs on non-linux Travis builds or Azure Pipelines"
 fi
-
 echo "done"

ci/setup_env.sh

@@ -1095,6 +1095,25 @@ and **alpha** to the EW functions:
   one half.
 * **Alpha** specifies the smoothing factor directly.
 
+.. versionadded:: 1.1.0
+
+You can also specify ``halflife`` in terms of a timedelta convertible unit to specify the amount of
+time it takes for an observation to decay to half its value when also specifying a sequence
+of ``times``.
+
+.. ipython:: python
+
+    df = pd.DataFrame({'B': [0, 1, 2, np.nan, 4]})
+    df
+    times = ['2020-01-01', '2020-01-03', '2020-01-10', '2020-01-15', '2020-01-17']
+    df.ewm(halflife='4 days', times=pd.DatetimeIndex(times)).mean()
+
+The following formula is used to compute exponentially weighted mean with an input vector of times:
+
+.. math::
+
+    y_t = \frac{\sum_{i=0}^t 0.5^\frac{t_{t} - t_{i}}{\lambda} x_{t-i}}{0.5^\frac{t_{t} - t_{i}}{\lambda}},
+
 Here is an example for a univariate time series:
 
 .. ipython:: python

doc/source/user_guide/computation.rst

@@ -328,6 +328,8 @@ Other enhancements
 - :meth:`DataFrame.to_html` and :meth:`DataFrame.to_string`'s ``col_space`` parameter now accepts a list or dict to change only some specific columns' width (:issue:`28917`).
 - :meth:`DataFrame.to_excel` can now also write OpenOffice spreadsheet (.ods) files (:issue:`27222`)
 - :meth:`~Series.explode` now accepts ``ignore_index`` to reset the index, similarly to :meth:`pd.concat` or :meth:`DataFrame.sort_values` (:issue:`34932`).
+- :meth:`read_csv` now accepts string values like "0", "0.0", "1", "1.0" as convertible to the nullable boolean dtype (:issue:`34859`)
+- :class:`pandas.core.window.ExponentialMovingWindow` now supports a ``times`` argument that allows ``mean`` to be calculated with observations spaced by the timestamps in ``times`` (:issue:`34839`)
 
 .. ---------------------------------------------------------------------------
 
@@ -787,6 +789,7 @@ Deprecations
 - :meth:`Categorical.to_dense` is deprecated and will be removed in a future version, use ``np.asarray(cat)`` instead (:issue:`32639`)
 - The ``fastpath`` keyword in the ``SingleBlockManager`` constructor is deprecated and will be removed in a future version (:issue:`33092`)
 - Providing ``suffixes`` as a ``set`` in :func:`pandas.merge` is deprecated. Provide a tuple instead (:issue:`33740`, :issue:`34741`).
+- Indexing a series with a multi-dimensional indexer like ``[:, None]`` to return an ndarray now raises a ``FutureWarning``. Convert to a NumPy array before indexing instead (:issue:`27837`)
 - :meth:`Index.is_mixed` is deprecated and will be removed in a future version, check ``index.inferred_type`` directly instead (:issue:`32922`)
 
 - Passing any arguments but the first one to  :func:`read_html` as
@@ -818,6 +821,7 @@ Deprecations
 - :meth:`util.testing.assert_almost_equal` now accepts both relative and absolute
   precision through the ``rtol``, and ``atol`` parameters, thus deprecating the
   ``check_less_precise`` parameter. (:issue:`13357`).
+- :func:`DataFrame.melt` accepting a value_name that already exists is deprecated, and will be removed in a future version (:issue:`34731`)
 
 .. ---------------------------------------------------------------------------
 
@@ -974,6 +978,7 @@ Indexing
 - Bug in :meth:`DataFrame.loc` with dictionary of values changes columns with dtype of ``int`` to ``float`` (:issue:`34573`)
 - Bug in :meth:`Series.loc` when used with a :class:`MultiIndex` would raise an IndexingError when accessing a None value (:issue:`34318`)
 - Bug in :meth:`DataFrame.reset_index` and :meth:`Series.reset_index` would not preserve data types on an empty :class:`DataFrame` or :class:`Series` with a :class:`MultiIndex` (:issue:`19602`)
+- Bug in :class:`Series` and :class:`DataFrame` indexing with a ``time`` key on a :class:`DatetimeIndex` with ``NaT`` entries (:issue:`35114`)
 
 Missing
 ^^^^^^^

doc/source/whatsnew/v1.1.0.rst

@@ -20,14 +20,14 @@ dependencies:
   - flake8<3.8.0  # temporary pin, GH#34150
   - flake8-comprehensions>=3.1.0  # used by flake8, linting of unnecessary comprehensions
   - flake8-rst>=0.6.0,<=0.7.0  # linting of code blocks in rst files
-  - isort  # check that imports are in the right order
+  - isort=4.3.21  # check that imports are in the right order
   - mypy=0.730
   - pycodestyle  # used by flake8
 
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - gitdb2=2.0.6  # GH-32060
-  - sphinx
+  - sphinx<=3.1.1
 
   # documentation (jupyter notebooks)
   - nbconvert>=5.4.1

environment.yml

@@ -73,7 +73,7 @@ from pandas._libs.tslibs.nattype cimport (
 )
 from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
-from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare
+from pandas._libs.tslibs.timezones cimport tz_compare
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.offsets cimport is_offset_object
 
@@ -1789,7 +1789,7 @@ def is_datetime_with_singletz_array(values: ndarray) -> bool:
     for i in range(n):
         base_val = values[i]
         if base_val is not NaT:
-            base_tz = get_timezone(getattr(base_val, 'tzinfo', None))
+            base_tz = getattr(base_val, 'tzinfo', None)
             break
 
     for j in range(i, n):