Merge branch 'master' into catDtype_copy_nan_codes

Author: ghasemnaddaf

.travis.yml

@@ -52,10 +52,6 @@ matrix:
     - dist: trusty
       env:
         - JOB="3.5" TEST_ARGS="--skip-slow --skip-network" COVERAGE=true
-      addons:
-        apt:
-          packages:
-          - xsel
     - dist: trusty
       env:
         - JOB="3.6" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true
@@ -66,7 +62,11 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
+        - JOB="3.6_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
+      addons:
+        apt:
+          packages:
+          - xsel
     # In allow_failures
     - dist: trusty
       env:
@@ -75,17 +75,17 @@ matrix:
     - dist: trusty
       env:
         - JOB="3.6_DOC" DOC=true
-      addons:
-        apt:
-          packages:
-          - xsel
     allow_failures:
       - dist: trusty
         env:
           - JOB="2.7_SLOW" SLOW=true
       - dist: trusty
         env:
-          - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
+          - JOB="3.6_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
+        addons:
+          apt:
+            packages:
+            - xsel
       - dist: trusty
         env:
           - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
@@ -102,8 +102,6 @@ before_install:
   - uname -a
   - git --version
   - git tag
-  - ci/before_install_travis.sh
-  - export DISPLAY=":99.0"
 
 install:
   - echo "install start"
@@ -114,6 +112,8 @@ install:
 
 before_script:
   - ci/install_db_travis.sh
+  - export DISPLAY=":99.0"
+  - ci/before_script_travis.sh
 
 script:
   - echo "script start"

asv_bench/benchmarks/timedelta.py

@@ -40,3 +40,46 @@ def setup(self):
 
     def test_add_td_ts(self):
         self.td + self.ts
+
+
+class TimedeltaProperties(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.td = Timedelta(days=365, minutes=35, seconds=25, milliseconds=35)
+
+    def time_timedelta_days(self):
+        self.td.days
+
+    def time_timedelta_seconds(self):
+        self.td.seconds
+
+    def time_timedelta_microseconds(self):
+        self.td.microseconds
+
+    def time_timedelta_nanoseconds(self):
+        self.td.nanoseconds
+
+
+class DatetimeAccessor(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.N = 100000
+        self.series = pd.Series(
+            pd.timedelta_range('1 days', periods=self.N, freq='h')
+        )
+    def time_dt_accessor(self):
+        self.series.dt
+
+    def time_timedelta_dt_accessor_days(self):
+        self.series.dt.days
+
+    def time_timedelta_dt_accessor_seconds(self):
+        self.series.dt.seconds
+
+    def time_timedelta_dt_accessor_microseconds(self):
+        self.series.dt.microseconds
+
+    def time_timedelta_dt_accessor_nanoseconds(self):
+        self.series.dt.nanoseconds

asv_bench/benchmarks/timeseries.py

@@ -346,17 +346,22 @@ class ToDatetime(object):
 
     def setup(self):
         self.rng = date_range(start='1/1/2000', periods=10000, freq='D')
-        self.stringsD = Series((((self.rng.year * 10000) + (self.rng.month * 100)) + self.rng.day), dtype=np.int64).apply(str)
+        self.stringsD = Series(self.rng.strftime('%Y%m%d'))
 
         self.rng = date_range(start='1/1/2000', periods=20000, freq='H')
-        self.strings = [x.strftime('%Y-%m-%d %H:%M:%S') for x in self.rng]
-        self.strings_nosep = [x.strftime('%Y%m%d %H:%M:%S') for x in self.rng]
+        self.strings = self.rng.strftime('%Y-%m-%d %H:%M:%S').tolist()
+        self.strings_nosep = self.rng.strftime('%Y%m%d %H:%M:%S').tolist()
         self.strings_tz_space = [x.strftime('%Y-%m-%d %H:%M:%S') + ' -0800'
                                  for x in self.rng]
 
         self.s = Series((['19MAY11', '19MAY11:00:00:00'] * 100000))
         self.s2 = self.s.str.replace(':\\S+$', '')
 
+        self.unique_numeric_seconds = range(10000)
+        self.dup_numeric_seconds = [1000] * 10000
+        self.dup_string_dates = ['2000-02-11'] * 10000
+        self.dup_string_with_tz = ['2000-02-11 15:00:00-0800'] * 10000
+
     def time_format_YYYYMMDD(self):
         to_datetime(self.stringsD, format='%Y%m%d')
 
@@ -381,6 +386,36 @@ def time_format_exact(self):
     def time_format_no_exact(self):
         to_datetime(self.s, format='%d%b%y', exact=False)
 
+    def time_cache_true_with_unique_seconds_and_unit(self):
+        to_datetime(self.unique_numeric_seconds, unit='s', cache=True)
+
+    def time_cache_false_with_unique_seconds_and_unit(self):
+        to_datetime(self.unique_numeric_seconds, unit='s', cache=False)
+
+    def time_cache_true_with_dup_seconds_and_unit(self):
+        to_datetime(self.dup_numeric_seconds, unit='s', cache=True)
+
+    def time_cache_false_with_dup_seconds_and_unit(self):
+        to_datetime(self.dup_numeric_seconds, unit='s', cache=False)
+
+    def time_cache_true_with_dup_string_dates(self):
+        to_datetime(self.dup_string_dates, cache=True)
+
+    def time_cache_false_with_dup_string_dates(self):
+        to_datetime(self.dup_string_dates, cache=False)
+
+    def time_cache_true_with_dup_string_dates_and_format(self):
+        to_datetime(self.dup_string_dates, format='%Y-%m-%d', cache=True)
+
+    def time_cache_false_with_dup_string_dates_and_format(self):
+        to_datetime(self.dup_string_dates, format='%Y-%m-%d', cache=False)
+
+    def time_cache_true_with_dup_string_tzoffset_dates(self):
+        to_datetime(self.dup_string_with_tz, cache=True)
+
+    def time_cache_false_with_dup_string_tzoffset_dates(self):
+        to_datetime(self.dup_string_with_tz, cache=False)
+
 
 class Offsets(object):
     goal_time = 0.2

ci/before_install_travis.sh renamed to ci/before_script_travis.sh

@@ -4,6 +4,7 @@ echo "inside $0"
 
 if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
    sh -e /etc/init.d/xvfb start
+   sleep 3
 fi
 
 # Never fail because bad things happened here.

ci/requirements-2.7_BUILD_TEST.build

@@ -0,0 +1,6 @@
+python=3.6*
+python-dateutil
+pytz
+nomkl
+numpy
+cython

ci/requirements-3.6_BUILD_TEST.build

@@ -2,6 +2,6 @@
 
 source activate pandas
 
-echo "install 27 BUILD_TEST"
+echo "install 36 BUILD_TEST"
 
 conda install -n pandas -c conda-forge pyarrow dask

ci/requirements-2.7_BUILD_TEST.pip renamed to ci/requirements-3.6_BUILD_TEST.pip

@@ -12,10 +12,7 @@ PRE_WHEELS="https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf
 pip install --pre --upgrade --timeout=60 -f $PRE_WHEELS numpy scipy
 
 # install dateutil from master
-
-# TODO(jreback), temp disable dateutil master has changed
-# pip install -U git+git://github.com/dateutil/dateutil.git
-pip install python-dateutil
+pip install -U git+git://github.com/dateutil/dateutil.git
 
 # cython via pip
 pip install cython

ci/requirements-2.7_BUILD_TEST.sh renamed to ci/requirements-3.6_BUILD_TEST.sh

@@ -27,6 +27,11 @@ if [ "$BUILD_TEST" ]; then
     echo "[running]"
     cd /tmp
     unset PYTHONPATH
+
+    echo "[build-test: single]"
+    python -c 'import pandas; pandas.test(["--skip-slow", "--skip-network", "-r xX", "-m single"])'
+
+    echo "[build-test: not single]"
     python -c 'import pandas; pandas.test(["-n 2", "--skip-slow", "--skip-network", "-r xX", "-m not single"])'
 
 elif [ "$DOC" ]; then

ci/requirements-3.6_NUMPY_DEV.build.sh

@@ -1870,8 +1870,52 @@ Methods
     Timedelta.to_timedelta64
     Timedelta.total_seconds
 
+.. _api.frequencies:
+
+Frequencies
+-----------
+
+.. currentmodule:: pandas.tseries.frequencies
+
+
+.. autosummary::
+   :toctree: generated/
+
+   to_offset
+
+.. _api.offsets:
+
+Offsets
+-------
+
+.. currentmodule:: pandas.tseries.offsets
+
+.. autosummary::
+   :toctree: generated/
+
+   DateOffset
+   Week
+   Day
+   Hour
+   Minute
+   Second
+   Milli
+   Micro
+   Nano
+
+.. autosummary::
+   :toctree: generated/
+
+   MonthBegin
+   MonthEnd
+   QuarterBegin
+   QuarterEnd
+   YearBegin
+   YearEnd
+
 Window
 ------
+
 .. currentmodule:: pandas.core.window
 
 Rolling objects are returned by ``.rolling`` calls: :func:`pandas.DataFrame.rolling`, :func:`pandas.Series.rolling`, etc.

ci/script_multi.sh

@@ -4427,8 +4427,10 @@ Several caveats.
 
 - This is a newer library, and the format, though stable, is not guaranteed to be backward compatible
   to the earlier versions.
-- The format will NOT write an ``Index``, or ``MultiIndex`` for the ``DataFrame`` and will raise an
-  error if a non-default one is provided. You can simply ``.reset_index()`` in order to store the index.
+- The format will NOT write an ``Index``, or ``MultiIndex`` for the
+  ``DataFrame`` and will raise an error if a non-default one is provided. You
+  can ``.reset_index()`` to store the index or ``.reset_index(drop=True)`` to
+  ignore it.
 - Duplicate column names and non-string columns names are not supported
 - Non supported types include ``Period`` and actual python object types. These will raise a helpful error message
   on an attempt at serialization.
@@ -4491,8 +4493,10 @@ dtypes, including extension dtypes such as datetime with tz.
 
 Several caveats.
 
-- The format will NOT write an ``Index``, or ``MultiIndex`` for the ``DataFrame`` and will raise an
-  error if a non-default one is provided. You can simply ``.reset_index(drop=True)`` in order to store the index.
+- The format will NOT write an ``Index``, or ``MultiIndex`` for the
+  ``DataFrame`` and will raise an error if a non-default one is provided. You
+  can ``.reset_index()`` to store the index or ``.reset_index(drop=True)`` to
+  ignore it.
 - Duplicate column names and non-string columns names are not supported
 - Categorical dtypes can be serialized to parquet, but will de-serialize as ``object`` dtype.
 - Non supported types include ``Period`` and actual python object types. These will raise a helpful error message
@@ -4538,7 +4542,7 @@ Read from a parquet file.
 
    result.dtypes
 
-Read only certain columns of a parquet file. 
+Read only certain columns of a parquet file.
 
 .. ipython:: python
 

doc/source/api.rst

@@ -52,7 +52,7 @@ Highlights include:
 - Integration with `Apache Parquet <https://parquet.apache.org/>`__, including a new top-level :func:`read_parquet` function and :meth:`DataFrame.to_parquet` method, see :ref:`here <whatsnew_0210.enhancements.parquet>`.
 - New user-facing :class:`pandas.api.types.CategoricalDtype` for specifying
   categoricals independent of the data, see :ref:`here <whatsnew_0210.enhancements.categorical_dtype>`.
-- The behavior of ``sum`` and ``prod`` on all-NaN Series/DataFrames is now consistent and no longer depends on whether `bottleneck <http://berkeleyanalytics.com/bottleneck>`__ is installed, see :ref:`here <whatsnew_0210.api_breaking.bottleneck>`.
+- The behavior of ``sum`` and ``prod`` on all-NaN Series/DataFrames is now consistent and no longer depends on whether `bottleneck <http://berkeleyanalytics.com/bottleneck>`__ is installed, and ``sum`` and ``prod`` on empty Series now return NaN instead of 0, see :ref:`here <whatsnew_0210.api_breaking.bottleneck>`.
 - Compatibility fixes for pypy, see :ref:`here <whatsnew_0210.pypy>`.
 - Additions to the ``drop``, ``reindex`` and ``rename`` API to make them more consistent, see :ref:`here <whatsnew_0210.enhancements.drop_api>`.
 - Addition of the new methods ``DataFrame.infer_objects`` (see :ref:`here <whatsnew_0210.enhancements.infer_objects>`) and ``GroupBy.pipe`` (see :ref:`here <whatsnew_0210.enhancements.GroupBy_pipe>`).

doc/source/io.rst

@@ -18,6 +18,10 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.22.0.txt
+
+.. include:: whatsnew/v0.21.1.txt
+
 .. include:: whatsnew/v0.21.0.txt
 
 .. include:: whatsnew/v0.20.3.txt