merge master

* master: (47 commits)
  Run tests in conda build [ci skip] (pandas-dev#22190)
  TST: Check DatetimeIndex.drop on DST boundary (pandas-dev#22165)
  CI: Fix Travis failures due to lint.sh on pandas/core/strings.py (pandas-dev#22184)
  Documentation: typo fixes in MultiIndex / Advanced Indexing (pandas-dev#22179)
  DOC: added .join to 'see also' in Series.str.cat (pandas-dev#22175)
  DOC: updated Series.str.contains see also section (pandas-dev#22176)
  0.23.4 whatsnew (pandas-dev#22177)
  fix: scalar timestamp assignment (pandas-dev#19843) (pandas-dev#19973)
  BUG: Fix get dummies unicode error (pandas-dev#22131)
  Fixed py36-only syntax [ci skip] (pandas-dev#22167)
  DEPR: pd.read_table (pandas-dev#21954)
  DEPR: Removing previously deprecated datetools module (pandas-dev#6581) (pandas-dev#19119)
  BUG: Matplotlib scatter datetime (pandas-dev#22039)
  CLN: Use public method to capture UTC offsets (pandas-dev#22164)
  implement tslibs/src to make tslibs self-contained (pandas-dev#22152)
  Fix categorical from codes nan 21767 (pandas-dev#21775)
  BUG: Better handling of invalid na_option argument for groupby.rank(pandas-dev#22124) (pandas-dev#22125)
  use memoryviews instead of ndarrays (pandas-dev#22147)
  Remove depr. warning in SeriesGroupBy.count (pandas-dev#22155)
  API: Default to_* methods to compression='infer' (pandas-dev#22011)
  ...

Author: minggli

asv_bench/benchmarks/reshape.py

@@ -141,7 +141,7 @@ class GetDummies(object):
 
     def setup(self):
         categories = list(string.ascii_letters[:12])
-        s = pd.Series(np.random.choice(categories, size=1_000_000),
+        s = pd.Series(np.random.choice(categories, size=1000000),
                       dtype=pd.api.types.CategoricalDtype(categories))
         self.s = s
 

asv_bench/benchmarks/timeseries.py

@@ -343,6 +343,25 @@ def time_iso8601_tz_spaceformat(self):
         to_datetime(self.strings_tz_space)
 
 
+class ToDatetimeNONISO8601(object):
+
+    goal_time = 0.2
+
+    def setup(self):
+        N = 10000
+        half = int(N / 2)
+        ts_string_1 = 'March 1, 2018 12:00:00+0400'
+        ts_string_2 = 'March 1, 2018 12:00:00+0500'
+        self.same_offset = [ts_string_1] * N
+        self.diff_offset = [ts_string_1] * half + [ts_string_2] * half
+
+    def time_same_offset(self):
+        to_datetime(self.same_offset)
+
+    def time_different_offset(self):
+        to_datetime(self.diff_offset)
+
+
 class ToDatetimeFormat(object):
 
     goal_time = 0.2

ci/environment-dev.yaml

@@ -6,6 +6,7 @@ dependencies:
   - Cython>=0.28.2
   - NumPy
   - flake8
+  - flake8-comprehensions
   - moto
   - pytest>=3.1
   - python-dateutil>=2.5.0

ci/lint.sh

@@ -8,44 +8,55 @@ RET=0
 
 if [ "$LINT" ]; then
 
+    # We're ignoring the following codes across the board
+    #E402,  # module level import not at top of file
+    #E731,  # do not assign a lambda expression, use a def
+    #E741,  # do not use variables named 'l', 'O', or 'I'
+    #W503,  # line break before binary operator
+    #C405,  # Unnecessary (list/tuple) literal - rewrite as a set literal.
+    #C406,  # Unnecessary (list/tuple) literal - rewrite as a dict literal.
+    #C408,  # Unnecessary (dict/list/tuple) call - rewrite as a literal.
+    #C409,  # Unnecessary (list/tuple) passed to tuple() - (remove the outer call to tuple()/rewrite as a tuple literal).
+    #C410   # Unnecessary (list/tuple) passed to list() - (remove the outer call to list()/rewrite as a list literal).
+
     # pandas/_libs/src is C code, so no need to search there.
     echo "Linting *.py"
-    flake8 pandas --filename=*.py --exclude pandas/_libs/src
+    flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting *.py DONE"
 
     echo "Linting setup.py"
-    flake8 setup.py
+    flake8 setup.py --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting setup.py DONE"
 
     echo "Linting asv_bench/benchmarks/"
-    flake8 asv_bench/benchmarks/  --exclude=asv_bench/benchmarks/*.py --ignore=F811
+    flake8 asv_bench/benchmarks/  --exclude=asv_bench/benchmarks/*.py --ignore=F811,C405,C406,C408,C409,C410
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting asv_bench/benchmarks/*.py DONE"
 
     echo "Linting scripts/*.py"
-    flake8 scripts --filename=*.py
+    flake8 scripts --filename=*.py --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting scripts/*.py DONE"
 
     echo "Linting doc scripts"
-    flake8 doc/make.py doc/source/conf.py
+    flake8 doc/make.py doc/source/conf.py --ignore=C405,C406,C408,C409,C410,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting doc scripts DONE"
 
     echo "Linting *.pyx"
-    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403
+    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C407,C411
     if [ $? -ne "0" ]; then
         RET=1
     fi
@@ -131,19 +142,6 @@ if [ "$LINT" ]; then
     fi
     echo "Check for non-standard imports DONE"
 
-    echo "Check for use of lists instead of generators in built-in Python functions"
-
-    # Example: Avoid `any([i for i in some_iterator])` in favor of `any(i for i in some_iterator)`
-    #
-    # Check the following functions:
-    # any(), all(), sum(), max(), min(), list(), dict(), set(), frozenset(), tuple(), str.join()
-    grep -R --include="*.py*" -E "[^_](any|all|sum|max|min|list|dict|set|frozenset|tuple|join)\(\[.* for .* in .*\]\)" pandas
-
-    if [ $? = "0" ]; then
-        RET=1
-    fi
-    echo "Check for use of lists instead of generators in built-in Python functions DONE"
-
     echo "Check for incorrect sphinx directives"
     SPHINX_DIRECTIVES=$(echo \
        "autosummary|contents|currentmodule|deprecated|function|image|"\

ci/requirements_dev.txt

@@ -1,8 +1,9 @@
 # This file was autogenerated by scripts/convert_deps.py
 # Do not modify directly
-Cython
+Cython>=0.28.2
 NumPy
 flake8
+flake8-comprehensions
 moto
 pytest>=3.1
 python-dateutil>=2.5.0

ci/travis-27.yaml

@@ -9,6 +9,7 @@ dependencies:
   - fastparquet
   - feather-format
   - flake8=3.4.1
+  - flake8-comprehensions
   - gcsfs
   - html5lib
   - ipython

conda.recipe/meta.yaml

@@ -29,8 +29,11 @@ requirements:
     - pytz
 
 test:
-  imports:
-    - pandas
+  requires:
+    - pytest
+  commands:
+    - python -c "import pandas; pandas.test()"
+
 
 about:
   home: http://pandas.pydata.org

doc/cheatsheet/Pandas_Cheat_Sheet_JP.pdf

@@ -21,7 +21,7 @@ See the :ref:`Indexing and Selecting Data <indexing>` for general indexing docum
 
 .. warning::
 
-   Whether a copy or a reference is returned for a setting operation, may
+   Whether a copy or a reference is returned for a setting operation may
    depend on the context.  This is sometimes called ``chained assignment`` and
    should be avoided.  See :ref:`Returning a View versus Copy
    <indexing.view_versus_copy>`.
@@ -172,7 +172,7 @@ Defined Levels
 ~~~~~~~~~~~~~~
 
 The repr of a ``MultiIndex`` shows all the defined levels of an index, even
-if the they are not actually used. When slicing an index, you may notice this.
+if they are not actually used. When slicing an index, you may notice this.
 For example:
 
 .. ipython:: python
@@ -379,7 +379,7 @@ slicers on a single axis.
 
    dfmi.loc(axis=0)[:, :, ['C1', 'C3']]
 
-Furthermore you can *set* the values using the following methods.
+Furthermore, you can *set* the values using the following methods.
 
 .. ipython:: python
 
@@ -559,7 +559,7 @@ return a copy of the data rather than a view:
 
 .. _advanced.unsorted:
 
-Furthermore if you try to index something that is not fully lexsorted, this can raise:
+Furthermore, if you try to index something that is not fully lexsorted, this can raise:
 
 .. code-block:: ipython
 
@@ -659,7 +659,7 @@ Index Types
 
 We have discussed ``MultiIndex`` in the previous sections pretty extensively. ``DatetimeIndex`` and ``PeriodIndex``
 are shown :ref:`here <timeseries.overview>`, and information about
-`TimedeltaIndex`` is found :ref:`here <timedeltas.timedeltas>`.
+``TimedeltaIndex`` is found :ref:`here <timedeltas.timedeltas>`.
 
 In the following sub-sections we will highlight some other index types.
 
@@ -835,8 +835,8 @@ In non-float indexes, slicing using floats will raise a ``TypeError``.
 
 
 Here is a typical use-case for using this type of indexing. Imagine that you have a somewhat
-irregular timedelta-like indexing scheme, but the data is recorded as floats. This could for
-example be millisecond offsets.
+irregular timedelta-like indexing scheme, but the data is recorded as floats. This could, for
+example, be millisecond offsets.
 
 .. ipython:: python
 

doc/cheatsheet/Pandas_Cheat_Sheet_JP.pptx

@@ -298,7 +298,7 @@ compression : {``'infer'``, ``'gzip'``, ``'bz2'``, ``'zip'``, ``'xz'``, ``None``
   Set to ``None`` for no decompression.
 
   .. versionadded:: 0.18.1 support for 'zip' and 'xz' compression.
-
+  .. versionchanged:: 0.24.0 'infer' option added and set to default.
 thousands : str, default ``None``
   Thousands separator.
 decimal : str, default ``'.'``

doc/source/advanced.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0234:
 
-v0.23.4
--------
+v0.23.4 (August 3, 2018)
+------------------------
 
 This is a minor bug-fix release in the 0.23.x series and includes some small regression fixes
 and bug fixes. We recommend that all users upgrade to this version.
@@ -21,7 +21,6 @@ Fixed Regressions
 ~~~~~~~~~~~~~~~~~
 
 - Python 3.7 with Windows gave all missing values for rolling variance calculations (:issue:`21813`)
--
 
 .. _whatsnew_0234.bug_fixes:
 
@@ -32,37 +31,6 @@ Bug Fixes
 
 - Bug where calling :func:`DataFrameGroupBy.agg` with a list of functions including ``ohlc`` as the non-initial element would raise a ``ValueError`` (:issue:`21716`)
 - Bug in ``roll_quantile`` caused a memory leak when calling ``.rolling(...).quantile(q)`` with ``q`` in (0,1) (:issue:`21965`)
--
-
-**Conversion**
-
--
--
-
-**Indexing**
-
--
--
-
-**I/O**
-
--
--
-
-**Categorical**
-
--
--
-
-**Timezones**
-
--
--
-
-**Timedelta**
-
--
--
 
 **Missing**