Merge branch 'master' into issue4889

Author: Dr-Irv

.github/actions/build_pandas/action.yml

@@ -13,5 +13,5 @@ runs:
     - name: Build Pandas
       run: |
         python setup.py build_ext -j 2
-        python -m pip install -e . --no-build-isolation --no-use-pep517
+        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -l {0}

.github/workflows/python-dev.yml

@@ -21,12 +21,20 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
     name: actions-310-dev
     timeout-minutes: 60
 
+    env:
+      NUMPY_WHEELS_AVAILABLE: ${{ matrix.os == 'ubuntu-latest' }}
+
     concurrency:
-      group: ${{ github.ref }}-dev
+      group: ${{ github.ref }}-${{ matrix.os }}-dev
       cancel-in-progress: ${{github.event_name == 'pull_request'}}
 
     steps:
@@ -40,12 +48,16 @@ jobs:
         python-version: '3.10-dev'
 
     - name: Install dependencies
+      shell: bash
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
-        pip install git+https://github.com/pytest-dev/pytest.git
+        if [[ "$NUMPY_WHEELS_AVAILABLE" == "true" ]]; then
+          pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
+        else
+          pip install git+https://github.com/numpy/numpy.git
+        fi
         pip install git+https://github.com/nedbat/coveragepy.git
-        pip install cython python-dateutil pytz hypothesis pytest-xdist pytest-cov
+        pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov
         pip list
 
     - name: Build Pandas
@@ -58,10 +70,9 @@ jobs:
         python -c "import pandas; pandas.show_versions();"
 
     - name: Test with pytest
+      shell: bash
       run: |
         ci/run_tests.sh
-      # GH 41935
-      continue-on-error: true
 
     - name: Publish test results
       uses: actions/upload-artifact@master

.github/workflows/sdist.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.8", "3.9", "3.10"]
     concurrency:
       group: ${{github.ref}}-${{matrix.python-version}}-sdist
       cancel-in-progress: ${{github.event_name == 'pull_request'}}
@@ -53,13 +53,24 @@ jobs:
     - uses: conda-incubator/setup-miniconda@v2
       with:
         activate-environment: pandas-sdist
-        python-version: ${{ matrix.python-version }}
+        python-version: '${{ matrix.python-version }}'
 
     - name: Install pandas from sdist
       run: |
-        conda list
+        pip list
         python -m pip install dist/*.gz
 
+    - name: Force oldest supported NumPy
+      run: |
+        case "${{matrix.python-version}}" in
+        3.8)
+          pip install numpy==1.18.5 ;;
+        3.9)
+          pip install numpy==1.19.3 ;;
+        3.10)
+          pip install numpy==1.21.2 ;;
+        esac
+
     - name: Import pandas
       run: |
         cd ..

.pre-commit-config.yaml

@@ -135,6 +135,12 @@ repos:
         entry: 'np\.random\.seed'
         files: ^asv_bench/benchmarks
         exclude: ^asv_bench/benchmarks/pandas_vb_common\.py
+    -   id: np-testing-array-equal
+        name: Check for usage of numpy testing or array_equal
+        language: pygrep
+        entry: '(numpy|np)(\.testing|\.array_equal)'
+        files: ^pandas/tests/
+        types: [python]
     -   id: invalid-ea-testing
         name: Check for invalid EA testing
         language: pygrep

asv_bench/benchmarks/indexing_engines.py

@@ -35,25 +35,49 @@ class NumericEngineIndexing:
     params = [
         _get_numeric_engines(),
         ["monotonic_incr", "monotonic_decr", "non_monotonic"],
+        [True, False],
+        [10 ** 5, 2 * 10 ** 6],  # 2e6 is above SIZE_CUTOFF
     ]
-    param_names = ["engine_and_dtype", "index_type"]
+    param_names = ["engine_and_dtype", "index_type", "unique", "N"]
 
-    def setup(self, engine_and_dtype, index_type):
+    def setup(self, engine_and_dtype, index_type, unique, N):
         engine, dtype = engine_and_dtype
-        N = 10 ** 5
-        values = list([1] * N + [2] * N + [3] * N)
-        arr = {
-            "monotonic_incr": np.array(values, dtype=dtype),
-            "monotonic_decr": np.array(list(reversed(values)), dtype=dtype),
-            "non_monotonic": np.array([1, 2, 3] * N, dtype=dtype),
-        }[index_type]
+
+        if index_type == "monotonic_incr":
+            if unique:
+                arr = np.arange(N * 3, dtype=dtype)
+            else:
+                values = list([1] * N + [2] * N + [3] * N)
+                arr = np.array(values, dtype=dtype)
+        elif index_type == "monotonic_decr":
+            if unique:
+                arr = np.arange(N * 3, dtype=dtype)[::-1]
+            else:
+                values = list([1] * N + [2] * N + [3] * N)
+                arr = np.array(values, dtype=dtype)[::-1]
+        else:
+            assert index_type == "non_monotonic"
+            if unique:
+                arr = np.empty(N * 3, dtype=dtype)
+                arr[:N] = np.arange(N * 2, N * 3, dtype=dtype)
+                arr[N:] = np.arange(N * 2, dtype=dtype)
+            else:
+                arr = np.array([1, 2, 3] * N, dtype=dtype)
 
         self.data = engine(arr)
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc(2)
 
-    def time_get_loc(self, engine_and_dtype, index_type):
-        self.data.get_loc(2)
+        self.key_middle = arr[len(arr) // 2]
+        self.key_early = arr[2]
+
+    def time_get_loc(self, engine_and_dtype, index_type, unique, N):
+        self.data.get_loc(self.key_early)
+
+    def time_get_loc_near_middle(self, engine_and_dtype, index_type, unique, N):
+        # searchsorted performance may be different near the middle of a range
+        #  vs near an endpoint
+        self.data.get_loc(self.key_middle)
 
 
 class ObjectEngineIndexing:

asv_bench/benchmarks/io/csv.py

@@ -10,6 +10,7 @@
 from pandas import (
     Categorical,
     DataFrame,
+    concat,
     date_range,
     read_csv,
     to_datetime,
@@ -459,6 +460,34 @@ def time_read_special_date(self, value, engine):
         )
 
 
+class ReadCSVMemMapUTF8:
+
+    fname = "__test__.csv"
+    number = 5
+
+    def setup(self):
+        lines = []
+        line_length = 128
+        start_char = " "
+        end_char = "\U00010080"
+        # This for loop creates a list of 128-char strings
+        # consisting of consecutive Unicode chars
+        for lnum in range(ord(start_char), ord(end_char), line_length):
+            line = "".join([chr(c) for c in range(lnum, lnum + 0x80)]) + "\n"
+            try:
+                line.encode("utf-8")
+            except UnicodeEncodeError:
+                # Some 16-bit words are not valid Unicode chars and must be skipped
+                continue
+            lines.append(line)
+        df = DataFrame(lines)
+        df = concat([df for n in range(100)], ignore_index=True)
+        df.to_csv(self.fname, index=False, header=False, encoding="utf-8")
+
+    def time_read_memmapped_utf8(self):
+        read_csv(self.fname, header=None, memory_map=True, encoding="utf-8", engine="c")
+
+
 class ParseDateComparison(StringIORewind):
     params = ([False, True],)
     param_names = ["cache_dates"]

ci/deps/actions-38-db-min.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-db.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3

ci/deps/actions-38-locale.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-locale_slow.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-minimum_versions.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8.0
 
   # tools
-  - cython=0.29.21
+  - cython=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38-slow.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-38.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-39-numpydev.yaml

@@ -15,7 +15,7 @@ dependencies:
   - pytz
   - pip
   - pip:
-    - cython==0.29.21 # GH#34014
+    - cython==0.29.24 # GH#34014
     - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
     - "numpy"

ci/deps/actions-39-slow.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.9
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/actions-39.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.9
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31

ci/deps/azure-macos-38.yaml

@@ -32,6 +32,6 @@ dependencies:
   - xlwt
   - pip
   - pip:
-    - cython>=0.29.21
+    - cython>=0.29.24
     - pyreadstat
     - pyxlsb

ci/deps/azure-windows-38.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3

ci/deps/azure-windows-39.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.9
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3

ci/deps/circle-38-arm64.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8
 
   # tools
-  - cython>=0.29.21
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3

doc/source/reference/extensions.rst

@@ -60,6 +60,7 @@ objects.
       api.extensions.ExtensionArray.nbytes
       api.extensions.ExtensionArray.ndim
       api.extensions.ExtensionArray.shape
+      api.extensions.ExtensionArray.tolist
 
 Additionally, we have some utility methods for ensuring your object
 behaves correctly.

doc/source/reference/window.rst

@@ -88,6 +88,7 @@ Exponentially-weighted window functions
    :toctree: api/
 
    ExponentialMovingWindow.mean
+   ExponentialMovingWindow.sum
    ExponentialMovingWindow.std
    ExponentialMovingWindow.var
    ExponentialMovingWindow.corr

doc/source/user_guide/indexing.rst

@@ -997,6 +997,15 @@ a list of items you want to check for.
 
    df.isin(values)
 
+To return the DataFrame of booleans where the values are *not* in the original DataFrame,
+use the ``~`` operator:
+
+.. ipython:: python
+
+   values = {'ids': ['a', 'b'], 'vals': [1, 3]}
+
+   ~df.isin(values)
+
 Combine DataFrame's ``isin`` with the ``any()`` and ``all()`` methods to
 quickly select subsets of your data that meet a given criteria.
 To select a row where each column meets its own criterion: