Merge remote-tracking branch 'upstream/main' into td64-tests

Author: patrickmckenna

.github/actions/build_pandas/action.yml

@@ -12,6 +12,9 @@ runs:
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j 2
+        python setup.py build_ext -j $N_JOBS
         python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -el {0}
+      env:
+        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
+        N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}

.github/workflows/code-checks.yml

@@ -74,7 +74,7 @@ jobs:
 
     - name: Install pyright
       # note: keep version in sync with .pre-commit-config.yaml
-      run: npm install -g [email protected].230
+      run: npm install -g [email protected].245
 
     - name: Build Pandas
       id: build

.github/workflows/posix.yml

@@ -180,7 +180,6 @@ jobs:
       run: ci/run_tests.sh
       # TODO: Don't continue on error for PyPy
       continue-on-error: ${{ env.IS_PYPY == 'true' }}
-      if: always()
 
     - name: Build Version
       run: conda list

.github/workflows/windows.yml

@@ -0,0 +1,75 @@
+name: Windows
+
+on:
+  push:
+    branches:
+      - main
+      - 1.4.x
+  pull_request:
+    branches:
+      - main
+      - 1.4.x
+    paths-ignore:
+      - "doc/**"
+
+env:
+  PANDAS_CI: 1
+  PYTEST_TARGET: pandas
+  PYTEST_WORKERS: auto
+  PATTERN: "not slow and not db and not network and not single_cpu"
+
+
+jobs:
+  pytest:
+    runs-on: windows-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+      fail-fast: false
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-windows
+      cancel-in-progress: true
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Install Dependencies
+      uses: conda-incubator/[email protected]
+      with:
+        mamba-version: "*"
+        channels: conda-forge
+        activate-environment: pandas-dev
+        channel-priority: strict
+        environment-file: ci/deps/${{ matrix.env_file }}
+        use-only-tar-bz2: true
+
+    - name: Build Pandas
+      uses: ./.github/actions/build_pandas
+
+    - name: Test
+      run: ci/run_tests.sh
+
+    - name: Build Version
+      run: conda list
+
+    - name: Publish test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: Test results
+        path: test-data.xml
+      if: failure()
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        flags: unittests
+        name: codecov-pandas
+        fail_ci_if_error: false

.pre-commit-config.yaml

@@ -89,7 +89,7 @@ repos:
         types: [python]
         stages: [manual]
         # note: keep version in sync with .github/workflows/code-checks.yml
-        additional_dependencies: ['[email protected].230']
+        additional_dependencies: ['[email protected].245']
 -   repo: local
     hooks:
     -   id: flake8-rst

asv_bench/benchmarks/tslibs/fields.py

@@ -66,9 +66,9 @@ class TimeGetStartEndField:
 
     def setup(self, size, side, period, freqstr, month_kw):
         arr = np.random.randint(0, 10, size=size, dtype="i8")
-        self.dt64data = arr.view("M8[ns]")
+        self.i8data = arr
 
         self.attrname = f"is_{period}_{side}"
 
     def time_get_start_end_field(self, size, side, period, freqstr, month_kw):
-        get_start_end_field(self.dt64data, self.attrname, freqstr, month_kw=month_kw)
+        get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw)

azure-pipelines.yml

@@ -27,11 +27,6 @@ jobs:
     name: macOS
     vmImage: macOS-10.15
 
-- template: ci/azure/windows.yml
-  parameters:
-    name: Windows
-    vmImage: windows-2019
-
 - job: py38_32bit
   pool:
     vmImage: ubuntu-18.04

ci/azure/windows.yml

@@ -26,6 +26,6 @@ consists of the following data columns:
 .. raw:: html
 
             </p>
-            <a href="https://github.com/pandas-dev/pandas/tree/main/doc/data/titanic.csv" class="btn btn-dark btn-sm">To raw data</a>
+            <a href="https://github.com/pandas-dev/pandas/raw/main/doc/data/titanic.csv" class="btn btn-dark btn-sm">To raw data</a>
         </div>
     </div>

doc/source/getting_started/intro_tutorials/includes/titanic.rst

@@ -511,7 +511,7 @@ Unlike agg, apply's callable is passed a sub-DataFrame which gives you access to
 
    def replace(g):
        mask = g < 0
-       return g.where(mask, g[~mask].mean())
+       return g.where(~mask, g[~mask].mean())
 
    gb.transform(replace)
 

doc/source/user_guide/cookbook.rst

@@ -100,6 +100,31 @@ as seen in the following example.
                1 2021-01-02 08:00:00  4
                2 2021-01-02 16:00:00  5
 
+.. _whatsnew_150.enhancements.tar:
+
+Reading directly from TAR archives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+I/O methods like :func:`read_csv` or :meth:`DataFrame.to_json` now allow reading and writing
+directly on TAR archives (:issue:`44787`).
+
+.. code-block:: python
+
+   df = pd.read_csv("./movement.tar.gz")
+   # ...
+   df.to_csv("./out.tar.gz")
+
+This supports ``.tar``, ``.tar.gz``, ``.tar.bz`` and ``.tar.xz2`` archives.
+The used compression method is inferred from the filename.
+If the compression method cannot be inferred, use the ``compression`` argument:
+
+.. code-block:: python
+
+   df = pd.read_csv(some_file_obj, compression={"method": "tar", "mode": "r:gz"}) # noqa F821
+
+(``mode`` being one of ``tarfile.open``'s modes: https://docs.python.org/3/library/tarfile.html#tarfile.open)
+
+
 .. _whatsnew_150.enhancements.other:
 
 Other enhancements

doc/source/whatsnew/v1.5.0.rst

@@ -3,33 +3,33 @@
 __docformat__ = "restructuredtext"
 
 # Let users know if they're missing any of our hard dependencies
-hard_dependencies = ("numpy", "pytz", "dateutil")
-missing_dependencies = []
+_hard_dependencies = ("numpy", "pytz", "dateutil")
+_missing_dependencies = []
 
-for dependency in hard_dependencies:
+for _dependency in _hard_dependencies:
     try:
-        __import__(dependency)
-    except ImportError as e:
-        missing_dependencies.append(f"{dependency}: {e}")
+        __import__(_dependency)
+    except ImportError as _e:
+        _missing_dependencies.append(f"{_dependency}: {_e}")
 
-if missing_dependencies:
+if _missing_dependencies:
     raise ImportError(
-        "Unable to import required dependencies:\n" + "\n".join(missing_dependencies)
+        "Unable to import required dependencies:\n" + "\n".join(_missing_dependencies)
     )
-del hard_dependencies, dependency, missing_dependencies
+del _hard_dependencies, _dependency, _missing_dependencies
 
 # numpy compat
 from pandas.compat import is_numpy_dev as _is_numpy_dev
 
 try:
     from pandas._libs import hashtable as _hashtable, lib as _lib, tslib as _tslib
-except ImportError as err:  # pragma: no cover
-    module = err.name
+except ImportError as _err:  # pragma: no cover
+    _module = _err.name
     raise ImportError(
-        f"C extension: {module} not built. If you want to import "
+        f"C extension: {_module} not built. If you want to import "
         "pandas from the source directory, you may need to run "
         "'python setup.py build_ext --force' to build the C extensions first."
-    ) from err
+    ) from _err
 else:
     del _tslib, _lib, _hashtable