Merge branch 'main' into add-emscripten-ci

Author: agriyakhetarpal

.github/actions/run-tests/action.yml

@@ -1,16 +1,9 @@
 name: Run tests and report results
-inputs:
-  preload:
-    description: Preload arguments for sanitizer
-    required: false
-  asan_options:
-    description: Arguments for Address Sanitizer (ASAN)
-    required: false
 runs:
   using: composite
   steps:
     - name: Test
-      run: ${{ inputs.asan_options }} ${{ inputs.preload }} ci/run_tests.sh
+      run: ci/run_tests.sh
       shell: bash -el {0}
 
     - name: Publish test results

.github/workflows/unit-tests.yml

@@ -68,14 +68,6 @@ jobs:
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
-          - name: "ASAN / UBSAN"
-            env_file: actions-311-sanitizers.yaml
-            pattern: "not slow and not network and not single_cpu and not skip_ubsan"
-            asan_options: "ASAN_OPTIONS=detect_leaks=0"
-            preload: LD_PRELOAD=$(gcc -print-file-name=libasan.so)
-            meson_args: --config-settings=setup-args="-Db_sanitize=address,undefined"
-            cflags_adds: -fno-sanitize-recover=all
-            pytest_workers: -1  # disable pytest-xdist as it swallows stderr from ASAN
       fail-fast: false
     name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
     env:
@@ -161,18 +153,12 @@ jobs:
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
       if: ${{ matrix.name != 'Pypy' }}
-      with:
-        preload: ${{ matrix.preload }}
-        asan_options: ${{ matrix.asan_options }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
 
     - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
-      with:
-        preload: ${{ matrix.preload }}
-        asan_options: ${{ matrix.asan_options }}
       env:
         PATTERN: 'single_cpu'
         PYTEST_WORKERS: 0

.pre-commit-config.yaml

@@ -78,7 +78,7 @@ repos:
     hooks:
     -   id: pylint
         stages: [manual]
-        args: [--load-plugins=pylint.extensions.redefined_loop_name]
+        args: [--load-plugins=pylint.extensions.redefined_loop_name, --fail-on=I0021]
     -   id: pylint
         alias: redefined-outer-name
         name: Redefining name from outer scope

Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get install -y build-essential
 RUN apt-get install -y libhdf5-dev libgles2-mesa-dev
 
 RUN python -m pip install --upgrade pip
-RUN python -m pip install \
-    -r https://raw.githubusercontent.com/pandas-dev/pandas/main/requirements-dev.txt
+COPY requirements-dev.txt /tmp
+RUN python -m pip install -r /tmp/requirements-dev.txt
+RUN git config --global --add safe.directory /home/pandas
 CMD ["/bin/bash"]

asv_bench/benchmarks/categoricals.py

@@ -88,7 +88,7 @@ def setup(self):
         )
 
         for col in ("int", "float", "timestamp"):
-            self.df[col + "_as_str"] = self.df[col].astype(str)
+            self.df[f"{col}_as_str"] = self.df[col].astype(str)
 
         for col in self.df.columns:
             self.df[col] = self.df[col].astype("category")

asv_bench/benchmarks/join_merge.py

@@ -328,6 +328,23 @@ def time_i8merge(self, how):
         merge(self.left, self.right, how=how)
 
 
+class UniqueMerge:
+    params = [4_000_000, 1_000_000]
+    param_names = ["unique_elements"]
+
+    def setup(self, unique_elements):
+        N = 1_000_000
+        self.left = DataFrame({"a": np.random.randint(1, unique_elements, (N,))})
+        self.right = DataFrame({"a": np.random.randint(1, unique_elements, (N,))})
+        uniques = self.right.a.drop_duplicates()
+        self.right["a"] = concat(
+            [uniques, Series(np.arange(0, -(N - len(uniques)), -1))], ignore_index=True
+        )
+
+    def time_unique_merge(self, unique_elements):
+        merge(self.left, self.right, how="inner")
+
+
 class MergeDatetime:
     params = [
         [

asv_bench/benchmarks/timeseries.py

@@ -183,7 +183,7 @@ def setup(self):
         self.dt_ts = Series(5, rng3, dtype="datetime64[ns]")
 
     def time_resample(self):
-        self.dt_ts.resample("1S").last()
+        self.dt_ts.resample("1s").last()
 
 
 class AsOf:

ci/code_checks.sh

@@ -26,7 +26,7 @@ dependencies:
   - beautifulsoup4>=4.11.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
-  - fastparquet>=2023.04.0
+  - fastparquet>=2023.10.0
   - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
@@ -57,7 +57,7 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
-    - adbc-driver-postgresql>=0.8.0
+    - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
     - pytest-localserver>=0.7.1

ci/deps/actions-310.yaml

@@ -28,7 +28,7 @@ dependencies:
   - beautifulsoup4>=4.11.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
-  - fastparquet>=2023.04.0
+  - fastparquet>=2023.10.0
   - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
@@ -72,6 +72,6 @@ dependencies:
   - pyyaml
   - py
   - pip:
-    - adbc-driver-postgresql>=0.8.0
+    - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7

ci/deps/actions-311-downstream_compat.yaml

@@ -13,10 +13,7 @@ dependencies:
   # test dependencies
   - pytest>=7.3.2
   - pytest-cov
-  # Once pytest-cov > 4 comes out, unpin this
-  # Right now, a DeprecationWarning related to rsyncdir
-  # causes an InternalError within pytest
-  - pytest-xdist>=2.2.0, <3
+  - pytest-xdist>=2.2.0
   - hypothesis>=6.46.1
 
   # pandas dependencies

ci/deps/actions-311-numpydev.yaml

@@ -26,7 +26,7 @@ dependencies:
   - beautifulsoup4>=4.11.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
-  - fastparquet>=2023.04.0
+  - fastparquet>=2023.10.0
   - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
@@ -57,6 +57,6 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
-    - adbc-driver-postgresql>=0.8.0
+    - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - pytest-localserver>=0.7.1

ci/deps/actions-311-sanitizers.yaml

@@ -26,15 +26,15 @@ dependencies:
   - beautifulsoup4>=4.11.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
-  - fastparquet>=2023.04.0
+  - fastparquet>=2023.10.0
   - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
   - gcsfs>=2022.11.0
   - jinja2>=3.1.2
   - lxml>=4.9.2
   - matplotlib>=3.6.3
-  # - numba>=0.56.4
+  - numba>=0.56.4
   - numexpr>=2.8.4
   - odfpy>=1.4.1
   - qtpy>=2.3.0
@@ -44,7 +44,7 @@ dependencies:
   - pyarrow>=10.0.1
   - pymysql>=1.0.2
   - pyreadstat>=1.2.0
-  # - pytables>=3.8.0
+  - pytables>=3.8.0
   - python-calamine>=0.1.7
   - pyxlsb>=1.0.10
   - s3fs>=2022.11.0
@@ -57,7 +57,7 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
-    - adbc-driver-postgresql>=0.8.0
+    - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
     - pytest-localserver>=0.7.1

ci/deps/actions-311.yaml

@@ -29,7 +29,7 @@ dependencies:
   - beautifulsoup4=4.11.2
   - blosc=1.21.3
   - bottleneck=1.3.6
-  - fastparquet=2023.04.0
+  - fastparquet=2023.10.0
   - fsspec=2022.11.0
   - html5lib=1.1
   - hypothesis=6.46.1
@@ -60,6 +60,6 @@ dependencies:
   - zstandard=0.19.0
 
   - pip:
-    - adbc-driver-postgresql==0.8.0
+    - adbc-driver-postgresql==0.10.0
     - adbc-driver-sqlite==0.8.0
     - tzdata==2022.7

ci/deps/actions-312.yaml

@@ -26,7 +26,7 @@ dependencies:
   - beautifulsoup4>=4.11.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
-  - fastparquet>=2023.04.0
+  - fastparquet>=2023.10.0
   - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1
@@ -57,7 +57,7 @@ dependencies:
   - zstandard>=0.19.0
 
   - pip:
-    - adbc-driver-postgresql>=0.8.0
+    - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
     - pytest-localserver>=0.7.1

ci/deps/actions-39-minimum_versions.yaml

@@ -27,7 +27,7 @@ dependencies:
   - beautifulsoup4>=4.11.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
-  - fastparquet>=2023.04.0
+  - fastparquet>=2023.10.0
   - fsspec>=2022.11.0
   - html5lib>=1.1
   - hypothesis>=6.46.1

ci/deps/actions-39.yaml

@@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
-PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fE -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fE -n $PYTEST_WORKERS --dist=worksteal $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""

ci/deps/circle-310-arm64.yaml

@@ -248,6 +248,7 @@ ul.task-bullet > li > p:first-child {
 }
 
 .tutorial-card .card-header {
+  --bs-card-cap-color: var(--pst-color-text-base);
   cursor: pointer;
   background-color: var(--pst-color-surface);
   border: 1px solid var(--pst-color-border)
@@ -269,7 +270,7 @@ ul.task-bullet > li > p:first-child {
 
 
 .tutorial-card .gs-badge-link a {
-  color: var(--pst-color-text-base);
+  color: var(--pst-color-primary-text);
   text-decoration: none;
 }
 

ci/run_tests.sh

@@ -940,7 +940,7 @@ Finally, docstrings can also be appended to with the ``doc`` decorator.
 
 In this example, we'll create a parent docstring normally (this is like
 ``pandas.core.generic.NDFrame``). Then we'll have two children (like
-``pandas.core.series.Series`` and ``pandas.core.frame.DataFrame``). We'll
+``pandas.core.series.Series`` and ``pandas.DataFrame``). We'll
 substitute the class names in this docstring.
 
 .. code-block:: python

doc/source/_static/css/getting_started.css

@@ -151,15 +151,15 @@ and then run::
     git bisect start
     git bisect good v1.4.0
     git bisect bad v1.5.0
-    git bisect run bash -c "python setup.py build_ext -j 4; python t.py"
+    git bisect run bash -c "python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true; python t.py"
 
 This finds the first commit that changed the behavior. The C extensions have to be
 rebuilt at every step, so the search can take a while.
 
 Exit bisect and rebuild the current version::
 
     git bisect reset
-    python setup.py build_ext -j 4
+    python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true
 
 Report your findings under the corresponding issue and ping the commit author to get
 their input.

doc/source/development/contributing_docstring.rst

@@ -346,7 +346,7 @@ SQLAlchemy                2.0.0              postgresql,     SQL support for dat
                                              sql-other
 psycopg2                  2.9.6              postgresql      PostgreSQL engine for sqlalchemy
 pymysql                   1.0.2              mysql           MySQL engine for sqlalchemy
-adbc-driver-postgresql    0.8.0              postgresql      ADBC Driver for PostgreSQL
+adbc-driver-postgresql    0.10.0             postgresql      ADBC Driver for PostgreSQL
 adbc-driver-sqlite        0.8.0              sql-other       ADBC Driver for SQLite
 ========================= ================== =============== =============================================================
 
@@ -361,7 +361,7 @@ Dependency                Minimum Version    pip extra        Notes
 PyTables                  3.8.0              hdf5             HDF5-based reading / writing
 blosc                     1.21.3             hdf5             Compression for HDF5; only available on ``conda``
 zlib                                         hdf5             Compression for HDF5
-fastparquet               2023.04.0          -                Parquet reading / writing (pyarrow is default)
+fastparquet               2023.10.0          -                Parquet reading / writing (pyarrow is default)
 pyarrow                   10.0.1             parquet, feather Parquet, ORC, and feather reading / writing
 pyreadstat                1.2.0              spss             SPSS files (.sav) reading
 odfpy                     1.4.1              excel            Open document format (.odf, .ods, .odt) reading / writing