Merge branch 'master' into single_level

Author: LucasG0

.devcontainer.json

@@ -17,7 +17,9 @@
 		"python.linting.pylintEnabled": false,
 		"python.linting.mypyEnabled": true,
 		"python.testing.pytestEnabled": true,
-		"python.testing.cwd": "pandas/tests"
+		"python.testing.pytestArgs": [
+			"pandas"
+		]
 	},
 
 	// Add the IDs of extensions you want installed when the container is created in the array below.

.travis.yml

@@ -69,9 +69,9 @@ matrix:
       env:
         - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
     - dist: bionic
-      python: 3.9-dev
       env:
-        - JOB="3.9-dev" PATTERN="(not slow and not network)"
+        - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
+
 
 before_install:
   - echo "before_install"

asv_bench/benchmarks/categoricals.py

@@ -34,6 +34,7 @@ def setup(self):
         self.values_all_int8 = np.ones(N, "int8")
         self.categorical = pd.Categorical(self.values, self.categories)
         self.series = pd.Series(self.categorical)
+        self.intervals = pd.interval_range(0, 1, periods=N // 10)
 
     def time_regular(self):
         pd.Categorical(self.values, self.categories)
@@ -44,6 +45,9 @@ def time_fastpath(self):
     def time_datetimes(self):
         pd.Categorical(self.datetimes)
 
+    def time_interval(self):
+        pd.Categorical(self.datetimes, categories=self.datetimes)
+
     def time_datetimes_with_nat(self):
         pd.Categorical(self.datetimes_with_nat)
 

asv_bench/benchmarks/groupby.py

@@ -16,7 +16,7 @@
 
 from .pandas_vb_common import tm
 
-method_blacklist = {
+method_blocklist = {
     "object": {
         "median",
         "prod",
@@ -403,7 +403,7 @@ class GroupByMethods:
     ]
 
     def setup(self, dtype, method, application):
-        if method in method_blacklist.get(dtype, {}):
+        if method in method_blocklist.get(dtype, {}):
             raise NotImplementedError  # skip benchmark
         ngroups = 1000
         size = ngroups * 2

asv_bench/benchmarks/io/json.py

@@ -1,3 +1,5 @@
+import sys
+
 import numpy as np
 
 from pandas import DataFrame, concat, date_range, read_json, timedelta_range
@@ -82,6 +84,7 @@ def setup(self, orient, frame):
         timedeltas = timedelta_range(start=1, periods=N, freq="s")
         datetimes = date_range(start=1, periods=N, freq="s")
         ints = np.random.randint(100000000, size=N)
+        longints = sys.maxsize * np.random.randint(100000000, size=N)
         floats = np.random.randn(N)
         strings = tm.makeStringIndex(N)
         self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N))
@@ -120,6 +123,18 @@ def setup(self, orient, frame):
             index=index,
         )
 
+        self.df_longint_float_str = DataFrame(
+            {
+                "longint_1": longints,
+                "longint_2": longints,
+                "float_1": floats,
+                "float_2": floats,
+                "str_1": strings,
+                "str_2": strings,
+            },
+            index=index,
+        )
+
     def time_to_json(self, orient, frame):
         getattr(self, frame).to_json(self.fname, orient=orient)
 
@@ -172,6 +187,7 @@ def setup(self):
         timedeltas = timedelta_range(start=1, periods=N, freq="s")
         datetimes = date_range(start=1, periods=N, freq="s")
         ints = np.random.randint(100000000, size=N)
+        longints = sys.maxsize * np.random.randint(100000000, size=N)
         floats = np.random.randn(N)
         strings = tm.makeStringIndex(N)
         self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N))
@@ -209,6 +225,17 @@ def setup(self):
             },
             index=index,
         )
+        self.df_longint_float_str = DataFrame(
+            {
+                "longint_1": longints,
+                "longint_2": longints,
+                "float_1": floats,
+                "float_2": floats,
+                "str_1": strings,
+                "str_2": strings,
+            },
+            index=index,
+        )
 
     def time_floats_with_int_idex_lines(self):
         self.df.to_json(self.fname, orient="records", lines=True)
@@ -225,6 +252,9 @@ def time_float_int_lines(self):
     def time_float_int_str_lines(self):
         self.df_int_float_str.to_json(self.fname, orient="records", lines=True)
 
+    def time_float_longint_str_lines(self):
+        self.df_longint_float_str.to_json(self.fname, orient="records", lines=True)
+
 
 class ToJSONMem:
     def setup_cache(self):

asv_bench/benchmarks/pandas_vb_common.py

@@ -33,7 +33,7 @@
     np.uint8,
 ]
 datetime_dtypes = [np.datetime64, np.timedelta64]
-string_dtypes = [np.object]
+string_dtypes = [object]
 try:
     extension_dtypes = [
         pd.Int8Dtype,

asv_bench/benchmarks/series_methods.py

@@ -58,17 +58,15 @@ def time_isin_nan_values(self):
 
 class IsInForObjects:
     def setup(self):
-        self.s_nans = Series(np.full(10 ** 4, np.nan)).astype(np.object)
-        self.vals_nans = np.full(10 ** 4, np.nan).astype(np.object)
-        self.s_short = Series(np.arange(2)).astype(np.object)
-        self.s_long = Series(np.arange(10 ** 5)).astype(np.object)
-        self.vals_short = np.arange(2).astype(np.object)
-        self.vals_long = np.arange(10 ** 5).astype(np.object)
+        self.s_nans = Series(np.full(10 ** 4, np.nan)).astype(object)
+        self.vals_nans = np.full(10 ** 4, np.nan).astype(object)
+        self.s_short = Series(np.arange(2)).astype(object)
+        self.s_long = Series(np.arange(10 ** 5)).astype(object)
+        self.vals_short = np.arange(2).astype(object)
+        self.vals_long = np.arange(10 ** 5).astype(object)
         # because of nans floats are special:
-        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float)).astype(
-            np.object
-        )
-        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float).astype(np.object)
+        self.s_long_floats = Series(np.arange(10 ** 5, dtype=np.float)).astype(object)
+        self.vals_long_floats = np.arange(10 ** 5, dtype=np.float).astype(object)
 
     def time_isin_nans(self):
         # if nan-objects are different objects,

asv_bench/benchmarks/sparse.py

@@ -32,7 +32,7 @@ def time_series_to_frame(self):
 
 class SparseArrayConstructor:
 
-    params = ([0.1, 0.01], [0, np.nan], [np.int64, np.float64, np.object])
+    params = ([0.1, 0.01], [0, np.nan], [np.int64, np.float64, object])
     param_names = ["dense_proportion", "fill_value", "dtype"]
 
     def setup(self, dense_proportion, fill_value, dtype):

ci/code_checks.sh

@@ -248,19 +248,19 @@ fi
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 
-    MSG='Check import. No warnings, and blacklist some optional dependencies' ; echo $MSG
+    MSG='Check import. No warnings, and blocklist some optional dependencies' ; echo $MSG
     python -W error -c "
 import sys
 import pandas
 
-blacklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
+blocklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
              'lxml', 'matplotlib', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
              'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
 
 # GH#28227 for some of these check for top-level modules, while others are
 #  more specific (e.g. urllib.request)
 import_mods = set(m.split('.')[0] for m in sys.modules) | set(sys.modules)
-mods = blacklist & import_mods
+mods = blocklist & import_mods
 if mods:
     sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods)))
     sys.exit(len(mods))

ci/deps/azure-36-locale.yaml

@@ -15,7 +15,6 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - gcsfs
   - html5lib
   - ipython
   - jinja2
@@ -31,7 +30,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
   - scipy
   - xarray
   - xlrd

ci/deps/azure-37-locale.yaml

@@ -27,7 +27,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
   - scipy
   - xarray
   - xlrd

ci/deps/azure-windows-37.yaml

@@ -15,7 +15,8 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - gcsfs
+  - fsspec>=0.7.4
+  - gcsfs>=0.6.0
   - html5lib
   - jinja2
   - lxml
@@ -28,7 +29,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-36-cov.yaml

@@ -18,7 +18,8 @@ dependencies:
   - cython>=0.29.16
   - dask
   - fastparquet>=0.3.2
-  - gcsfs
+  - fsspec>=0.7.4
+  - gcsfs>=0.6.0
   - geopandas
   - html5lib
   - matplotlib
@@ -35,7 +36,7 @@ dependencies:
   - pytables
   - python-snappy
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scikit-learn
   - scipy
   - sqlalchemy

ci/deps/travis-36-locale.yaml

@@ -16,7 +16,6 @@ dependencies:
   - blosc=1.14.3
   - python-blosc
   - fastparquet=0.3.2
-  - gcsfs=0.2.2
   - html5lib
   - ipython
   - jinja2
@@ -33,7 +32,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs=0.3.0
   - scipy
   - sqlalchemy=1.1.4
   - xarray=0.10

ci/deps/travis-36-slow.yaml

@@ -13,6 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
+  - fsspec>=0.7.4
   - html5lib
   - lxml
   - matplotlib
@@ -25,7 +26,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-37.yaml

@@ -13,12 +13,13 @@ dependencies:
 
   # pandas dependencies
   - botocore>=1.11
+  - fsspec>=0.7.4
   - numpy
   - python-dateutil
   - nomkl
   - pyarrow
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - tabulate
   - pyreadstat
   - pip

doc/redirects.csv

@@ -269,11 +269,11 @@ generated/pandas.core.resample.Resampler.std,../reference/api/pandas.core.resamp
 generated/pandas.core.resample.Resampler.sum,../reference/api/pandas.core.resample.Resampler.sum
 generated/pandas.core.resample.Resampler.transform,../reference/api/pandas.core.resample.Resampler.transform
 generated/pandas.core.resample.Resampler.var,../reference/api/pandas.core.resample.Resampler.var
-generated/pandas.core.window.EWM.corr,../reference/api/pandas.core.window.EWM.corr
-generated/pandas.core.window.EWM.cov,../reference/api/pandas.core.window.EWM.cov
-generated/pandas.core.window.EWM.mean,../reference/api/pandas.core.window.EWM.mean
-generated/pandas.core.window.EWM.std,../reference/api/pandas.core.window.EWM.std
-generated/pandas.core.window.EWM.var,../reference/api/pandas.core.window.EWM.var
+generated/pandas.core.window.ExponentialMovingWindow.corr,../reference/api/pandas.core.window.ExponentialMovingWindow.corr
+generated/pandas.core.window.ExponentialMovingWindow.cov,../reference/api/pandas.core.window.ExponentialMovingWindow.cov
+generated/pandas.core.window.ExponentialMovingWindow.mean,../reference/api/pandas.core.window.ExponentialMovingWindow.mean
+generated/pandas.core.window.ExponentialMovingWindow.std,../reference/api/pandas.core.window.ExponentialMovingWindow.std
+generated/pandas.core.window.ExponentialMovingWindow.var,../reference/api/pandas.core.window.ExponentialMovingWindow.var
 generated/pandas.core.window.Expanding.aggregate,../reference/api/pandas.core.window.Expanding.aggregate
 generated/pandas.core.window.Expanding.apply,../reference/api/pandas.core.window.Expanding.apply
 generated/pandas.core.window.Expanding.corr,../reference/api/pandas.core.window.Expanding.corr

doc/source/getting_started/install.rst

@@ -267,8 +267,9 @@ SQLAlchemy                1.1.4              SQL support for databases other tha
 SciPy                     0.19.0             Miscellaneous statistical functions
 XLsxWriter                0.9.8              Excel writing
 blosc                                        Compression for HDF5
+fsspec                    0.7.4              Handling files aside from local and HTTP
 fastparquet               0.3.2              Parquet reading / writing
-gcsfs                     0.2.2              Google Cloud Storage access
+gcsfs                     0.6.0              Google Cloud Storage access
 html5lib                                     HTML parser for read_html (see :ref:`note <optional_html>`)
 lxml                      3.8.0              HTML parser for read_html (see :ref:`note <optional_html>`)
 matplotlib                2.2.2              Visualization
@@ -282,7 +283,7 @@ pyreadstat                                   SPSS files (.sav) reading
 pytables                  3.4.3              HDF5 reading / writing
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
-s3fs                      0.3.0              Amazon S3 access
+s3fs                      0.4.0              Amazon S3 access
 tabulate                  0.8.3              Printing in Markdown-friendly format (see `tabulate`_)
 xarray                    0.8.2              pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux