[ArrayManager] TST: enable IO tests (#40298)

Author: jorisvandenbossche

.github/workflows/ci.yml

@@ -185,6 +185,12 @@ jobs:
         pytest pandas/tests/dtypes/
         pytest pandas/tests/generic/
         pytest pandas/tests/indexes/
+        pytest pandas/tests/io/test_* -m "not slow and not clipboard"
+        pytest pandas/tests/io/excel/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/formats/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/parser/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/sas/ -m "not slow and not clipboard"
+        pytest pandas/tests/io/xml/ -m "not slow and not clipboard"
         pytest pandas/tests/libs/
         pytest pandas/tests/plotting/
         pytest pandas/tests/scalar/

pandas/tests/io/pytables/test_append.py

@@ -7,6 +7,7 @@
 import pytest
 
 from pandas._libs.tslibs import Timestamp
+import pandas.util._test_decorators as td
 
 import pandas as pd
 from pandas import (
@@ -24,7 +25,7 @@
     ensure_clean_store,
 )
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 @pytest.mark.filterwarnings("ignore:object name:tables.exceptions.NaturalNameWarning")

pandas/tests/io/pytables/test_categorical.py

@@ -1,6 +1,8 @@
 import numpy as np
 import pytest
 
+import pandas.util._test_decorators as td
+
 from pandas import (
     Categorical,
     DataFrame,
@@ -15,7 +17,7 @@
     ensure_clean_store,
 )
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_categorical(setup_path):

pandas/tests/io/pytables/test_compat.py

@@ -1,11 +1,15 @@
 import pytest
 
+import pandas.util._test_decorators as td
+
 import pandas as pd
 import pandas._testing as tm
 from pandas.tests.io.pytables.common import ensure_clean_path
 
 tables = pytest.importorskip("tables")
 
+pytestmark = td.skip_array_manager_not_yet_implemented
+
 
 @pytest.fixture
 def pytables_hdf5_file():

pandas/tests/io/pytables/test_errors.py

@@ -6,6 +6,8 @@
 import numpy as np
 import pytest
 
+import pandas.util._test_decorators as td
+
 from pandas import (
     CategoricalIndex,
     DataFrame,
@@ -25,7 +27,7 @@
     _maybe_adjust_name,
 )
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_pass_spec_to_storer(setup_path):

pandas/tests/io/pytables/test_file_handling.py

@@ -4,6 +4,7 @@
 import pytest
 
 from pandas.compat import is_platform_little_endian
+import pandas.util._test_decorators as td
 
 from pandas import (
     DataFrame,
@@ -26,7 +27,7 @@
     Term,
 )
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_mode(setup_path):

pandas/tests/io/pytables/test_keys.py

@@ -1,5 +1,7 @@
 import pytest
 
+import pandas.util._test_decorators as td
+
 from pandas import (
     DataFrame,
     HDFStore,
@@ -11,7 +13,7 @@
     tables,
 )
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_keys(setup_path):

pandas/tests/io/pytables/test_put.py

@@ -29,7 +29,7 @@
 )
 from pandas.util import _test_decorators as td
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_format_type(setup_path):

pandas/tests/io/pytables/test_pytables_missing.py

@@ -5,6 +5,8 @@
 import pandas as pd
 import pandas._testing as tm
 
+pytestmark = td.skip_array_manager_not_yet_implemented
+
 
 @td.skip_if_installed("tables")
 def test_pytables_raises():

pandas/tests/io/pytables/test_read.py

@@ -25,7 +25,7 @@
 
 from pandas.io.pytables import TableIterator
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_read_missing_key_close_store(setup_path):

pandas/tests/io/pytables/test_retain_attributes.py

@@ -3,6 +3,7 @@
 import pytest
 
 from pandas._libs.tslibs import Timestamp
+import pandas.util._test_decorators as td
 
 from pandas import (
     DataFrame,
@@ -17,7 +18,7 @@
     ensure_clean_store,
 )
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_retain_index_attributes(setup_path):

pandas/tests/io/pytables/test_round_trip.py

@@ -30,7 +30,7 @@
 _default_compressor = "blosc"
 
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_conv_read_write(setup_path):

pandas/tests/io/pytables/test_select.py

@@ -5,6 +5,7 @@
 import pytest
 
 from pandas._libs.tslibs import Timestamp
+import pandas.util._test_decorators as td
 
 import pandas as pd
 from pandas import (
@@ -29,7 +30,7 @@
 
 from pandas.io.pytables import Term
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_select_columns_in_where(setup_path):

pandas/tests/io/pytables/test_store.py

@@ -32,10 +32,6 @@
     safe_close,
 )
 
-# TODO(ArrayManager) HDFStore relies on accessing the blocks
-pytestmark = td.skip_array_manager_not_yet_implemented
-
-
 _default_compressor = "blosc"
 ignore_natural_naming_warning = pytest.mark.filterwarnings(
     "ignore:object name:tables.exceptions.NaturalNameWarning"
@@ -46,7 +42,8 @@
     read_hdf,
 )
 
-pytestmark = pytest.mark.single
+# TODO(ArrayManager) HDFStore relies on accessing the blocks
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_context(setup_path):

pandas/tests/io/pytables/test_subclass.py

@@ -1,5 +1,7 @@
 import numpy as np
 
+import pandas.util._test_decorators as td
+
 from pandas import (
     DataFrame,
     Series,
@@ -12,6 +14,8 @@
     read_hdf,
 )
 
+pytestmark = td.skip_array_manager_not_yet_implemented
+
 
 class TestHDFStoreSubclass:
     # GH 33748

pandas/tests/io/pytables/test_time_series.py

@@ -3,14 +3,16 @@
 import numpy as np
 import pytest
 
+import pandas.util._test_decorators as td
+
 from pandas import (
     DataFrame,
     Series,
     _testing as tm,
 )
 from pandas.tests.io.pytables.common import ensure_clean_store
 
-pytestmark = pytest.mark.single
+pytestmark = [pytest.mark.single, td.skip_array_manager_not_yet_implemented]
 
 
 def test_store_datetime_fractional_secs(setup_path):

pandas/tests/io/sas/test_sas7bdat.py

@@ -194,10 +194,13 @@ def test_compact_numerical_values(datapath):
     tm.assert_series_equal(result, expected, check_exact=True)
 
 
-def test_many_columns(datapath):
+def test_many_columns(datapath, using_array_manager):
     # Test for looking for column information in more places (PR #22628)
     fname = datapath("io", "sas", "data", "many_columns.sas7bdat")
-    with tm.assert_produces_warning(PerformanceWarning):
+    expected_warning = None
+    if not using_array_manager:
+        expected_warning = PerformanceWarning
+    with tm.assert_produces_warning(expected_warning):
         # Many DataFrame.insert calls
         df = pd.read_sas(fname, encoding="latin-1")
 

pandas/tests/io/test_common.py

@@ -307,6 +307,7 @@ def test_write_fspath_all(self, writer_name, writer_kwargs, module):
 
             assert result == expected
 
+    @td.skip_array_manager_not_yet_implemented  # TODO(ArrayManager) IO HDF5
     def test_write_fspath_hdf5(self):
         # Same test as write_fspath_all, except HDF5 files aren't
         # necessarily byte-for-byte identical for a given dataframe, so we'll
@@ -431,7 +432,10 @@ def test_is_fsspec_url():
 
 
 @pytest.mark.parametrize("encoding", [None, "utf-8"])
-@pytest.mark.parametrize("format", ["csv", "json"])
+@pytest.mark.parametrize(
+    "format",
+    ["csv", pytest.param("json", marks=td.skip_array_manager_not_yet_implemented)],
+)
 def test_codecs_encoding(encoding, format):
     # GH39247
     expected = tm.makeDataFrame()

pandas/tests/io/test_fsspec.py

@@ -166,6 +166,7 @@ def test_arrowparquet_options(fsspectest):
     assert fsspectest.test[0] == "parquet_read"
 
 
+@td.skip_array_manager_not_yet_implemented  # TODO(ArrayManager) fastparquet
 @td.skip_if_no("fastparquet")
 def test_fastparquet_options(fsspectest):
     """Regression test for writing to a not-yet-existent GCS Parquet file."""
@@ -210,6 +211,7 @@ def test_s3_protocols(s3_resource, tips_file, protocol, s3so):
     )
 
 
+@td.skip_array_manager_not_yet_implemented  # TODO(ArrayManager) fastparquet
 @td.skip_if_no("s3fs")
 @td.skip_if_no("fastparquet")
 def test_s3_parquet(s3_resource, s3so):

pandas/tests/io/test_gcs.py

@@ -44,7 +44,16 @@ def ls(self, path, **kwargs):
 
 
 @td.skip_if_no("gcsfs")
-@pytest.mark.parametrize("format", ["csv", "json", "parquet", "excel", "markdown"])
+@pytest.mark.parametrize(
+    "format",
+    [
+        "csv",
+        pytest.param("json", marks=td.skip_array_manager_not_yet_implemented),
+        "parquet",
+        "excel",
+        "markdown",
+    ],
+)
 def test_to_read_gcs(gcs_buffer, format):
     """
     Test that many to/read functions support GCS.

pandas/tests/io/test_parquet.py

@@ -41,9 +41,11 @@
     _HAVE_FASTPARQUET = False
 
 
-pytestmark = pytest.mark.filterwarnings(
-    "ignore:RangeIndex.* is deprecated:DeprecationWarning"
-)
+pytestmark = [
+    pytest.mark.filterwarnings("ignore:RangeIndex.* is deprecated:DeprecationWarning"),
+    # TODO(ArrayManager) fastparquet / pyarrow rely on BlockManager internals
+    td.skip_array_manager_not_yet_implemented,
+]
 
 
 # setup engines & skips

pandas/tests/io/test_pickle.py

@@ -54,6 +54,10 @@
 lzma = import_lzma()
 
 
+# TODO(ArrayManager) pickling
+pytestmark = td.skip_array_manager_not_yet_implemented
+
+
 @pytest.fixture(scope="module")
 def current_pickle_data():
     # our current version pickle data

pandas/tests/io/test_stata.py

@@ -12,8 +12,6 @@
 import numpy as np
 import pytest
 
-import pandas.util._test_decorators as td
-
 from pandas.core.dtypes.common import is_categorical_dtype
 
 import pandas as pd
@@ -35,9 +33,6 @@
     read_stata,
 )
 
-# TODO(ArrayManager) the stata code relies on BlockManager internals (eg blknos)
-pytestmark = td.skip_array_manager_not_yet_implemented
-
 
 @pytest.fixture()
 def mixed_frame():

pandas/tests/io/test_user_agent.py

@@ -189,7 +189,13 @@ def do_GET(self):
             marks=td.skip_array_manager_not_yet_implemented,
         ),
         (ParquetPyArrowUserAgentResponder, pd.read_parquet, "pyarrow"),
-        (ParquetFastParquetUserAgentResponder, pd.read_parquet, "fastparquet"),
+        pytest.param(
+            ParquetFastParquetUserAgentResponder,
+            pd.read_parquet,
+            "fastparquet",
+            # TODO(ArrayManager) fastparquet
+            marks=td.skip_array_manager_not_yet_implemented,
+        ),
         (PickleUserAgentResponder, pd.read_pickle, None),
         (StataUserAgentResponder, pd.read_stata, None),
         (GzippedCSVUserAgentResponder, pd.read_csv, None),
@@ -234,7 +240,13 @@ def test_server_and_default_headers(responder, read_method, parquet_engine):
             marks=td.skip_array_manager_not_yet_implemented,
         ),
         (ParquetPyArrowUserAgentResponder, pd.read_parquet, "pyarrow"),
-        (ParquetFastParquetUserAgentResponder, pd.read_parquet, "fastparquet"),
+        pytest.param(
+            ParquetFastParquetUserAgentResponder,
+            pd.read_parquet,
+            "fastparquet",
+            # TODO(ArrayManager) fastparquet
+            marks=td.skip_array_manager_not_yet_implemented,
+        ),
         (PickleUserAgentResponder, pd.read_pickle, None),
         (StataUserAgentResponder, pd.read_stata, None),
         (GzippedCSVUserAgentResponder, pd.read_csv, None),

pandas/tests/io/xml/test_to_xml.py

@@ -1172,6 +1172,7 @@ def test_style_to_string():
     assert out_xml == out_str
 
 
+@td.skip_array_manager_not_yet_implemented  # TODO(ArrayManager) JSON
 @td.skip_if_no("lxml")
 def test_style_to_json():
     xsl = """\