IO: Refactor and clean up test cases

Author: amotl

examples/cdc/aws/kinesis_put.py

@@ -1,7 +1,7 @@
 from yarl import URL
 
 from cratedb_toolkit.io.kinesis.adapter import KinesisStreamAdapter
-from tests.io.test_processor import DYNAMODB_CDC_INSERT_NESTED, DYNAMODB_CDC_MODIFY_NESTED, wrap_kinesis
+from tests.io.test_awslambda import DYNAMODB_CDC_INSERT_NESTED, DYNAMODB_CDC_MODIFY_NESTED, wrap_kinesis
 
 
 def main():

tests/io/test_import.py tests/cluster/test_import.pytests/io/test_import.py renamed to tests/cluster/test_import.py

@@ -6,64 +6,14 @@
 from cratedb_toolkit import ManagedCluster
 
 
-@pytest.fixture
-def dummy_csv(tmp_path):
-    """
-    Provide a dummy CSV file to the test cases.
-    """
-    csvfile = tmp_path / "dummy.csv"
-    csvfile.write_text("name,value\ntemperature,42.42\nhumidity,84.84")
-    return csvfile
-
-
-def test_import_standalone_csv_pandas(cratedb, dummy_csv):
-    """
-    Invoke convenience function `import_csv_pandas`, and verify database content.
-    """
-    result = cratedb.database.import_csv_pandas(filepath=dummy_csv, tablename="foobar")
-    assert result is None
-
-    cratedb.database.run_sql("REFRESH TABLE foobar;")
-    result = cratedb.database.run_sql("SELECT COUNT(*) FROM foobar;")
-    assert result == [(2,)]
-
-
-def test_import_standalone_csv_dask(cratedb, dummy_csv, needs_sqlalchemy2):
-    """
-    Invoke convenience function `import_csv_dask`, and verify database content.
-    """
-    pytest.importorskip("dask")
-    result = cratedb.database.import_csv_dask(filepath=dummy_csv, tablename="foobar")
-    assert result is None
-
-    cratedb.database.run_sql("REFRESH TABLE foobar;")
-    result = cratedb.database.run_sql("SELECT COUNT(*) FROM foobar;")
-    assert result == [(2,)]
-
-
-def test_import_standalone_csv_dask_with_progressbar(cratedb, dummy_csv, needs_sqlalchemy2):
-    """
-    Invoke convenience function `import_csv_dask`, and verify database content.
-    This time, use `progress=True` to make Dask display a progress bar.
-    However, the code does not verify it.
-    """
-    pytest.importorskip("dask")
-    result = cratedb.database.import_csv_dask(filepath=dummy_csv, tablename="foobar", progress=True)
-    assert result is None
-
-    cratedb.database.run_sql("REFRESH TABLE foobar;")
-    result = cratedb.database.run_sql("SELECT COUNT(*) FROM foobar;")
-    assert result == [(2,)]
-
-
 @pytest.mark.skipif(
     os.getenv("FORCE", "no") != "yes",
     reason="Only works when invoked exclusively, using "
     "`FORCE=yes pytest --no-cov tests/io/test_import.py -k managed_csv_local`. "
     "Otherwise croaks per `AssertionError: ERROR: The following arguments are required: --url`. "
     "We don't know why.",
 )
-def test_import_managed_csv_local(cloud_environment, dummy_csv, caplog):
+def test_csv_import_local(cloud_environment, dummy_csv, caplog):
     """
     CLI test: Invoke `ctk load table ...` for a CrateDB Cloud Import, from a local file.
     """
@@ -90,7 +40,7 @@ def test_import_managed_csv_local(cloud_environment, dummy_csv, caplog):
     assert results[0]["count"] >= 2
 
 
-def test_import_managed_parquet_remote(cloud_environment, tmp_path, caplog):
+def test_parquet_import_remote(cloud_environment, tmp_path, caplog):
     """
     CLI test: Invoke `ctk load table ...` for a CrateDB Cloud Import, from a URL.
     """

tests/iac/__init__.py

@@ -5,7 +5,7 @@
 from commons_codec.transform.dynamodb_model import PrimaryKeySchema
 
 from cratedb_toolkit.io.kinesis.relay import KinesisRelay
-from tests.io.test_processor import DYNAMODB_CDC_INSERT_NESTED, DYNAMODB_CDC_MODIFY_NESTED, wrap_kinesis
+from tests.io.test_awslambda import DYNAMODB_CDC_INSERT_NESTED, DYNAMODB_CDC_MODIFY_NESTED, wrap_kinesis
 
 pytestmark = pytest.mark.kinesis
 

tests/io/test_iac.py tests/iac/test_basic.pytests/io/test_iac.py renamed to tests/iac/test_basic.py

@@ -4,7 +4,7 @@
 from yarl import URL
 
 from cratedb_toolkit.io.kinesis.adapter import KinesisAdapterBase
-from tests.io.test_processor import wrap_kinesis
+from tests.io.test_awslambda import wrap_kinesis
 
 logger = logging.getLogger(__name__)
 

tests/io/dynamodb/test_relay.py

@@ -5,7 +5,7 @@
 from cratedb_toolkit.io.kinesis.model import RecipeDefinition
 from cratedb_toolkit.io.kinesis.relay import KinesisRelay
 from tests.io.kinesis.data import DMS_CDC_CREATE_TABLE, DMS_CDC_INSERT_BASIC
-from tests.io.test_processor import wrap_kinesis
+from tests.io.test_awslambda import wrap_kinesis
 
 pytestmark = pytest.mark.kinesis
 

tests/io/kinesis/manager.py

@@ -0,0 +1,51 @@
+import pytest
+
+
+@pytest.fixture
+def dummy_csv(tmp_path):
+    """
+    Provide a dummy CSV file to the test cases.
+    """
+    csvfile = tmp_path / "dummy.csv"
+    csvfile.write_text("name,value\ntemperature,42.42\nhumidity,84.84")
+    return csvfile
+
+
+def test_csv_import_pandas(cratedb, dummy_csv):
+    """
+    Invoke convenience function `import_csv_pandas`, and verify database content.
+    """
+    result = cratedb.database.import_csv_pandas(filepath=dummy_csv, tablename="foobar")
+    assert result is None
+
+    cratedb.database.run_sql("REFRESH TABLE foobar;")
+    result = cratedb.database.run_sql("SELECT COUNT(*) FROM foobar;")
+    assert result == [(2,)]
+
+
+def test_csv_import_dask(cratedb, dummy_csv, needs_sqlalchemy2):
+    """
+    Invoke convenience function `import_csv_dask`, and verify database content.
+    """
+    pytest.importorskip("dask")
+    result = cratedb.database.import_csv_dask(filepath=dummy_csv, tablename="foobar")
+    assert result is None
+
+    cratedb.database.run_sql("REFRESH TABLE foobar;")
+    result = cratedb.database.run_sql("SELECT COUNT(*) FROM foobar;")
+    assert result == [(2,)]
+
+
+def test_csv_import_dask_with_progressbar(cratedb, dummy_csv, needs_sqlalchemy2):
+    """
+    Invoke convenience function `import_csv_dask`, and verify database content.
+    This time, use `progress=True` to make Dask display a progress bar.
+    However, the code does not verify it.
+    """
+    pytest.importorskip("dask")
+    result = cratedb.database.import_csv_dask(filepath=dummy_csv, tablename="foobar", progress=True)
+    assert result is None
+
+    cratedb.database.run_sql("REFRESH TABLE foobar;")
+    result = cratedb.database.run_sql("SELECT COUNT(*) FROM foobar;")
+    assert result == [(2,)]