extract all optional dependencies from pyproject.toml

grayskull/strategy/py_base.py

@@ -737,11 +737,6 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) ->
         setup_metadata["entry_points"]["console_scripts"] = pyproject_metadata["build"][
             "entry_points"
         ]
-    if pyproject_metadata["test"]["requires"]:
-        setup_metadata["extras_require"]["testing"] = merge_deps_toml_setup(
-            setup_metadata["extras_require"].get("testing", []),
-            pyproject_metadata["test"]["requires"],
-        )
     if pyproject_metadata["requirements"]["host"]:
         setup_metadata["setup_requires"] = merge_deps_toml_setup(
             setup_metadata.get("setup_requires", []),
@@ -752,6 +747,13 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) ->
             setup_metadata.get("install_requires", []),
             pyproject_metadata["requirements"]["run"],
         )
+    for extra_name, extra_requirements in pyproject_metadata["requirements"][
+        "extra"
+    ].items():
+        setup_metadata["extras_require"][extra_name] = merge_deps_toml_setup(
+            setup_metadata["extras_require"].get(extra_name, []),
+            extra_requirements,
+        )
     # this is not a valid setup_metadata field, but we abuse it to pass it
     # through to the conda recipe generator downstream. It's because setup.py
     # does not have a notion of build vs. host requirements. It only has

grayskull/strategy/py_toml.py

@@ -121,7 +121,14 @@ def add_poetry_metadata(metadata: dict, toml_metadata: dict) -> dict:
     # add required test dependencies and ignore optional test dependencies, as
     # there doesn't appear to be a way to specify them in Conda recipe metadata.
     test_reqs, _ = encode_poetry_deps(poetry_test_deps)
-    metadata["test"].get("requires", []).extend(test_reqs)
+    if test_reqs:
+        extra_metadata = metadata["requirements"].setdefault("extra", {})
+        for name in ("test", "tests", "testing"):
+            if name in extra_metadata:
+                extra_metadata[name] += test_reqs
+                break
+        else:
+            extra_metadata["test"] = test_reqs
     return metadata
 
 
@@ -255,16 +262,11 @@ def get_all_toml_info(path_toml: Path | str) -> dict:
     toml_project = toml_metadata.get("project", {}) or {}
     metadata["requirements"]["host"] = toml_metadata["build-system"].get("requires", [])
     metadata["requirements"]["run"] = toml_project.get("dependencies", [])
+    metadata["requirements"]["extra"] = toml_project.get("optional-dependencies", {})
     license = toml_project.get("license")
     if isinstance(license, dict):
         license = license.get("text", "")
     metadata["about"]["license"] = license
-    optional_deps = toml_project.get("optional-dependencies", {})
-    metadata["test"]["requires"] = (
-        optional_deps.get("testing", [])
-        or optional_deps.get("test", [])
-        or optional_deps.get("tests", [])
-    )
 
     tom_urls = toml_project.get("urls", {})
     if homepage := tom_urls.get("Homepage"):

tests/test_py_toml.py

@@ -37,8 +37,8 @@ def test_add_poetry_metadata():
         "requirements": {
             "host": ["pkg_host1 >=1.0.0", "pkg_host2"],
             "run": ["pkg_run1", "pkg_run2 >=2.0.0"],
+            "extra": {"test": ["mock", "pkg_test >=1.0.0"]},
         },
-        "test": {"requires": ["mock", "pkg_test >=1.0.0"]},
     }
     assert add_poetry_metadata(metadata, toml_metadata) == {
         "requirements": {
@@ -49,18 +49,24 @@ def test_add_poetry_metadata():
                 "tomli >=1.0.0",
                 "requests >=1.0.0",
             ],
-        },
-        "test": {
-            "requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest >=1.0.0"]
+            "extra": {
+                "test": [
+                    "mock",
+                    "pkg_test >=1.0.0",
+                    "tox >=1.0.0",
+                    "pytest >=1.0.0",
+                ]
+            },
         },
     }
 
 
 def test_poetry_dependencies():
     toml_path = Path(__file__).parent / "data" / "poetry" / "poetry.toml"
     result = get_all_toml_info(toml_path)
-
-    assert result["test"]["requires"] == ["cachy 0.3.0", "deepdiff >=6.2.0,<7.0.0"]
+    assert result["requirements"]["extra"] == {
+        "test": ["cachy 0.3.0", "deepdiff >=6.2.0,<7.0.0"]
+    }
     assert result["requirements"]["host"] == ["setuptools>=1.1.0", "poetry-core"]
     assert result["requirements"]["run"] == [
         "python >=3.7.0,<4.0.0",

tests/test_pypi.py

@@ -176,18 +176,18 @@ def test_get_extra_from_requires_dist():
 
 
 @pytest.fixture(scope="module")
-def dask_sdist_metadata():
+def dask_sdist_metadata_setup():
     config = Configuration(name="dask")
     return get_sdist_metadata(
         "https://pypi.org/packages/source/d/dask/dask-2022.6.1.tar.gz",
         config,
     )
 
 
-def test_get_extra_requirements(dask_sdist_metadata):
+def test_get_extra_requirements_setup(dask_sdist_metadata_setup):
     received = {
         extra: set(req_lst)
-        for extra, req_lst in dask_sdist_metadata["extras_require"].items()
+        for extra, req_lst in dask_sdist_metadata_setup["extras_require"].items()
     }
     expected = {
         "array": {"numpy >= 1.18"},
@@ -208,10 +208,10 @@ def test_get_extra_requirements(dask_sdist_metadata):
     assert received == expected
 
 
-def test_extract_optional_requirements(dask_sdist_metadata):
+def test_extract_optional_requirements_setup(dask_sdist_metadata_setup):
     config = Configuration(name="dask")
 
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     assert not received
 
     all_optional_reqs = {
@@ -230,39 +230,39 @@ def test_extract_optional_requirements(dask_sdist_metadata):
     }
 
     config.extras_require_all = True
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     received = {k: set(v) for k, v in received.items()}
     expected = {k: set(v) for k, v in all_optional_reqs.items()}
     assert received == expected
 
     config.extras_require_all = True
     config.extras_require_include = None
     config.extras_require_exclude = ["complete"]
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     received = {k: set(v) for k, v in received.items()}
     expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"}
     assert received == expected
 
     config.extras_require_all = False
     config.extras_require_include = ["complete"]
     config.extras_require_exclude = None
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     received = {k: set(v) for k, v in received.items()}
     expected = {k: set(v) for k, v in all_optional_reqs.items() if k == "complete"}
     assert received == expected
 
     config.extras_require_all = True
     config.extras_require_include = ["complete"]
     config.extras_require_exclude = None
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     received = {k: set(v) for k, v in received.items()}
     expected = {k: set(v) for k, v in all_optional_reqs.items()}
     assert received == expected
 
     config.extras_require_all = True
     config.extras_require_include = None
     config.extras_require_exclude = ["complete", "test"]
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     received = {k: set(v) for k, v in received.items()}
     expected = {
         k: set(v) for k, v in all_optional_reqs.items() if k not in ("complete", "test")
@@ -273,50 +273,195 @@ def test_extract_optional_requirements(dask_sdist_metadata):
     config.extras_require_include = None
     config.extras_require_exclude = ["complete", "test"]
     config.extras_require_test = "test"
-    received = extract_optional_requirements(dask_sdist_metadata, config)
+    received = extract_optional_requirements(dask_sdist_metadata_setup, config)
     received = {k: set(v) for k, v in received.items()}
     expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"}
     assert received == expected
 
 
-def test_compose_test_section_with_console_scripts():
-    config = Configuration(name="pytest", version="7.1.2")
-    metadata1 = get_pypi_metadata(config)
-    metadata2 = get_sdist_metadata(
-        "https://pypi.org/packages/source/p/pytest/pytest-7.1.2.tar.gz", config
-    )
-    metadata = merge_pypi_sdist_metadata(metadata1, metadata2, config)
-    test_requirements = []
+def test_compose_test_section_with_requirements_setup(dask_sdist_metadata_setup):
+    config = Configuration(name="dask", version="2022.6.1")
+    metadata = get_pypi_metadata(config)
+    test_requirements = dask_sdist_metadata_setup["extras_require"]["test"]
     test_section = compose_test_section(metadata, test_requirements)
     test_section = {k: set(v) for k, v in test_section.items()}
     expected = {
-        "imports": {"pytest"},
-        "commands": {"pip check", "py.test --help", "pytest --help"},
-        "requires": {"pip"},
+        "imports": {"dask"},
+        "commands": {"pip check", "pytest --pyargs dask"},
+        "requires": {
+            "pip",
+            "pytest",
+            "pytest-xdist",
+            "pytest-rerunfailures",
+            "pre-commit",
+        },
     }
     assert test_section == expected
 
 
-def test_compose_test_section_with_requirements(dask_sdist_metadata):
-    config = Configuration(name="dask", version="2022.7.1")
+@pytest.fixture(scope="module")
+def dask_sdist_metadata_pyproject():
+    config = Configuration(name="dask")
+    return get_sdist_metadata(
+        "https://pypi.org/packages/source/d/dask/dask-2025.3.0.tar.gz",
+        config,
+    )
+
+
+def test_get_extra_requirements_pyproject(dask_sdist_metadata_pyproject):
+    received = {
+        extra: set(req_lst)
+        for extra, req_lst in dask_sdist_metadata_pyproject["extras_require"].items()
+    }
+    expected = {
+        "array": {"numpy >= 1.24"},
+        "bag": set(),
+        "complete": {
+            "dask[array,dataframe,distributed,diagnostics]",
+            "lz4 >= 4.3.2",
+            "pyarrow >= 14.0.1",
+        },
+        "dataframe": {"dask[array]", "pyarrow>=14.0.1", "pandas >= 2.0"},
+        "delayed": set(),
+        "diagnostics": {"jinja2 >= 2.10.3", "bokeh >= 3.1.0"},
+        "distributed": {"distributed == 2025.3.0"},
+        "test": {
+            "pandas[test]",
+            "pre-commit",
+            "pytest",
+            "pytest-cov",
+            "pytest-mock",
+            "pytest-rerunfailures",
+            "pytest-timeout",
+            "pytest-xdist",
+        },
+    }
+
+    assert received == expected
+
+
+def test_extract_optional_requirements_pyproject(dask_sdist_metadata_pyproject):
+    config = Configuration(name="dask")
+
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    assert not received
+
+    all_optional_reqs = {
+        "array": {"numpy >=1.24"},
+        "complete": {
+            "dask",
+            "lz4 >=4.3.2",
+            "pyarrow >=14.0.1",
+        },
+        "dataframe": {"dask", "pyarrow >=14.0.1", "pandas >=2.0"},
+        "diagnostics": {"jinja2 >=2.10.3", "bokeh >=3.1.0"},
+        "distributed": {"distributed ==2025.3.0"},
+        "test": {
+            "pandas",
+            "pre-commit",
+            "pytest",
+            "pytest-cov",
+            "pytest-mock",
+            "pytest-rerunfailures",
+            "pytest-timeout",
+            "pytest-xdist",
+        },
+    }
+
+    config.extras_require_all = True
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    received = {k: set(v) for k, v in received.items()}
+    expected = {k: set(v) for k, v in all_optional_reqs.items()}
+    assert received == expected
+
+    config.extras_require_all = True
+    config.extras_require_include = None
+    config.extras_require_exclude = ["complete"]
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    received = {k: set(v) for k, v in received.items()}
+    expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"}
+    assert received == expected
+
+    config.extras_require_all = False
+    config.extras_require_include = ["complete"]
+    config.extras_require_exclude = None
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    received = {k: set(v) for k, v in received.items()}
+    expected = {k: set(v) for k, v in all_optional_reqs.items() if k == "complete"}
+    assert received == expected
+
+    config.extras_require_all = True
+    config.extras_require_include = ["complete"]
+    config.extras_require_exclude = None
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    received = {k: set(v) for k, v in received.items()}
+    expected = {k: set(v) for k, v in all_optional_reqs.items()}
+    assert received == expected
+
+    config.extras_require_all = True
+    config.extras_require_include = None
+    config.extras_require_exclude = ["complete", "test"]
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    received = {k: set(v) for k, v in received.items()}
+    expected = {
+        k: set(v) for k, v in all_optional_reqs.items() if k not in ("complete", "test")
+    }
+    assert received == expected
+
+    config.extras_require_all = True
+    config.extras_require_include = None
+    config.extras_require_exclude = ["complete", "test"]
+    config.extras_require_test = "test"
+    received = extract_optional_requirements(dask_sdist_metadata_pyproject, config)
+    received = {k: set(v) for k, v in received.items()}
+    expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"}
+    assert received == expected
+
+
+def test_compose_test_section_with_requirements_pyproject(
+    dask_sdist_metadata_pyproject,
+):
+    config = Configuration(name="dask", version="2025.3.0")
     metadata = get_pypi_metadata(config)
-    test_requirements = dask_sdist_metadata["extras_require"]["test"]
+    test_requirements = dask_sdist_metadata_pyproject["extras_require"]["test"]
     test_section = compose_test_section(metadata, test_requirements)
     test_section = {k: set(v) for k, v in test_section.items()}
     expected = {
         "imports": {"dask"},
         "commands": {"pip check", "pytest --pyargs dask"},
         "requires": {
             "pip",
+            "pandas[test]",
+            "pre-commit",
             "pytest",
-            "pytest-xdist",
+            "pytest-cov",
+            "pytest-mock",
             "pytest-rerunfailures",
-            "pre-commit",
+            "pytest-timeout",
+            "pytest-xdist",
         },
     }
     assert test_section == expected
 
 
+def test_compose_test_section_with_console_scripts():
+    config = Configuration(name="pytest", version="7.1.2")
+    metadata1 = get_pypi_metadata(config)
+    metadata2 = get_sdist_metadata(
+        "https://pypi.org/packages/source/p/pytest/pytest-7.1.2.tar.gz", config
+    )
+    metadata = merge_pypi_sdist_metadata(metadata1, metadata2, config)
+    test_requirements = []
+    test_section = compose_test_section(metadata, test_requirements)
+    test_section = {k: set(v) for k, v in test_section.items()}
+    expected = {
+        "imports": {"pytest"},
+        "commands": {"pip check", "py.test --help", "pytest --help"},
+        "requires": {"pip"},
+    }
+    assert test_section == expected
+
+
 def test_get_include_extra_requirements():
     base_requirements = [
         "cloudpickle >=1.1.1",