diff --git a/grayskull/strategy/py_base.py b/grayskull/strategy/py_base.py index 61d09da7..9f944ef6 100644 --- a/grayskull/strategy/py_base.py +++ b/grayskull/strategy/py_base.py @@ -737,11 +737,6 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) -> setup_metadata["entry_points"]["console_scripts"] = pyproject_metadata["build"][ "entry_points" ] - if pyproject_metadata["test"]["requires"]: - setup_metadata["extras_require"]["testing"] = merge_deps_toml_setup( - setup_metadata["extras_require"].get("testing", []), - pyproject_metadata["test"]["requires"], - ) if pyproject_metadata["requirements"]["host"]: setup_metadata["setup_requires"] = merge_deps_toml_setup( setup_metadata.get("setup_requires", []), @@ -752,6 +747,13 @@ def merge_setup_toml_metadata(setup_metadata: dict, pyproject_metadata: dict) -> setup_metadata.get("install_requires", []), pyproject_metadata["requirements"]["run"], ) + for extra_name, extra_requirements in pyproject_metadata["requirements"][ + "extra" + ].items(): + setup_metadata["extras_require"][extra_name] = merge_deps_toml_setup( + setup_metadata["extras_require"].get(extra_name, []), + extra_requirements, + ) # this is not a valid setup_metadata field, but we abuse it to pass it # through to the conda recipe generator downstream. It's because setup.py # does not have a notion of build vs. host requirements. It only has diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 3d2437cd..1abf8c02 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -121,7 +121,14 @@ def add_poetry_metadata(metadata: dict, toml_metadata: dict) -> dict: # add required test dependencies and ignore optional test dependencies, as # there doesn't appear to be a way to specify them in Conda recipe metadata. test_reqs, _ = encode_poetry_deps(poetry_test_deps) - metadata["test"].get("requires", []).extend(test_reqs) + if test_reqs: + extra_metadata = metadata["requirements"].setdefault("extra", {}) + for name in ("test", "tests", "testing"): + if name in extra_metadata: + extra_metadata[name] += test_reqs + break + else: + extra_metadata["test"] = test_reqs return metadata @@ -255,16 +262,11 @@ def get_all_toml_info(path_toml: Path | str) -> dict: toml_project = toml_metadata.get("project", {}) or {} metadata["requirements"]["host"] = toml_metadata["build-system"].get("requires", []) metadata["requirements"]["run"] = toml_project.get("dependencies", []) + metadata["requirements"]["extra"] = toml_project.get("optional-dependencies", {}) license = toml_project.get("license") if isinstance(license, dict): license = license.get("text", "") metadata["about"]["license"] = license - optional_deps = toml_project.get("optional-dependencies", {}) - metadata["test"]["requires"] = ( - optional_deps.get("testing", []) - or optional_deps.get("test", []) - or optional_deps.get("tests", []) - ) tom_urls = toml_project.get("urls", {}) if homepage := tom_urls.get("Homepage"): diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index a3e63de0..e3319ef7 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -37,8 +37,8 @@ def test_add_poetry_metadata(): "requirements": { "host": ["pkg_host1 >=1.0.0", "pkg_host2"], "run": ["pkg_run1", "pkg_run2 >=2.0.0"], + "extra": {"test": ["mock", "pkg_test >=1.0.0"]}, }, - "test": {"requires": ["mock", "pkg_test >=1.0.0"]}, } assert add_poetry_metadata(metadata, toml_metadata) == { "requirements": { @@ -49,9 +49,14 @@ def test_add_poetry_metadata(): "tomli >=1.0.0", "requests >=1.0.0", ], - }, - "test": { - "requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest >=1.0.0"] + "extra": { + "test": [ + "mock", + "pkg_test >=1.0.0", + "tox >=1.0.0", + "pytest >=1.0.0", + ] + }, }, } @@ -59,8 +64,9 @@ def test_add_poetry_metadata(): def test_poetry_dependencies(): toml_path = Path(__file__).parent / "data" / "poetry" / "poetry.toml" result = get_all_toml_info(toml_path) - - assert result["test"]["requires"] == ["cachy 0.3.0", "deepdiff >=6.2.0,<7.0.0"] + assert result["requirements"]["extra"] == { + "test": ["cachy 0.3.0", "deepdiff >=6.2.0,<7.0.0"] + } assert result["requirements"]["host"] == ["setuptools>=1.1.0", "poetry-core"] assert result["requirements"]["run"] == [ "python >=3.7.0,<4.0.0", diff --git a/tests/test_pypi.py b/tests/test_pypi.py index 94de9ffc..e948c63d 100644 --- a/tests/test_pypi.py +++ b/tests/test_pypi.py @@ -176,7 +176,7 @@ def test_get_extra_from_requires_dist(): @pytest.fixture(scope="module") -def dask_sdist_metadata(): +def dask_sdist_metadata_setup(): config = Configuration(name="dask") return get_sdist_metadata( "https://pypi.org/packages/source/d/dask/dask-2022.6.1.tar.gz", @@ -184,10 +184,10 @@ def dask_sdist_metadata(): ) -def test_get_extra_requirements(dask_sdist_metadata): +def test_get_extra_requirements_setup(dask_sdist_metadata_setup): received = { extra: set(req_lst) - for extra, req_lst in dask_sdist_metadata["extras_require"].items() + for extra, req_lst in dask_sdist_metadata_setup["extras_require"].items() } expected = { "array": {"numpy >= 1.18"}, @@ -208,10 +208,10 @@ def test_get_extra_requirements(dask_sdist_metadata): assert received == expected -def test_extract_optional_requirements(dask_sdist_metadata): +def test_extract_optional_requirements_setup(dask_sdist_metadata_setup): config = Configuration(name="dask") - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) assert not received all_optional_reqs = { @@ -230,7 +230,7 @@ def test_extract_optional_requirements(dask_sdist_metadata): } config.extras_require_all = True - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) received = {k: set(v) for k, v in received.items()} expected = {k: set(v) for k, v in all_optional_reqs.items()} assert received == expected @@ -238,7 +238,7 @@ def test_extract_optional_requirements(dask_sdist_metadata): config.extras_require_all = True config.extras_require_include = None config.extras_require_exclude = ["complete"] - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) received = {k: set(v) for k, v in received.items()} expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"} assert received == expected @@ -246,7 +246,7 @@ def test_extract_optional_requirements(dask_sdist_metadata): config.extras_require_all = False config.extras_require_include = ["complete"] config.extras_require_exclude = None - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) received = {k: set(v) for k, v in received.items()} expected = {k: set(v) for k, v in all_optional_reqs.items() if k == "complete"} assert received == expected @@ -254,7 +254,7 @@ def test_extract_optional_requirements(dask_sdist_metadata): config.extras_require_all = True config.extras_require_include = ["complete"] config.extras_require_exclude = None - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) received = {k: set(v) for k, v in received.items()} expected = {k: set(v) for k, v in all_optional_reqs.items()} assert received == expected @@ -262,7 +262,7 @@ def test_extract_optional_requirements(dask_sdist_metadata): config.extras_require_all = True config.extras_require_include = None config.extras_require_exclude = ["complete", "test"] - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) received = {k: set(v) for k, v in received.items()} expected = { k: set(v) for k, v in all_optional_reqs.items() if k not in ("complete", "test") @@ -273,34 +273,157 @@ def test_extract_optional_requirements(dask_sdist_metadata): config.extras_require_include = None config.extras_require_exclude = ["complete", "test"] config.extras_require_test = "test" - received = extract_optional_requirements(dask_sdist_metadata, config) + received = extract_optional_requirements(dask_sdist_metadata_setup, config) received = {k: set(v) for k, v in received.items()} expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"} assert received == expected -def test_compose_test_section_with_console_scripts(): - config = Configuration(name="pytest", version="7.1.2") - metadata1 = get_pypi_metadata(config) - metadata2 = get_sdist_metadata( - "https://pypi.org/packages/source/p/pytest/pytest-7.1.2.tar.gz", config - ) - metadata = merge_pypi_sdist_metadata(metadata1, metadata2, config) - test_requirements = [] +def test_compose_test_section_with_requirements_setup(dask_sdist_metadata_setup): + config = Configuration(name="dask", version="2022.6.1") + metadata = get_pypi_metadata(config) + test_requirements = dask_sdist_metadata_setup["extras_require"]["test"] test_section = compose_test_section(metadata, test_requirements) test_section = {k: set(v) for k, v in test_section.items()} expected = { - "imports": {"pytest"}, - "commands": {"pip check", "py.test --help", "pytest --help"}, - "requires": {"pip"}, + "imports": {"dask"}, + "commands": {"pip check", "pytest --pyargs dask"}, + "requires": { + "pip", + "pytest", + "pytest-xdist", + "pytest-rerunfailures", + "pre-commit", + }, } assert test_section == expected -def test_compose_test_section_with_requirements(dask_sdist_metadata): - config = Configuration(name="dask", version="2022.7.1") +@pytest.fixture(scope="module") +def dask_sdist_metadata_pyproject(): + config = Configuration(name="dask") + return get_sdist_metadata( + "https://pypi.org/packages/source/d/dask/dask-2025.3.0.tar.gz", + config, + ) + + +def test_get_extra_requirements_pyproject(dask_sdist_metadata_pyproject): + received = { + extra: set(req_lst) + for extra, req_lst in dask_sdist_metadata_pyproject["extras_require"].items() + } + expected = { + "array": {"numpy >= 1.24"}, + "bag": set(), + "complete": { + "dask[array,dataframe,distributed,diagnostics]", + "lz4 >= 4.3.2", + "pyarrow >= 14.0.1", + }, + "dataframe": {"dask[array]", "pyarrow>=14.0.1", "pandas >= 2.0"}, + "delayed": set(), + "diagnostics": {"jinja2 >= 2.10.3", "bokeh >= 3.1.0"}, + "distributed": {"distributed == 2025.3.0"}, + "test": { + "pandas[test]", + "pre-commit", + "pytest", + "pytest-cov", + "pytest-mock", + "pytest-rerunfailures", + "pytest-timeout", + "pytest-xdist", + }, + } + + assert received == expected + + +def test_extract_optional_requirements_pyproject(dask_sdist_metadata_pyproject): + config = Configuration(name="dask") + + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + assert not received + + all_optional_reqs = { + "array": {"numpy >=1.24"}, + "complete": { + "dask", + "lz4 >=4.3.2", + "pyarrow >=14.0.1", + }, + "dataframe": {"dask", "pyarrow >=14.0.1", "pandas >=2.0"}, + "diagnostics": {"jinja2 >=2.10.3", "bokeh >=3.1.0"}, + "distributed": {"distributed ==2025.3.0"}, + "test": { + "pandas", + "pre-commit", + "pytest", + "pytest-cov", + "pytest-mock", + "pytest-rerunfailures", + "pytest-timeout", + "pytest-xdist", + }, + } + + config.extras_require_all = True + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + received = {k: set(v) for k, v in received.items()} + expected = {k: set(v) for k, v in all_optional_reqs.items()} + assert received == expected + + config.extras_require_all = True + config.extras_require_include = None + config.extras_require_exclude = ["complete"] + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + received = {k: set(v) for k, v in received.items()} + expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"} + assert received == expected + + config.extras_require_all = False + config.extras_require_include = ["complete"] + config.extras_require_exclude = None + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + received = {k: set(v) for k, v in received.items()} + expected = {k: set(v) for k, v in all_optional_reqs.items() if k == "complete"} + assert received == expected + + config.extras_require_all = True + config.extras_require_include = ["complete"] + config.extras_require_exclude = None + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + received = {k: set(v) for k, v in received.items()} + expected = {k: set(v) for k, v in all_optional_reqs.items()} + assert received == expected + + config.extras_require_all = True + config.extras_require_include = None + config.extras_require_exclude = ["complete", "test"] + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + received = {k: set(v) for k, v in received.items()} + expected = { + k: set(v) for k, v in all_optional_reqs.items() if k not in ("complete", "test") + } + assert received == expected + + config.extras_require_all = True + config.extras_require_include = None + config.extras_require_exclude = ["complete", "test"] + config.extras_require_test = "test" + received = extract_optional_requirements(dask_sdist_metadata_pyproject, config) + received = {k: set(v) for k, v in received.items()} + expected = {k: set(v) for k, v in all_optional_reqs.items() if k != "complete"} + assert received == expected + + +def test_compose_test_section_with_requirements_pyproject( + dask_sdist_metadata_pyproject, +): + config = Configuration(name="dask", version="2025.3.0") metadata = get_pypi_metadata(config) - test_requirements = dask_sdist_metadata["extras_require"]["test"] + test_requirements = dask_sdist_metadata_pyproject["extras_require"]["test"] test_section = compose_test_section(metadata, test_requirements) test_section = {k: set(v) for k, v in test_section.items()} expected = { @@ -308,15 +431,37 @@ def test_compose_test_section_with_requirements(dask_sdist_metadata): "commands": {"pip check", "pytest --pyargs dask"}, "requires": { "pip", + "pandas[test]", + "pre-commit", "pytest", - "pytest-xdist", + "pytest-cov", + "pytest-mock", "pytest-rerunfailures", - "pre-commit", + "pytest-timeout", + "pytest-xdist", }, } assert test_section == expected +def test_compose_test_section_with_console_scripts(): + config = Configuration(name="pytest", version="7.1.2") + metadata1 = get_pypi_metadata(config) + metadata2 = get_sdist_metadata( + "https://pypi.org/packages/source/p/pytest/pytest-7.1.2.tar.gz", config + ) + metadata = merge_pypi_sdist_metadata(metadata1, metadata2, config) + test_requirements = [] + test_section = compose_test_section(metadata, test_requirements) + test_section = {k: set(v) for k, v in test_section.items()} + expected = { + "imports": {"pytest"}, + "commands": {"pip check", "py.test --help", "pytest --help"}, + "requires": {"pip"}, + } + assert test_section == expected + + def test_get_include_extra_requirements(): base_requirements = [ "cloudpickle >=1.1.1", diff --git a/tests/test_tox.py b/tests/test_tox.py index e3b68d13..34008f14 100644 --- a/tests/test_tox.py +++ b/tests/test_tox.py @@ -16,24 +16,36 @@ def test_get_all_toml_info(): "home": "http://tox.readthedocs.org", "summary": "tox is a generic virtualenv management and test command line tool", } - assert result["test"]["requires"] == [ - "build[virtualenv]>=0.9", - "covdefaults>=2.2.2", - "devpi-process>=0.3", - "diff-cover>=7.3", - "distlib>=0.3.6", - "flaky>=3.7", - "hatch-vcs>=0.3", - "hatchling>=1.12.2", - "psutil>=5.9.4", - "pytest>=7.2", - "pytest-cov>=4", - "pytest-mock>=3.10", - "pytest-xdist>=3.1", - "re-assert>=1.1", - "wheel>=0.38.4", - 'time-machine>=2.8.2; implementation_name != "pypy"', - ] + assert result["requirements"]["extra"] == { + "testing": [ + "build[virtualenv]>=0.9", + "covdefaults>=2.2.2", + "devpi-process>=0.3", + "diff-cover>=7.3", + "distlib>=0.3.6", + "flaky>=3.7", + "hatch-vcs>=0.3", + "hatchling>=1.12.2", + "psutil>=5.9.4", + "pytest>=7.2", + "pytest-cov>=4", + "pytest-mock>=3.10", + "pytest-xdist>=3.1", + "re-assert>=1.1", + "wheel>=0.38.4", + 'time-machine>=2.8.2; implementation_name != "pypy"', + ], + "docs": [ + "furo>=2022.12.7", + "sphinx>=6.1.3", + "sphinx-argparse-cli>=1.11", + "sphinx-autodoc-typehints>=1.20.1", + "sphinx-copybutton>=0.5.1", + "sphinx-inline-tabs>=2022.1.2b11", + "sphinxcontrib-towncrier>=0.2.1a0", + "towncrier>=22.12", + ], + } assert result["requirements"]["host"] == [ "hatchling>=1.12.2", "hatch-vcs>=0.3",