Fix extras require issue (#326)

* black

* issue to be fixed: extras_require is a dictionary, not a list

* fix extras_require issue

Author: woutdenolf

docs/source/conf.py

@@ -17,24 +17,23 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'Grayskull'
-copyright = '2022, Grayskull Developers.'
-author = 'Grayskull Developers'
+project = "Grayskull"
+copyright = "2022, Grayskull Developers."
+author = "Grayskull Developers"
 
 # The full version, including alpha/beta/rc tags
-release = '1.1.0'
+release = "1.1.0"
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-]
+extensions = []
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -47,9 +46,9 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'furo'
+html_theme = "furo"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]

grayskull/strategy/py_base.py

@@ -22,6 +22,8 @@
 from grayskull.utils import (
     PyVer,
     get_vendored_dependencies,
+    merge_dict_of_lists_item,
+    merge_list_item,
     origin_is_github,
     sha256_checksum,
 )
@@ -256,23 +258,14 @@ def merge_sdist_metadata(setup_py: dict, setup_cfg: dict) -> dict:
         if key not in result:
             result[key] = value
 
-    def get_full_list(sec_key: str) -> List:
-        if sec_key not in setup_py:
-            return setup_cfg.get(sec_key, [])
-        cfg_val = set(setup_cfg.get(sec_key, []))
-        result_val = set(result.get(sec_key, []))
-        return list(cfg_val.union(result_val))
-
-    if "install_requires" in result:
-        result["install_requires"] = get_full_list("install_requires")
-    if "extras_require" in result:
-        result["extras_require"] = get_full_list("extras_require")
-    if "setup_requires" in result:
-        result["setup_requires"] = get_full_list("setup_requires")
-        if "setuptools-scm" in result["setup_requires"]:
-            result["setup_requires"].remove("setuptools-scm")
-    if "compilers" in result:
-        result["compilers"] = get_full_list("compilers")
+    merge_list_item(result, setup_cfg, "install_requires")
+    merge_list_item(result, setup_cfg, "setup_requires")
+    merge_list_item(result, setup_cfg, "compilers")
+    merge_dict_of_lists_item(result, setup_cfg, "extras_require")
+
+    if "setuptools-scm" in result.get("setup_requires", []):
+        result["setup_requires"].remove("setuptools-scm")
+
     return result
 
 

grayskull/utils.py

@@ -235,3 +235,19 @@ def _clean_yaml(recipe, all_obj_to_delete=None):
             value_to_delete = _clean_yaml(value)
         all_obj_to_delete.extend(value_to_delete)
     return all_obj_to_delete
+
+
+def merge_list_item(destination: dict, add: dict, key: str) -> None:
+    lst = set(destination.get(key, []))
+    lst |= set(add.get(key, []))
+    if lst:
+        destination[key] = list(lst)
+
+
+def merge_dict_of_lists_item(destination: dict, add: dict, key: str) -> None:
+    sub_destination = destination.get(key, {})
+    sub_add = add.get(key, {})
+    for sub_key in set(sub_destination) | set(sub_add):
+        merge_list_item(sub_destination, sub_add, sub_key)
+    if sub_destination:
+        destination[key] = sub_destination

tests/test_pypi.py

@@ -122,6 +122,33 @@ def test_get_extra_from_requires_dist():
     ]
 
 
+def test_get_extra_requirements():
+    config = Configuration(name="pypushflow")
+    data = get_sdist_metadata(
+        "https://pypi.io/packages/source/p/pypushflow/pypushflow-0.3.0rc2.tar.gz",
+        config,
+    )
+    received = {extra: set(lst) for extra, lst in data["extras_require"].items()}
+    expected = {
+        "mx": {"pymongo >=4, <5"},
+        "test": {
+            "mongita >=1, <2",
+            "pytest >=7, <8",
+            "psutil >=5.8, <6",
+            "pytest-subtests >=0.4, <1",
+        },
+        "dev": {
+            "pytest >=7, <8",
+            "black >=22, <23",
+            "psutil >=5.8, <6",
+            "flake8 >=4, <5",
+            "mongita >=1, <2",
+            "pytest-subtests >=0.4, <1",
+        },
+    }
+    assert received == expected
+
+
 def test_get_all_selectors_pypi(recipe_config):
     _, config = recipe_config
     config.version = "5.3.1"

tests/test_utils.py

@@ -7,6 +7,8 @@
     get_all_modules_imported_script,
     get_std_modules,
     get_vendored_dependencies,
+    merge_dict_of_lists_item,
+    merge_list_item,
     origin_is_local_sdist,
 )
 
@@ -57,3 +59,62 @@ def test_origin_is_not_local_sdist_file(tmp_path):
 )
 def test_origin_is_not_local_sdist_filename(filename):
     assert not origin_is_local_sdist(filename)
+
+
+def test_merge_lists_item():
+    destination = {}
+    add = {}
+    merge_list_item(destination, add, "name")
+    assert destination == {}
+
+    destination = {"name": [1]}
+    add = {}
+    merge_list_item(destination, add, "name")
+    destination = {key: set(lst) for key, lst in destination.items()}
+    assert destination == {"name": {1}}
+
+    destination = {}
+    add = {"name": [2]}
+    merge_list_item(destination, add, "name")
+    destination = {key: set(lst) for key, lst in destination.items()}
+    assert destination == {"name": {2}}
+
+    destination = {"name": [1]}
+    add = {"name": [2]}
+    merge_list_item(destination, add, "name")
+    destination = {key: set(lst) for key, lst in destination.items()}
+    assert destination == {"name": {1, 2}}
+
+
+def test_merge_dict_of_lists_item():
+    destination = {}
+    add = {}
+    merge_dict_of_lists_item(destination, add, "name")
+    assert destination == {}
+
+    destination = {"name": {"sub_name": [1]}}
+    add = {}
+    merge_dict_of_lists_item(destination, add, "name")
+    for key in destination:
+        destination[key] = {
+            sub_key: set(lst) for sub_key, lst in destination[key].items()
+        }
+    assert destination == {"name": {"sub_name": {1}}}
+
+    destination = {}
+    add = {"name": {"sub_name": [2]}}
+    merge_dict_of_lists_item(destination, add, "name")
+    for key in destination:
+        destination[key] = {
+            sub_key: set(lst) for sub_key, lst in destination[key].items()
+        }
+    assert destination == {"name": {"sub_name": {2}}}
+
+    destination = {"name": {"sub_name": [1]}}
+    add = {"name": {"sub_name": [2]}}
+    merge_dict_of_lists_item(destination, add, "name")
+    for key in destination:
+        destination[key] = {
+            sub_key: set(lst) for sub_key, lst in destination[key].items()
+        }
+    assert destination == {"name": {"sub_name": {1, 2}}}