Merge latest main branch

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

README.rst

@@ -1,4 +1,4 @@
-python-inspector - inspect Python packages dependencies and metadata
+python-inspector - inspect Python package dependencies and metadata
 =====================================================================
 
 
@@ -14,7 +14,7 @@ Homepage: https://github.com/aboutcode-org/python-inspector and https://www.abou
 - parse various requirements.txt files and setup.py files as input
   for resolving dependencies.
 
-- parse additionally various manifests and packages files such as
+- parse various manifests and packages files such as
   Pipfile, pyproject.toml, poetry.lock and setup.cfg and legacy and
   current metadata file formats for eggs, wheels and sdist. These
   have not been wired with the command line yet.
@@ -35,7 +35,7 @@ Testing
 
     pytest -vvs
 
-- These are live tests to regenrate the tests with updated data run::
+- There are live tests to regenerate the tests with updated data run::
 
       PYINSP_REGEN_TEST_FIXTURES=yes pytest -vvs
 

src/python_inspector/api.py

@@ -153,8 +153,7 @@ def resolve_dependencies(
     for req_file in requirement_files:
         deps = dependencies.get_dependencies_from_requirements(requirements_file=req_file)
         for extra_data in dependencies.get_extra_data_from_requirements(requirements_file=req_file):
-            index_urls = (*index_urls, *tuple(extra_data.get("extra_index_urls") or []))
-            index_urls = (*index_urls, *tuple(extra_data.get("index_url") or []))
+            index_urls = get_index_urls(index_urls, extra_data)
         direct_dependencies.extend(deps)
         package_data = [
             pkg_data.to_dict() for pkg_data in PipRequirementsFileHandler.parse(location=req_file)
@@ -321,6 +320,25 @@ async def get_pypi_data(package):
     )
 
 
+def get_index_urls(index_urls: Tuple, extra_data: Dict) -> Tuple:
+    """
+    Return a list of index URLs from the extra_data.
+    The extra_data is a dictionary that may contain
+    "extra_index_urls" and "index_url" keys.
+    """
+    if isinstance(index_urls, str):
+        index_urls = [index_urls]
+    if isinstance(index_urls, tuple):
+        index_urls = list(index_urls)
+    extra_index_urls = extra_data.get("extra_index_urls") or []
+    index_url = extra_data.get("index_url") or []
+    if isinstance(extra_index_urls, list):
+        index_urls = (*index_urls, *tuple(extra_index_urls))
+    if isinstance(index_url, str):
+        index_urls = (*index_urls, *tuple([index_url]))
+    return tuple(set(index_urls))
+
+
 resolver_api = resolve_dependencies
 
 

src/python_inspector/resolution.py

@@ -320,10 +320,6 @@ def get_requirements_from_python_manifest(
                         and elem.value.func.id == "setup"
                     )
                 ]
-                if len(setup_fct) == 0:
-                    raise Exception(
-                        f"Unable to collect setup.py dependencies securely: {setup_py_location}"
-                    )
                 if len(setup_fct) > 1:
                     print(
                         f"Warning: identified multiple definitions of 'setup()' in {setup_py_location}, "
@@ -333,20 +329,17 @@ def get_requirements_from_python_manifest(
                 install_requires = [
                     k.value for k in setup_fct.value.keywords if k.arg == "install_requires"
                 ]
-                if len(install_requires) == 0:
-                    raise Exception(
-                        f"Unable to collect setup.py dependencies securely: {setup_py_location}"
-                    )
-                if len(install_requires) > 1:
-                    print(
-                        f"Warning: identified multiple definitions of 'install_requires' in "
-                        "{setup_py_location}, defaulting to the first occurrence"
-                    )
-                install_requires = install_requires[0].elts
-                if len(install_requires) != 0:
-                    raise Exception(
-                        f"Unable to collect setup.py dependencies securely: {setup_py_location}"
-                    )
+                if install_requires:
+                    if len(install_requires) > 1:
+                        print(
+                            f"Warning: identified multiple definitions of 'install_requires' in "
+                            "{setup_py_location}, defaulting to the first occurrence"
+                        )
+                    install_requires = install_requires[0].elts
+                    if len(install_requires) != 0:
+                        raise Exception(
+                            f"Unable to collect setup.py dependencies securely: {setup_py_location}"
+                        )
 
 
 DEFAULT_ENVIRONMENT = utils_pypi.Environment.from_pyver_and_os(

src/python_inspector/utils_pypi.py

@@ -10,6 +10,7 @@
 #
 import asyncio
 import email
+import hashlib
 import itertools
 import os
 import pathlib
@@ -1661,6 +1662,9 @@ class Cache:
     def __attrs_post_init__(self):
         os.makedirs(self.directory, exist_ok=True)
 
+    def sha256_hash(self, text: str) -> str:
+        return hashlib.sha256(text.encode()).hexdigest()
+
     async def get(
         self,
         credentials,
@@ -1676,7 +1680,7 @@ async def get(
         True otherwise as treat as binary. `path_or_url` can be a path or a URL
         to a file.
         """
-        cache_key = quote_plus(path_or_url.strip("/"))
+        cache_key = self.sha256_hash(quote_plus(path_or_url.strip("/")))
         cached = os.path.join(self.directory, cache_key)
 
         if force or not os.path.exists(cached):

tests/data/azure-devops.req-310-expected.json

@@ -225,12 +225,12 @@
       "type": "pypi",
       "namespace": null,
       "name": "packaging",
-      "version": "24.2",
+      "version": "25.0",
       "qualifiers": {},
       "subpath": null,
       "primary_language": "Python",
       "description": "Core utilities for Python packages\npackaging\n=========\n\n.. start-intro\n\nReusable core utilities for various Python Packaging\n`interoperability specifications <https://packaging.python.org/specifications/>`_.\n\nThis library provides utilities that implement the interoperability\nspecifications which have clearly one correct behaviour (eg: :pep:`440`)\nor benefit greatly from having a single shared implementation (eg: :pep:`425`).\n\n.. end-intro\n\nThe ``packaging`` project includes the following: version handling, specifiers,\nmarkers, requirements, tags, utilities.\n\nDocumentation\n-------------\n\nThe `documentation`_ provides information and the API for the following:\n\n- Version Handling\n- Specifiers\n- Markers\n- Requirements\n- Tags\n- Utilities\n\nInstallation\n------------\n\nUse ``pip`` to install these utilities::\n\n    pip install packaging\n\nThe ``packaging`` library uses calendar-based versioning (``YY.N``).\n\nDiscussion\n----------\n\nIf you run into bugs, you can file them in our `issue tracker`_.\n\nYou can also join ``#pypa`` on Freenode to ask questions or get involved.\n\n\n.. _`documentation`: https://packaging.pypa.io/\n.. _`issue tracker`: https://github.com/pypa/packaging/issues\n\n\nCode of Conduct\n---------------\n\nEveryone interacting in the packaging project's codebases, issue trackers, chat\nrooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.\n\n.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md\n\nContributing\n------------\n\nThe ``CONTRIBUTING.rst`` file outlines how to contribute to this project as\nwell as how to report a potential security issue. The documentation for this\nproject also covers information about `project development`_ and `security`_.\n\n.. _`project development`: https://packaging.pypa.io/en/latest/development/\n.. _`security`: https://packaging.pypa.io/en/latest/security/\n\nProject History\n---------------\n\nPlease review the ``CHANGELOG.rst`` file or the `Changelog documentation`_ for\nrecent changes and project history.\n\n.. _`Changelog documentation`: https://packaging.pypa.io/en/latest/changelog/",
-      "release_date": "2024-11-08T09:47:44",
+      "release_date": "2025-04-19T11:48:57",
       "parties": [
         {
           "type": "person",
@@ -257,11 +257,11 @@
         "Typing :: Typed"
       ],
       "homepage_url": null,
-      "download_url": "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl",
-      "size": 65451,
+      "download_url": "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl",
+      "size": 66469,
       "sha1": null,
-      "md5": "137b07612433f1ad2cd27dd8ab38ce49",
-      "sha256": "09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759",
+      "md5": "5fa4842e2eb0d7883b4b0e7c42d6229e",
+      "sha256": "29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484",
       "sha512": null,
       "bug_tracking_url": null,
       "code_view_url": "https://github.com/pypa/packaging",
@@ -281,9 +281,9 @@
       "dependencies": [],
       "repository_homepage_url": null,
       "repository_download_url": null,
-      "api_data_url": "https://pypi.org/pypi/packaging/24.2/json",
+      "api_data_url": "https://pypi.org/pypi/packaging/25.0/json",
       "datasource_id": null,
-      "purl": "pkg:pypi/packaging@24.2"
+      "purl": "pkg:pypi/packaging@25.0"
     },
     {
       "type": "pypi",
@@ -492,7 +492,7 @@
       "dependencies": []
     },
     {
-      "package": "pkg:pypi/packaging@24.2",
+      "package": "pkg:pypi/packaging@25.0",
       "dependencies": []
     },
     {
@@ -504,7 +504,7 @@
       "dependencies": [
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]",
-        "pkg:pypi/packaging@24.2",
+        "pkg:pypi/packaging@25.0",
         "pkg:pypi/[email protected]",
         "pkg:pypi/[email protected]"
       ]

tests/data/azure-devops.req-312-expected.json

@@ -0,0 +1,92 @@
+{
+  "headers": {
+    "tool_name": "python-inspector",
+    "tool_homepageurl": "https://github.com/aboutcode-org/python-inspector",
+    "tool_version": "0.13.0",
+    "options": [
+      "--index-url https://pypi.org/simple",
+      "--json <file>",
+      "--operating-system linux",
+      "--python-version 38",
+      "--specifier crontab==1.0.4"
+    ],
+    "notice": "Dependency tree generated with python-inspector.\npython-inspector is a free software tool from nexB Inc. and others.\nVisit https://github.com/aboutcode-org/python-inspector/ for support and download.",
+    "warnings": [],
+    "errors": []
+  },
+  "files": [],
+  "packages": [
+    {
+      "type": "pypi",
+      "namespace": null,
+      "name": "crontab",
+      "version": "1.0.4",
+      "qualifiers": {},
+      "subpath": null,
+      "primary_language": "Python",
+      "description": "Parse and use crontab schedules in Python\nCopyright 2011-2021 Josiah Carlson\n\nReleased under the LGPL license version 2.1 and version 3 (you can choose\nwhich you'd like to be bound under).\n\nDescription\n===========\n\nThis package intends to offer a method of parsing crontab schedule entries and\ndetermining when an item should next be run. More specifically, it calculates\na delay in seconds from when the .next() method is called to when the item\nshould next be executed.\n\nComparing the below chart to http://en.wikipedia.org/wiki/Cron#CRON_expression\nyou will note that W and # symbols are not supported.\n\n============= =========== ================= ============== ===========================\nField Name    Mandatory   Allowed Values    Default Value  Allowed Special Characters\n============= =========== ================= ============== ===========================\nSeconds       No          0-59              0              \\* / , -\nMinutes       Yes         0-59              N/A            \\* / , -\nHours         Yes         0-23              N/A            \\* / , -\nDay of month  Yes         1-31              N/A            \\* / , - ? L\nMonth         Yes         1-12 or JAN-DEC   N/A            \\* / , -\nDay of week   Yes         0-6 or SUN-SAT    N/A            \\* / , - ? L\nYear          No          1970-2099         *              \\* / , -\n============= =========== ================= ============== ===========================\n\nIf your cron entry has 5 values, minutes-day of week are used, default seconds\nis and default year is appended. If your cron entry has 6 values, minutes-year\nare used, and default seconds are prepended.\n\nAs such, only 5-7 value crontab entries are accepted (and mangled to 7 values,\nas necessary).\n\n\nSample individual crontab fields\n================================\n\nExamples of supported entries are as follows::\n\n    *\n    */5\n    7/8\n    3-25/7\n    3,7,9\n    0-10,30-40/5\n\nFor month or day of week entries, 3 letter abbreviations of the month or day\ncan be used to the left of any optional / where a number could be used.\n\nFor days of the week::\n\n    mon-fri\n    sun-thu/2\n\nFor month::\n\n    apr-jul\n    mar-sep/3\n\nInstallation\n============\n\n::\n\n    pip install crontab\n\n\nExample uses\n============\n\n::\n\n    >>> from crontab import CronTab\n    >>> from datetime import datetime\n    >>> # define the crontab for 25 minutes past the hour every hour\n    ... entry = CronTab('25 * * * *')\n    >>> # find the delay from when this was run (around 11:13AM)\n    ... entry.next()\n    720.81637899999998\n    >>> # find the delay from when it was last scheduled\n    ... entry.next(datetime(2011, 7, 17, 11, 25))\n    3600.0\n\n\n\n\nNotes\n=====\n\nAt most one of 'day of week' or 'day of month' can be a value other than '?'\nor '*'. We violate spec here and allow '*' to be an alias for '?', in the case\nwhere one of those values is specified (seeing as some platforms don't support\n'?').\n\nThis module also supports the convenient aliases::\n\n    @yearly\n    @annually\n    @monthly\n    @weekly\n    @daily\n    @hourly\n\nExample full crontab entries and their meanings::\n\n    30 */2 * * * -> 30 minutes past the hour every 2 hours\n    15,45 23 * * * -> 11:15PM and 11:45PM every day\n    0 1 ? * SUN -> 1AM every Sunday\n    0 1 * * SUN -> 1AM every Sunday (same as above)\n    0 0 1 jan/2 * 2011-2013 ->\n        midnight on January 1, 2011 and the first of every odd month until\n        the end of 2013\n    24 7 L * * -> 7:24 AM on the last day of every month\n    24 7 * * L5 -> 7:24 AM on the last friday of every month\n    24 7 * * Lwed-fri ->\n        7:24 AM on the last wednesday, thursday, and friday of every month",
+      "release_date": "2025-04-09T18:23:59",
+      "parties": [
+        {
+          "type": "person",
+          "role": "author",
+          "name": "Josiah Carlson",
+          "email": "[email protected]",
+          "url": null
+        }
+      ],
+      "keywords": [
+        "Development Status :: 5 - Production/Stable",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.2",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9"
+      ],
+      "homepage_url": "https://github.com/josiahcarlson/parse-crontab",
+      "download_url": "https://files.pythonhosted.org/packages/1e/8b/3ea72ac8e26090b63779b4e0074af79b02bbbab7ddd01b36109bc0892d31/crontab-1.0.4.tar.gz",
+      "size": 21677,
+      "sha1": null,
+      "md5": "ad190b69ff4199c44a5170daf896e73f",
+      "sha256": "715b0e5e105bc62c9683cbb93c1cc5821e07a3e28d17404576d22dba7a896c92",
+      "sha512": null,
+      "bug_tracking_url": null,
+      "code_view_url": null,
+      "vcs_url": null,
+      "copyright": null,
+      "license_expression": null,
+      "declared_license": {
+        "license": "GNU LGPL v2.1",
+        "classifiers": [
+          "License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)",
+          "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+          "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)"
+        ]
+      },
+      "notice_text": null,
+      "source_packages": [],
+      "file_references": [],
+      "extra_data": {},
+      "dependencies": [],
+      "repository_homepage_url": null,
+      "repository_download_url": null,
+      "api_data_url": "https://pypi.org/pypi/crontab/1.0.4/json",
+      "datasource_id": null,
+      "purl": "pkg:pypi/[email protected]"
+    }
+  ],
+  "resolved_dependencies_graph": [
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    }
+  ]
+}

tests/data/azure-devops.req-313-expected.json

@@ -0,0 +1,57 @@
+[
+  [
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]",
+        "pkg:pypi/[email protected]",
+        "pkg:pypi/[email protected]",
+        "pkg:pypi/[email protected]"
+      ]
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]"
+      ]
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]"
+      ]
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]"
+      ]
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    }
+  ],
+  [
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]"
+  ]
+]

tests/data/azure-devops.req-38-expected.json

@@ -0,0 +1,45 @@
+[
+  [
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]",
+        "pkg:pypi/[email protected]",
+        "pkg:pypi/[email protected]",
+        "pkg:pypi/[email protected]"
+      ]
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]"
+      ]
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": []
+    },
+    {
+      "package": "pkg:pypi/[email protected]",
+      "dependencies": [
+        "pkg:pypi/[email protected]"
+      ]
+    }
+  ],
+  [
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]",
+    "pkg:pypi/[email protected]"
+  ]
+]