diff --git a/src/python_inspector/api.py b/src/python_inspector/api.py index ce98ff31..ee858175 100644 --- a/src/python_inspector/api.py +++ b/src/python_inspector/api.py @@ -147,7 +147,7 @@ def resolve_dependencies( files = [] if PYPI_SIMPLE_URL not in index_urls: - index_urls = tuple([PYPI_SIMPLE_URL]) + tuple(index_urls) + index_urls = tuple(index_urls) + tuple([PYPI_SIMPLE_URL]) # requirements for req_file in requirement_files: diff --git a/src/python_inspector/resolve_cli.py b/src/python_inspector/resolve_cli.py index 047a8c78..3fbff9ad 100644 --- a/src/python_inspector/resolve_cli.py +++ b/src/python_inspector/resolve_cli.py @@ -89,6 +89,7 @@ def print_version(ctx, param, value): @click.option( "--index-url", "index_urls", + envvar="PYINSP_INDEX_URL", type=str, metavar="INDEX", show_default=True, @@ -120,6 +121,7 @@ def print_version(ctx, param, value): "--netrc", "netrc_file", type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + envvar="PYINSP_NETRC_FILE", metavar="NETRC-FILE", hidden=True, required=False, @@ -161,6 +163,7 @@ def print_version(ctx, param, value): ) @click.option( "--verbose", + envvar="PYINSP_VERBOSE", is_flag=True, help="Enable verbose debug output.", ) diff --git a/src/python_inspector/utils_pypi.py b/src/python_inspector/utils_pypi.py index 2af5d57f..263e412a 100644 --- a/src/python_inspector/utils_pypi.py +++ b/src/python_inspector/utils_pypi.py @@ -16,6 +16,7 @@ import shutil import tempfile import time + from collections import defaultdict from typing import List from typing import NamedTuple @@ -27,6 +28,7 @@ import attr import packageurl import requests + from bs4 import BeautifulSoup from commoncode import fileutils from commoncode.hash import multi_checksums @@ -215,7 +217,6 @@ def get_python_dot_version(version): class DistributionNotFound(Exception): pass - def download_wheel( name, version, @@ -252,6 +253,7 @@ def download_wheel( ) continue for wheel in supported_and_valid_wheels: + wheel.credentials = repo.credentials fetched_wheel_filename = wheel.download( dest_dir=dest_dir, verbose=verbose, @@ -1130,7 +1132,8 @@ def to_filename(self): pyvers = ".".join(self.python_versions) abis = ".".join(self.abis) plats = ".".join(self.platforms) - return f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl" + name = f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl" + return name def is_pure(self): """ @@ -1635,7 +1638,10 @@ def resolve_relative_url(package_url, url): path = urlunparse( ("", "", url_parts.path, url_parts.params, url_parts.query, url_parts.fragment) ) - resolved_url_parts = base_url_parts._replace(path=path) + if base_url_parts.path != "": + resolved_url_parts = base_url_parts._replace(path=base_url_parts.path + "/" + path) + else: + resolved_url_parts = base_url_parts._replace(path=path) url = urlunparse(resolved_url_parts) return url @@ -1678,6 +1684,8 @@ def get( True otherwise as treat as binary. `path_or_url` can be a path or a URL to a file. """ + + cache_key = quote_plus(path_or_url.strip("/")) cached = os.path.join(self.directory, cache_key) @@ -1782,21 +1790,25 @@ def get_remote_file_content( if verbose: echo_func(f"DOWNLOADING: {url}") - auth = None + if TRACE: + print(f"DOWNLOADING: {url}") + if credentials: auth = (credentials.get("login"), credentials.get("password")) + else: + auth = None stream = requests.get( url, allow_redirects=True, stream=True, headers=headers, - auth=auth, + auth=auth ) with stream as response: status = response.status_code - if status != requests.codes.ok: # NOQA + if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: # too many requests: start some exponential delay increased_delay = (_delay * 2) or 1 diff --git a/tests/test_cli.py b/tests/test_cli.py index 8491ad36..ff23ab62 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -160,6 +160,64 @@ def test_cli_with_multiple_index_url_and_tilde_req(): regen=REGEN_TEST_FIXTURES, ) +@pytest.mark.online +def test_cli_with_single_env_var_index_url_flag_override(): + # Click default is to override env vars via flag as shown here + expected_file = test_env.get_test_loc("single-url-env-var-expected.json", must_exist=True) + specifier = "zipp==3.8.0" + os.environ["PYINSP_INDEX_URL"] = "https://thirdparty.aboutcode.org/pypi/simple/" + extra_options = [ + "--index-url", + "https://pypi.org/simple", + ] + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=extra_options, + regen=REGEN_TEST_FIXTURES + ) + os.unsetenv("PYINSP_INDEX_URL") + +@pytest.mark.online +def test_cli_with_single_env_var_index_url_except_pypi_simple(): + expected_file = test_env.get_test_loc( + "single-url-env-var-except-simple-expected.json", must_exist=True) + # using flask since it's not present in thirdparty + specifier = "flask" + os.environ["PYINSP_INDEX_URL"] = "https://thirdparty.aboutcode.org/pypi/simple/" + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=[], + regen=REGEN_TEST_FIXTURES, + ) + os.unsetenv("PYINSP_INDEX_URL") + +@pytest.mark.online +def test_cli_with_multiple_env_var_index_url_and_tilde_req(): + expected_file = test_env.get_test_loc("tilde_req-expected.json", must_exist=True) + specifier = "zipp~=3.8.0" + os.environ["PYINSP_INDEX_URL"] = "https://pypi.org/simple https://thirdparty.aboutcode.org/pypi/simple/" + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=[], + regen=REGEN_TEST_FIXTURES, + ) + os.unsetenv("PYINSP_INDEX_URL") + +@pytest.mark.online +def test_cli_with_single_env_var_index_url(): + expected_file = test_env.get_test_loc("single-url-env-var-expected.json", must_exist=True) + specifier = "zipp==3.8.0" + os.environ["PYINSP_INDEX_URL"] = "https://pypi.org/simple" + check_specs_resolution( + specifier=specifier, + expected_file=expected_file, + extra_options=[], + regen=REGEN_TEST_FIXTURES + ) + os.unsetenv("PYINSP_INDEX_URL") @pytest.mark.online def test_cli_with_environment_marker_and_complex_ranges():