diff --git a/src/docstub/_cli.py b/src/docstub/_cli.py index 0820c42..5254d40 100644 --- a/src/docstub/_cli.py +++ b/src/docstub/_cli.py @@ -17,8 +17,8 @@ from ._config import Config from ._path_utils import ( STUB_HEADER_COMMENT, - walk_python_package, walk_source_and_targets, + walk_source_package, ) from ._stubs import Py2StubTransformer, try_format_stub from ._utils import ErrorReporter, GroupedErrorReporter, module_name_from_path @@ -100,7 +100,7 @@ def _collect_type_info(root_path, *, ignore=()): type_prefixes = {} if root_path.is_dir(): - for source_path in walk_python_package(root_path, ignore=ignore): + for source_path in walk_source_package(root_path, ignore=ignore): module = module_name_from_path(source_path) module = module.replace(".", "/") @@ -228,8 +228,8 @@ def run(root_path, out_dir, config_paths, ignore, group_errors, allow_errors, ve root_path = Path(root_path) if root_path.is_file(): logger.warning( - "Running docstub on a single file is experimental. Relative imports " - "or type references won't work." + "Running docstub on a single file. Relative imports " + "or type references outside this file won't work." ) config = _load_configuration(config_paths) diff --git a/src/docstub/_path_utils.py b/src/docstub/_path_utils.py index f29a1b6..8560cdc 100644 --- a/src/docstub/_path_utils.py +++ b/src/docstub/_path_utils.py @@ -17,27 +17,68 @@ STUB_HEADER_COMMENT = "# File generated with docstub" -def is_docstub_generated(path): +def is_docstub_generated(stub_path): """Check if the stub file was generated by docstub. Parameters ---------- - path : Path + stub_path : Path + Path to a stub file. Returns ------- is_generated : bool + + Examples + -------- + >>> from pathlib import Path + >>> from docstub import _version + >>> is_docstub_generated(Path(_version.__file__).with_suffix(".pyi")) + False + + >>> is_docstub_generated(Path(__file__)) + Traceback (most recent call last): + ... + TypeError: expected stub file (ending with '.pyi'), ... """ - assert path.suffix == ".pyi" - with path.open("r") as fo: + if stub_path.suffix != ".pyi": + raise TypeError(f"expected stub file (ending with '.pyi'), got {stub_path}") + with stub_path.open("r") as fo: content = fo.read() if re.match(f"^{re.escape(STUB_HEADER_COMMENT)}", content): return True return False -def is_python_package(path): +def is_python_or_stub_file(path): + """Check whether `path` is a Python source file. + + Parameters + ---------- + path : Path + + Returns + ------- + is_python_or_stub_file : bool + + See Also + -------- + is_python_package_dir + + Examples + -------- + >>> from pathlib import Path + >>> is_python_or_stub_file(Path(__file__)) + True + >>> is_python_or_stub_file(Path(__file__).parent) + False """ + return path.is_file() and path.suffix in (".py", ".pyi") + + +def is_python_package_dir(path): + """Check whether `path` is a valid Python package and a directory. + Parameters ---------- path : Path @@ -46,14 +87,18 @@ def is_python_package(path): ------- is_package : bool + See Also + -------- + is_python_or_stub_file + Examples -------- >>> from pathlib import Path - >>> is_python_package(Path(__file__)) + >>> is_python_package_dir(Path(__file__)) False - >>> is_python_package(Path(__file__).parent) + >>> is_python_package_dir(Path(__file__).parent) True - >>> is_python_package(Path(__file__).parent.parent) + >>> is_python_package_dir(Path(__file__).parent.parent) False """ has_init = (path / "__init__.py").is_file() or (path / "__init__.pyi").is_file() @@ -84,7 +129,7 @@ def find_package_root(path): root = root.parent for _ in range(2**16): - if not is_python_package(root): + if not is_python_package_dir(root): logger.debug("detected %s as the package root of %s", root, path) return root root = root.parent @@ -95,7 +140,7 @@ def find_package_root(path): @lru_cache(maxsize=10) def glob_patterns_to_regex(patterns, relative_to=None): - r"""Combine glob-style patterns into a single regex. + r"""Combine glob-style patterns into a single regex [1]. Parameters ---------- @@ -106,6 +151,10 @@ def glob_patterns_to_regex(patterns, relative_to=None): ------- regex : re.Pattern | None + References + ---------- + .. [1] https://docs.python.org/3/library/glob.html#glob.translate + Examples -------- >>> from pathlib import Path @@ -143,17 +192,59 @@ def prefix(pattern): return regex -def walk_python_package(root_dir, *, ignore=()): +def _walk_source_package(path, *, ignore_regex): """Iterate source files in a Python package. + .. note:: + Inner function of :func:`walk_source_package`. See that function + for more details. + + Parameters + ---------- + path : Path + Root directory of a Python package. Can also be a single Python or stub + file. + ignore_regex : re.Pattern + Don't yield files matching this regex-compiled glob-like pattern. + + Yields + ------ + source_path : Path + Either a Python file or a stub file that takes precedence. + """ + if ignore_regex and ignore_regex.match(str(path)): + logger.info("ignoring %s", path) + return + + if is_python_package_dir(path): + for sub_path in path.iterdir(): + yield from _walk_source_package(sub_path, ignore_regex=ignore_regex) + + elif is_python_or_stub_file(path): + stub_path = path.with_suffix(".pyi") + if stub_path == path or not stub_path.is_file(): + # If `path` is a stub file return it. If it is a regular Python + # file, only return it if no corresponding stub file exists. + yield path + + elif path.is_dir(): + logger.debug("skipping directory %s which isn't a Python package", path) + + elif path.is_file(): + logger.debug("skipping non-Python file %s", path) + + +def walk_source_package(path, *, ignore=()): + """Iterate over a source package for docstub. + Given a Python package, yield the path of contained Python modules. If an alternate stub file already exists and isn't generated by docstub, it is returned instead. Parameters ---------- - root_dir : Path - Root directory of a Python package. + path : Path + A Python package, either a directory or a single file. ignore : Sequence[str], optional Don't yield files matching these glob-like patterns. The pattern is interpreted relative to the root of the Python package unless it starts @@ -163,36 +254,61 @@ def walk_python_package(root_dir, *, ignore=()): Yields ------ source_path : Path - Either a Python file or a stub file that takes precedence. + Either a Python file or a stub file that takes precedence. Note that + stub files generated by docstub itself are not returned. + + Raises + ------ + TypeError + If `path` is not a valid Python package. Note that a single + Python file is considered a "package". + + See Also + -------- + walk_source_and_targets + + Examples + -------- + >>> from pathlib import Path + >>> this_file = Path(__file__) + + Walk `path` to current file + >>> package_files = sorted(walk_source_package(this_file)) + >>> len(package_files) + 1 + >>> package_files[0].as_posix() + '.../docstub/_path_utils.py' + + Walk `path` to directory of current file + >>> package_files = walk_source_package(this_file.parent) + >>> sorted(package_files) + [.../docstub/__init__.py'), ...] + + Ignoring all files ending with '.py' will return nothing + >>> next(walk_source_package(this_file.parent, ignore=("*.py"))) + Traceback (most recent call last): + ... + StopIteration """ - package_root = find_package_root(root_dir) - regex = glob_patterns_to_regex(tuple(ignore), relative_to=package_root) + if not is_python_package_dir(path) and not is_python_or_stub_file(path): + raise TypeError(f"{path} must be a Python file or package") - if regex and regex.match(str(root_dir)): - logger.info("ignoring %s", root_dir) - return + regex = glob_patterns_to_regex(tuple(ignore), relative_to=path) - for path in root_dir.iterdir(): - if regex and regex.match(str(path.resolve())): - logger.info("ignoring %s", path) - continue - if path.is_dir(): - if is_python_package(path): - yield from walk_python_package(path, ignore=ignore) - else: - logger.debug("skipping directory %s which isn't a Python package", path) - continue - - assert path.is_file() - suffix = path.suffix.lower() - - if suffix == ".py": - stub = path.with_suffix(".pyi") - if stub.exists() and not is_docstub_generated(stub): - # Non-generated stub file already exists and takes precedence - yield stub - else: - yield path + if is_python_or_stub_file(path): + stub_file = path.with_suffix(".pyi") + if ( + stub_file != path + and stub_file.is_file() + and not is_docstub_generated(stub_file) + ): + # Special case: `path` is a Python file for which a stub file + # exists, we want to return that one while taking into account + # `ignore` and other logic. A simple way to do so is to just pass + # the stub file instead of `path`. + path = stub_file + + yield from _walk_source_package(path, ignore_regex=regex) def walk_source_and_targets(root_path, target_dir, *, ignore=()): @@ -216,12 +332,37 @@ def walk_source_and_targets(root_path, target_dir, *, ignore=()): Either a Python file or a stub file that takes precedence. stub_path : Path Target stub file. + + Raises + ------ + TypeError + If `root_path` is not a valid Python package. Note that a single + Python file is considered a "package". + + See Also + -------- + walk_source_package + + Examples + -------- + >>> from pathlib import Path + >>> current_root = Path(__file__).parent + >>> sources_n_targets = sorted( + ... walk_source_and_targets(current_root, target_dir=current_root) + ... ) + >>> source_path, stub_path = sources_n_targets[0] + >>> source_path.as_posix() + '.../docstub/__init__.py' + >>> stub_path.as_posix() + '.../docstub/__init__.pyi' + >>> stub_path.is_file() + False """ if root_path.is_file(): stub_path = target_dir / root_path.with_suffix(".pyi").name yield root_path, stub_path return - for source_path in walk_python_package(root_path, ignore=ignore): + for source_path in walk_source_package(root_path, ignore=ignore): stub_path = target_dir / source_path.with_suffix(".pyi").relative_to(root_path) yield source_path, stub_path diff --git a/tests/test_path_utils.py b/tests/test_path_utils.py index 7be7a2a..186893c 100644 --- a/tests/test_path_utils.py +++ b/tests/test_path_utils.py @@ -1,30 +1,104 @@ -from docstub._path_utils import walk_python_package +import pytest +from docstub._path_utils import walk_source_package -class Test_walk_python_package: - def test_ignore(self, tmp_path): + +class Test_walk_source_package: + def test_single_file(self, tmp_path): + top_script = tmp_path / "script.py" + top_script.touch() + + paths = sorted(walk_source_package(top_script)) + assert paths == [top_script] + + def test_single_stub_precedence(self, tmp_path): + # Check that alternate stub file takes precedence + top_script = tmp_path / "script.py" + top_script.touch() + top_stub = tmp_path / "script.pyi" + top_stub.touch() + + paths = sorted(walk_source_package(top_script)) + assert paths == [top_stub] + + def test_single_stub(self, tmp_path): + top_stub = tmp_path / "script.pyi" + top_stub.touch() + paths = sorted(walk_source_package(top_stub)) + assert paths == [top_stub] + + def test_package_stub(self, tmp_path): + init_py = tmp_path / "__init__.py" + init_py.touch() + init_stub = tmp_path / "__init__.pyi" + init_stub.touch() + script_py = tmp_path / "_version.py" + script_py.touch() + script_stub = tmp_path / "_version.pyi" + script_stub.touch() + + paths = sorted(walk_source_package(tmp_path)) + assert paths == [init_stub, script_stub] + + def test_not_a_package(self, tmp_path): top_script = tmp_path / "script.py" top_script.touch() + + with pytest.raises(TypeError, match=".* must be a Python file or package"): + next(walk_source_package(tmp_path)) + + @pytest.mark.parametrize("name", ["script.py", "script.pyi"]) + def test_ignore_single_file(self, tmp_path, name): + top_stub = tmp_path / name + top_stub.touch() + paths = sorted(walk_source_package(top_stub, ignore=name)) + assert paths == [] + + def test_ignore_pyi(self, tmp_path): + for name in ("__init__.py", "script.py"): + (tmp_path / name).touch() + (tmp_path / name).with_suffix(".pyi").touch() + (tmp_path / "sub").mkdir(exist_ok=True) + (tmp_path / "sub" / name).touch() + (tmp_path / "sub" / name).with_suffix(".pyi").touch() + + paths = sorted(walk_source_package(tmp_path)) + assert len(paths) == 4 + + paths = sorted(walk_source_package(tmp_path, ignore="*.pyi")) + assert paths == [] + + paths = sorted(walk_source_package(tmp_path, ignore="**/*.pyi")) + assert paths == [] + + def test_ignore(self, tmp_path): + top_init = tmp_path / "__init__.py" + top_init.touch() sub_package = tmp_path / "sub_package" sub_package.mkdir() sub_init = sub_package / "__init__.py" sub_init.touch() - module_in_dir = sub_package / "module_in_dir.py" - module_in_dir.touch() + module_in_sub_package = sub_package / "module_in_sub_package.py" + module_in_sub_package.touch() + stub_in_sub_package = sub_package / "module_in_sub_package.pyi" + stub_in_sub_package.touch() + + paths = sorted(walk_source_package(tmp_path)) + assert paths == [top_init, sub_init, stub_in_sub_package] - paths = set(walk_python_package(tmp_path)) - assert paths == {top_script, sub_init, module_in_dir} + paths = sorted(walk_source_package(tmp_path, ignore=["**/*.py"])) + assert paths == [stub_in_sub_package] - paths = set(walk_python_package(tmp_path, ignore=["**/*.py"])) - assert paths == set() + paths = sorted(walk_source_package(tmp_path, ignore=["**/*.py*"])) + assert paths == [] - paths = set( - walk_python_package(tmp_path, ignore=["**/module*", "**/script.py"]) + paths = sorted( + walk_source_package(tmp_path, ignore=["**/module*", "__init__.py"]) ) - assert paths == {sub_init} + assert paths == [sub_init] - paths = set(walk_python_package(tmp_path, ignore=["**/sub_package"])) - assert paths == {top_script} + paths = sorted(walk_source_package(tmp_path, ignore=["**/sub_package"])) + assert paths == [top_init] - paths = set(walk_python_package(tmp_path, ignore=["**/*init*"])) - assert paths == {top_script, module_in_dir} + paths = sorted(walk_source_package(tmp_path, ignore=["**/*init*"])) + assert paths == [stub_in_sub_package]