From 498d1f59a220a9ab7bb163366c0ac7e20160c5c9 Mon Sep 17 00:00:00 2001 From: "Oriol (ZBook)" Date: Tue, 24 Jun 2025 20:40:48 +0200 Subject: [PATCH 1/8] first pass at keeping inline annotation over docstring --- examples/example_pkg-stubs/_basic.pyi | 6 +++++ examples/example_pkg/_basic.py | 25 +++++++++++++++++++ src/docstub/_stubs.py | 36 ++++++++++++++++++++++----- tests/test_stubs.py | 6 ++--- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/examples/example_pkg-stubs/_basic.pyi b/examples/example_pkg-stubs/_basic.pyi index b91c393..b5698c9 100644 --- a/examples/example_pkg-stubs/_basic.pyi +++ b/examples/example_pkg-stubs/_basic.pyi @@ -29,6 +29,12 @@ def func_contains( def func_literals( a1: Literal[1, 3, "foo"], a2: Literal["uno", 2, "drei", "four"] = ... ) -> None: ... +def override_docstring_param( + d1: dict[str, float], d2: dict[Literal["a", "b", "c"], int] +) -> None: ... +def override_docstring_return( + d1: dict[str, float], d2: dict[str, int] +) -> list[Literal[-1, 0, 1] | float]: ... def func_use_from_elsewhere( a1: CustomException, a2: ExampleClass, diff --git a/examples/example_pkg/_basic.py b/examples/example_pkg/_basic.py index b45b22b..3ebb72f 100644 --- a/examples/example_pkg/_basic.py +++ b/examples/example_pkg/_basic.py @@ -5,6 +5,7 @@ # Existing imports are preserved import logging +from typing import Literal # Assign-statements are preserved logger = logging.getLogger(__name__) # Inline comments are stripped @@ -51,6 +52,30 @@ def func_literals(a1, a2="uno"): """ +def override_docstring_param(d1, d2: dict[Literal["a", "b", "c"], int]): + """Check type hint is kept and overrides docstring. + + Parameters + ---------- + d1 : dict of {str : float} + d2 : dict of {str : int} + """ + + +def override_docstring_return(d1, d2) -> list[Literal[-1, 0, 1] | float]: + """Check type hint is kept and overrides docstring. + + Parameters + ---------- + d1 : dict of {str : float} + d2 : dict of {str : int} + + Returns + ------- + {"-inf", 0, 1, "inf"} + """ + + def func_use_from_elsewhere(a1, a2, a3, a4): """Check if types with full import names are matched. diff --git a/src/docstub/_stubs.py b/src/docstub/_stubs.py index 801ffac..cad78e6 100644 --- a/src/docstub/_stubs.py +++ b/src/docstub/_stubs.py @@ -572,23 +572,24 @@ def leave_FunctionDef(self, original_node, updated_node): annotation_value = ds_annotations.returns.value if original_node.returns is not None: + # TODO: either remove message or print only in verbose mode position = self.get_metadata( cst.metadata.PositionProvider, original_node ).start reporter = self.reporter.copy_with( path=self.current_source, line=position.line ) - replaced = _inline_node_as_code(original_node.returns.annotation) - details = ( - f"{replaced}\n{reporter.underline(replaced)} -> {annotation_value}" - ) + to_keep = _inline_node_as_code(original_node.returns.annotation) + details = f"{to_keep}\n{reporter.underline(to_keep)} instead of {annotation_value}" reporter.message( - short="Replacing existing inline return annotation", + short="Keeping existing inline return annotation", details=details, ) + annotation_value = to_keep annotation = cst.Annotation(cst.parse_expression(annotation_value)) node_changes["returns"] = annotation + # TODO: check imports self._required_imports |= ds_annotations.returns.imports elif original_node.returns is None: annotation = cst.Annotation(cst.parse_expression("None")) @@ -633,8 +634,31 @@ def leave_Param(self, original_node, updated_node): if pytype: if defaults_to_none: pytype = pytype.as_optional() - annotation = cst.Annotation(cst.parse_expression(pytype.value)) + annotation_value = pytype.value + if original_node.annotation is not None: + # TODO: either remove message or print only in verbose mode + position = self.get_metadata( + cst.metadata.PositionProvider, original_node + ).start + reporter = self.reporter.copy_with( + path=self.current_source, line=position.line + ) + to_keep = cst.Module([]).code_for_node( + original_node.annotation.annotation + ) + details = ( + f"{to_keep}\n" + f"{reporter.underline(to_keep)} instead of {annotation_value}" + ) + reporter.message( + short="Keeping existing inline parameter annotation", + details=details, + ) + annotation_value = to_keep + + annotation = cst.Annotation(cst.parse_expression(annotation_value)) node_changes["annotation"] = annotation + # TODO: check imports if pytype.imports: self._required_imports |= pytype.imports diff --git a/tests/test_stubs.py b/tests/test_stubs.py index ff96328..64aa4a6 100644 --- a/tests/test_stubs.py +++ b/tests/test_stubs.py @@ -317,11 +317,11 @@ def foo() -> str: ... def test_overwriting_typed_return(self, capsys): source = dedent( ''' - def foo() -> dict[str, int]: + def foo() -> int: """ Returns ------- - out : int + out : dict[str, int] """ pass ''' @@ -336,7 +336,7 @@ def foo() -> int: ... assert expected == result captured = capsys.readouterr() - assert "Replacing existing inline return annotation" in captured.out + assert "Keeping existing inline return annotation" in captured.out def test_preserved_type_comment(self): source = dedent( From 98a96a5c33346a72f323f0071428ef235ae30884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Sat, 12 Jul 2025 15:01:34 +0200 Subject: [PATCH 2/8] Make override_docsring tests more independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeping parameters is already tested in `override_docstring_param` above – no need to do so again. --- examples/example_pkg-stubs/_basic.pyi | 4 +--- examples/example_pkg/_basic.py | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/examples/example_pkg-stubs/_basic.pyi b/examples/example_pkg-stubs/_basic.pyi index b5698c9..6ca6ef3 100644 --- a/examples/example_pkg-stubs/_basic.pyi +++ b/examples/example_pkg-stubs/_basic.pyi @@ -32,9 +32,7 @@ def func_literals( def override_docstring_param( d1: dict[str, float], d2: dict[Literal["a", "b", "c"], int] ) -> None: ... -def override_docstring_return( - d1: dict[str, float], d2: dict[str, int] -) -> list[Literal[-1, 0, 1] | float]: ... +def override_docstring_return() -> list[Literal[-1, 0, 1] | float]: ... def func_use_from_elsewhere( a1: CustomException, a2: ExampleClass, diff --git a/examples/example_pkg/_basic.py b/examples/example_pkg/_basic.py index 3ebb72f..a443818 100644 --- a/examples/example_pkg/_basic.py +++ b/examples/example_pkg/_basic.py @@ -62,14 +62,9 @@ def override_docstring_param(d1, d2: dict[Literal["a", "b", "c"], int]): """ -def override_docstring_return(d1, d2) -> list[Literal[-1, 0, 1] | float]: +def override_docstring_return() -> list[Literal[-1, 0, 1] | float]: """Check type hint is kept and overrides docstring. - Parameters - ---------- - d1 : dict of {str : float} - d2 : dict of {str : int} - Returns ------- {"-inf", 0, 1, "inf"} From 26ccd67c1506dc3678c4a551ccad07e1cc4bf12c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Sat, 12 Jul 2025 15:04:35 +0200 Subject: [PATCH 3/8] Remove duplication in error message The `underline` method of the `ErrorReporter` includes the underlined line already. --- src/docstub/_stubs.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/docstub/_stubs.py b/src/docstub/_stubs.py index cad78e6..11e4ccd 100644 --- a/src/docstub/_stubs.py +++ b/src/docstub/_stubs.py @@ -580,7 +580,10 @@ def leave_FunctionDef(self, original_node, updated_node): path=self.current_source, line=position.line ) to_keep = _inline_node_as_code(original_node.returns.annotation) - details = f"{to_keep}\n{reporter.underline(to_keep)} instead of {annotation_value}" + details = ( + f"{reporter.underline(to_keep)} " + f"ignoring docstring: {annotation_value}" + ) reporter.message( short="Keeping existing inline return annotation", details=details, @@ -647,8 +650,8 @@ def leave_Param(self, original_node, updated_node): original_node.annotation.annotation ) details = ( - f"{to_keep}\n" - f"{reporter.underline(to_keep)} instead of {annotation_value}" + f"{reporter.underline(to_keep)} " + f"ignoring docstring: {annotation_value}" ) reporter.message( short="Keeping existing inline parameter annotation", @@ -802,7 +805,7 @@ def leave_AnnAssign(self, original_node, updated_node): updated_node.annotation.annotation ) details = ( - f"{replaced}\n{reporter.underline(replaced)} -> {pytype.value}" + f"{reporter.underline(to_keep)} ignoring docstring: {pytype.value}" ) reporter.message( short="Replacing existing inline annotation", From 2c7dc85d32fb10f11e8b5e78491e8c93d83faca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Sat, 12 Jul 2025 15:07:24 +0200 Subject: [PATCH 4/8] Simplify logic when ignoring docstring annotation Also makes sure that imports of a doctype are not unnecessarily included an inline annotation already exists. --- src/docstub/_stubs.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/docstub/_stubs.py b/src/docstub/_stubs.py index 11e4ccd..87d58c8 100644 --- a/src/docstub/_stubs.py +++ b/src/docstub/_stubs.py @@ -571,7 +571,14 @@ def leave_FunctionDef(self, original_node, updated_node): assert ds_annotations.returns.value annotation_value = ds_annotations.returns.value - if original_node.returns is not None: + if original_node.returns is None: + annotation = cst.Annotation(cst.parse_expression(annotation_value)) + node_changes["returns"] = annotation + # TODO: check imports + self._required_imports |= ds_annotations.returns.imports + + else: + # Notify about ignored docstring annotation # TODO: either remove message or print only in verbose mode position = self.get_metadata( cst.metadata.PositionProvider, original_node @@ -588,12 +595,7 @@ def leave_FunctionDef(self, original_node, updated_node): short="Keeping existing inline return annotation", details=details, ) - annotation_value = to_keep - annotation = cst.Annotation(cst.parse_expression(annotation_value)) - node_changes["returns"] = annotation - # TODO: check imports - self._required_imports |= ds_annotations.returns.imports elif original_node.returns is None: annotation = cst.Annotation(cst.parse_expression("None")) node_changes["returns"] = annotation @@ -638,7 +640,16 @@ def leave_Param(self, original_node, updated_node): if defaults_to_none: pytype = pytype.as_optional() annotation_value = pytype.value - if original_node.annotation is not None: + + if original_node.annotation is None: + annotation = cst.Annotation(cst.parse_expression(annotation_value)) + node_changes["annotation"] = annotation + # TODO: check imports + if pytype.imports: + self._required_imports |= pytype.imports + + else: + # Notify about ignored docstring annotation # TODO: either remove message or print only in verbose mode position = self.get_metadata( cst.metadata.PositionProvider, original_node @@ -657,13 +668,6 @@ def leave_Param(self, original_node, updated_node): short="Keeping existing inline parameter annotation", details=details, ) - annotation_value = to_keep - - annotation = cst.Annotation(cst.parse_expression(annotation_value)) - node_changes["annotation"] = annotation - # TODO: check imports - if pytype.imports: - self._required_imports |= pytype.imports # Potentially use "Incomplete" except for first param in (class)methods elif not is_self_or_cls and updated_node.annotation is None: From 792831c93f57ab2e363bd704ede1798397d0dc30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Sat, 12 Jul 2025 15:10:28 +0200 Subject: [PATCH 5/8] Keep existing inline annotation for assignments even if a different docstring annotation exists. This is the same behavior as for parameters and return types. --- examples/example_pkg-stubs/_basic.pyi | 3 +++ examples/example_pkg/_basic.py | 7 ++++++- src/docstub/_stubs.py | 20 +++++++++++--------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/examples/example_pkg-stubs/_basic.pyi b/examples/example_pkg-stubs/_basic.pyi index 6ca6ef3..7cdc627 100644 --- a/examples/example_pkg-stubs/_basic.pyi +++ b/examples/example_pkg-stubs/_basic.pyi @@ -41,6 +41,9 @@ def func_use_from_elsewhere( ) -> tuple[CustomException, ExampleClass.NestedClass]: ... class ExampleClass: + + b1: int + class NestedClass: def method_in_nested_class(self, a1: complex) -> None: ... diff --git a/examples/example_pkg/_basic.py b/examples/example_pkg/_basic.py index a443818..4f12dd0 100644 --- a/examples/example_pkg/_basic.py +++ b/examples/example_pkg/_basic.py @@ -95,10 +95,15 @@ class ExampleClass: ---------- a1 : str a2 : float, default 0 + + Attributes + ---------- + b1 : Sized """ - class NestedClass: + b1: int + class NestedClass: def method_in_nested_class(self, a1): """ diff --git a/src/docstub/_stubs.py b/src/docstub/_stubs.py index 87d58c8..ca00ca8 100644 --- a/src/docstub/_stubs.py +++ b/src/docstub/_stubs.py @@ -795,31 +795,33 @@ def leave_AnnAssign(self, original_node, updated_node): if pytypes and name in pytypes.attributes: pytype = pytypes.attributes[name] expr = cst.parse_expression(pytype.value) - self._required_imports |= pytype.imports - if updated_node.annotation is not None: - # Turn original annotation into str and print with context + if updated_node.annotation is None: + self._required_imports |= pytype.imports + updated_node = updated_node.with_deep_changes( + updated_node.annotation, annotation=expr + ) + + else: + # Notify about ignored docstring annotation + # TODO: either remove message or print only in verbose mode position = self.get_metadata( cst.metadata.PositionProvider, original_node ).start reporter = self.reporter.copy_with( path=self.current_source, line=position.line ) - replaced = cst.Module([]).code_for_node( + to_keep = cst.Module([]).code_for_node( updated_node.annotation.annotation ) details = ( f"{reporter.underline(to_keep)} ignoring docstring: {pytype.value}" ) reporter.message( - short="Replacing existing inline annotation", + short="Keeping existing inline annotation for assignment", details=details, ) - updated_node = updated_node.with_deep_changes( - updated_node.annotation, annotation=expr - ) - return updated_node def visit_Module(self, node): From 8a6aea4c8ee7174fda9921b8208a73fa0530121b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Sat, 12 Jul 2025 15:11:44 +0200 Subject: [PATCH 6/8] Add more tests --- tests/test_stubs.py | 132 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 126 insertions(+), 6 deletions(-) diff --git a/tests/test_stubs.py b/tests/test_stubs.py index 64aa4a6..60a1236 100644 --- a/tests/test_stubs.py +++ b/tests/test_stubs.py @@ -223,8 +223,8 @@ def test_attributes_no_doctype(self, assign, expected, scope): # ("plain = 3", "plain : int", "plain: int"), # ("plain = None", "plain : int", "plain: int"), # ("x, y = (1, 2)", "x : int", "x: int; y: Incomplete"), - # Replace pre-existing annotations - ("annotated: float = 1.0", "annotated : int", "annotated: int"), + # Keep pre-existing annotations + ("annotated: float = 1.0", "annotated : int", "annotated: float"), # Type aliases are untouched # ("alias: TypeAlias = int", "alias : str", "alias: TypeAlias = int"), # ("type alias = int", "alias : str", "type alias = int"), @@ -283,7 +283,7 @@ class Foo: a: int b: float - c: tuple + c: list d: ClassVar[bool] def __init__(self, a) -> None: ... @@ -298,7 +298,127 @@ def test_undocumented_objects(self): # https://typing.readthedocs.io/en/latest/guides/writing_stubs.html#undocumented-objects pass - def test_existing_typed_return(self): + def test_keep_assign_param(self): + source = dedent( + """ + a: str + """ + ) + expected = dedent( + """ + a: str + """ + ) + transformer = Py2StubTransformer() + result = transformer.python_to_stub(source) + assert expected == result + + def test_keep_inline_assign_with_doctype(self, capsys): + source = dedent( + ''' + """ + Attributes + ---------- + a : Sized + """ + a: str + ''' + ) + expected = dedent( + """ + a: str + """ + ) + transformer = Py2StubTransformer() + result = transformer.python_to_stub(source) + assert expected == result + + captured = capsys.readouterr() + assert "Keeping existing inline annotation for assignment" in captured.out + + def test_keep_class_assign_param(self): + source = dedent( + """ + class Foo: + a: str + """ + ) + expected = dedent( + """ + class Foo: + a: str + """ + ) + transformer = Py2StubTransformer() + result = transformer.python_to_stub(source) + assert expected == result + + def test_keep_inline_class_assign_with_doctype(self, capsys): + source = dedent( + ''' + class Foo: + """ + Attributes + ---------- + a : Sized + """ + a: str + ''' + ) + expected = dedent( + """ + class Foo: + a: str + """ + ) + transformer = Py2StubTransformer() + result = transformer.python_to_stub(source) + assert expected == result + + captured = capsys.readouterr() + assert "Keeping existing inline annotation for assignment" in captured.out + + def test_keep_inline_param(self): + source = dedent( + """ + def foo(a: str) -> None: + pass + """ + ) + expected = dedent( + """ + def foo(a: str) -> None: ... + """ + ) + transformer = Py2StubTransformer() + result = transformer.python_to_stub(source) + assert expected == result + + def test_keep_inline_param_with_doctype(self, capsys): + source = dedent( + ''' + def foo(a: int) -> None: + """ + Parameters + ---------- + a : Sized + """ + pass + ''' + ) + expected = dedent( + """ + def foo(a: int) -> None: ... + """ + ) + transformer = Py2StubTransformer() + result = transformer.python_to_stub(source) + assert expected == result + + captured = capsys.readouterr() + assert "Keeping existing inline parameter annotation" in captured.out + + def test_keep_inline_return(self): source = dedent( """ def foo() -> str: @@ -314,14 +434,14 @@ def foo() -> str: ... result = transformer.python_to_stub(source) assert expected == result - def test_overwriting_typed_return(self, capsys): + def test_keep_inline_return_with_doctype(self, capsys): source = dedent( ''' def foo() -> int: """ Returns ------- - out : dict[str, int] + out : Sized """ pass ''' From 4a2940504e2bfd6ac9fb2c672855cc2891d2f7e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Sat, 12 Jul 2025 15:12:23 +0200 Subject: [PATCH 7/8] Re-enable commented out tests Probably forgot to do so during debugging. --- tests/test_stubs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_stubs.py b/tests/test_stubs.py index 60a1236..4322ada 100644 --- a/tests/test_stubs.py +++ b/tests/test_stubs.py @@ -220,14 +220,14 @@ def test_attributes_no_doctype(self, assign, expected, scope): @pytest.mark.parametrize( ("assign", "doctype", "expected"), [ - # ("plain = 3", "plain : int", "plain: int"), - # ("plain = None", "plain : int", "plain: int"), - # ("x, y = (1, 2)", "x : int", "x: int; y: Incomplete"), + ("plain = 3", "plain : int", "plain: int"), + ("plain = None", "plain : int", "plain: int"), + ("x, y = (1, 2)", "x : int", "x: int; y: Incomplete"), # Keep pre-existing annotations ("annotated: float = 1.0", "annotated : int", "annotated: float"), # Type aliases are untouched - # ("alias: TypeAlias = int", "alias : str", "alias: TypeAlias = int"), - # ("type alias = int", "alias : str", "type alias = int"), + ("alias: TypeAlias = int", "alias : str", "alias: TypeAlias = int"), + ("type alias = int", "alias : str", "type alias = int"), ], ) @pytest.mark.parametrize("scope", ["module", "class", "nested class"]) From 629c65f6e85e301a616fc80e2e2e19d059076a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lars=20Gr=C3=BCter?= Date: Thu, 17 Jul 2025 13:09:10 +0200 Subject: [PATCH 8/8] Document inline annotation as overwrite solution --- docs/user_guide.md | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/user_guide.md b/docs/user_guide.md index 514fdbe..ac3a5da 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -172,9 +172,35 @@ Two command line options can help addressing these errors gradually: ## Dealing with typing problems -Docstub may not fully or correctly implement a particular part of Python's typing system yet. +For various reasons – missing features in docstub, or limitations of Python's typing system – it may not always be possible to correctly type something in a docstring. +In those cases, you docstub provides a few approaches to dealing with this. -In some cases, you can use a comment directive to selectively disable docstub for a specific block of lines: + +### Use inline type annotation + +Docstub will always preserve inline type annotations, regardless of what the docstring contains. +This is useful for example, if you want to express something that isn't yet supported by Python's type system. + +E.g., consider the docstring type of `ord` parameter in [`numpy.linalg.matrix_norm`](https://numpy.org/doc/stable/reference/generated/numpy.linalg.matrix_norm.html) +```rst +ord : {1, -1, 2, -2, inf, -inf, ‘fro’, ‘nuc’}, optional +``` +[Python's type system currently can't express floats as literal types](https://typing.python.org/en/latest/spec/literal.html#:~:text=Floats%3A%20e.g.%20Literal%5B3.14%5D) – such as `inf`. +We don't want to make the type description here less specific to users, so instead, you could handle this with a less constrained inline type annotation like +```python +ord: Literal[1, -1, 2, -2, 'fro', 'nuc'] | float +``` +Docstub will include the latter less constrained type in the stubs. +This allows you to keep the information in the docstring while still having valid – if a bit less constrained – stubs. + + +### Preserve code with comment directive + +At its heart, docstub transforms Python source files into stub files. +You can tell docstub to temporarily stop that transformation for a specific area with a comment directive. +Wrapping lines of code with `docstub: off` and `docstub: on` comments will preserve these lines completely. + +E.g., consider the following example: ```python class Foo: # docstub: off @@ -184,7 +210,7 @@ class Foo: c: int = None d: str = "" ``` -will leave the parameters within the `# docstub` guards untouched in the resulting stub file: +will leave the guarded parameters untouched in the resulting stub file: ```python class Foo: a: int = None @@ -193,5 +219,7 @@ class Foo: d: str ``` -If that is not possible, you can – for now – fallback to writing a correct stub file by hand. +### Write a manual stub file + +If all of the above does not solve your issue, you can fall back to writing a correct stub file by hand. Docstub will preserve this file and integrated it with other automatically generated stubs.