Skip to content

Commit a1b3a78

Browse files
Fix compatibility with Twisted 25 (#13502) (#13531)
As discussed in #13502, the fix for compatibility with Twisted 25+ is simpler. Therefore, it makes sense to implement both fixes (for Twisted 24 and Twisted 25) in parallel. This way, we can eventually drop support for Twisted <25 and keep only the simpler workaround. In addition, the `unittestextras` tox environment has been replaced with dedicated test environments for `asynctest`, `Twisted 24`, and `Twisted 25`. Fixes #13497 (cherry picked from commit 01dce85) Co-authored-by: Bruno Oliveira <[email protected]>
1 parent 4c161ab commit a1b3a78

File tree

4 files changed

+174
-44
lines changed

4 files changed

+174
-44
lines changed

.github/workflows/test.yml

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ jobs:
5454
fail-fast: false
5555
matrix:
5656
name: [
57-
"windows-py39-unittestextras",
57+
"windows-py39-unittest-asynctest",
58+
"windows-py39-unittest-twisted24",
59+
"windows-py39-unittest-twisted25",
5860
"windows-py39-pluggy",
5961
"windows-py39-xdist",
6062
"windows-py310",
@@ -63,6 +65,9 @@ jobs:
6365
"windows-py313",
6466
"windows-py314",
6567

68+
"ubuntu-py39-unittest-asynctest",
69+
"ubuntu-py39-unittest-twisted24",
70+
"ubuntu-py39-unittest-twisted25",
6671
"ubuntu-py39-lsof-numpy-pexpect",
6772
"ubuntu-py39-pluggy",
6873
"ubuntu-py39-freeze",
@@ -85,10 +90,23 @@ jobs:
8590
]
8691

8792
include:
88-
- name: "windows-py39-unittestextras"
93+
# Use separate jobs for different unittest flavors (twisted, asynctest) to ensure proper coverage.
94+
- name: "windows-py39-unittest-asynctest"
8995
python: "3.9"
9096
os: windows-latest
91-
tox_env: "py39-unittestextras"
97+
tox_env: "py39-asynctest"
98+
use_coverage: true
99+
100+
- name: "windows-py39-unittest-twisted24"
101+
python: "3.9"
102+
os: windows-latest
103+
tox_env: "py39-twisted24"
104+
use_coverage: true
105+
106+
- name: "windows-py39-unittest-twisted25"
107+
python: "3.9"
108+
os: windows-latest
109+
tox_env: "py39-twisted25"
92110
use_coverage: true
93111

94112
- name: "windows-py39-pluggy"
@@ -126,6 +144,25 @@ jobs:
126144
os: windows-latest
127145
tox_env: "py314"
128146

147+
# Use separate jobs for different unittest flavors (twisted, asynctest) to ensure proper coverage.
148+
- name: "ubuntu-py39-unittest-asynctest"
149+
python: "3.9"
150+
os: ubuntu-latest
151+
tox_env: "py39-asynctest"
152+
use_coverage: true
153+
154+
- name: "ubuntu-py39-unittest-twisted24"
155+
python: "3.9"
156+
os: ubuntu-latest
157+
tox_env: "py39-twisted24"
158+
use_coverage: true
159+
160+
- name: "ubuntu-py39-unittest-twisted25"
161+
python: "3.9"
162+
os: ubuntu-latest
163+
tox_env: "py39-twisted25"
164+
use_coverage: true
165+
129166
- name: "ubuntu-py39-lsof-numpy-pexpect"
130167
python: "3.9"
131168
os: ubuntu-latest

changelog/13497.bugfix.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed compatibility with ``Twisted 25+``.

src/_pytest/unittest.py

Lines changed: 119 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,21 @@
66
from collections.abc import Callable
77
from collections.abc import Generator
88
from collections.abc import Iterable
9+
from collections.abc import Iterator
10+
from enum import auto
11+
from enum import Enum
912
import inspect
1013
import sys
1114
import traceback
1215
import types
13-
from typing import Any
1416
from typing import TYPE_CHECKING
1517
from typing import Union
1618

1719
import _pytest._code
1820
from _pytest.compat import is_async_function
1921
from _pytest.config import hookimpl
2022
from _pytest.fixtures import FixtureRequest
23+
from _pytest.monkeypatch import MonkeyPatch
2124
from _pytest.nodes import Collector
2225
from _pytest.nodes import Item
2326
from _pytest.outcomes import exit
@@ -228,8 +231,7 @@ def startTest(self, testcase: unittest.TestCase) -> None:
228231
pass
229232

230233
def _addexcinfo(self, rawexcinfo: _SysExcInfoType) -> None:
231-
# Unwrap potential exception info (see twisted trial support below).
232-
rawexcinfo = getattr(rawexcinfo, "_rawexcinfo", rawexcinfo)
234+
rawexcinfo = _handle_twisted_exc_info(rawexcinfo)
233235
try:
234236
excinfo = _pytest._code.ExceptionInfo[BaseException].from_exc_info(
235237
rawexcinfo # type: ignore[arg-type]
@@ -385,49 +387,130 @@ def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> None:
385387
call.excinfo = call2.excinfo
386388

387389

388-
# Twisted trial support.
389-
classImplements_has_run = False
390+
def _is_skipped(obj) -> bool:
391+
"""Return True if the given object has been marked with @unittest.skip."""
392+
return bool(getattr(obj, "__unittest_skip__", False))
393+
394+
395+
def pytest_configure() -> None:
396+
"""Register the TestCaseFunction class as an IReporter if twisted.trial is available."""
397+
if _get_twisted_version() is not TwistedVersion.NotInstalled:
398+
from twisted.trial.itrial import IReporter
399+
from zope.interface import classImplements
400+
401+
classImplements(TestCaseFunction, IReporter)
402+
403+
404+
class TwistedVersion(Enum):
405+
"""
406+
The Twisted version installed in the environment.
407+
408+
We have different workarounds in place for different versions of Twisted.
409+
"""
410+
411+
# Twisted version 24 or prior.
412+
Version24 = auto()
413+
# Twisted version 25 or later.
414+
Version25 = auto()
415+
# Twisted version is not available.
416+
NotInstalled = auto()
417+
418+
419+
def _get_twisted_version() -> TwistedVersion:
420+
# We need to check if "twisted.trial.unittest" is specifically present in sys.modules.
421+
# This is because we intend to integrate with Trial only when it's actively running
422+
# the test suite, but not needed when only other Twisted components are in use.
423+
if "twisted.trial.unittest" not in sys.modules:
424+
return TwistedVersion.NotInstalled
425+
426+
import importlib.metadata
427+
428+
import packaging.version
429+
430+
version_str = importlib.metadata.version("twisted")
431+
version = packaging.version.parse(version_str)
432+
if version.major <= 24:
433+
return TwistedVersion.Version24
434+
else:
435+
return TwistedVersion.Version25
436+
437+
438+
# Name of the attribute in `twisted.python.Failure` instances that stores
439+
# the `sys.exc_info()` tuple.
440+
# See twisted.trial support in `pytest_runtest_protocol`.
441+
TWISTED_RAW_EXCINFO_ATTR = "_twisted_raw_excinfo"
390442

391443

392444
@hookimpl(wrapper=True)
393-
def pytest_runtest_protocol(item: Item) -> Generator[None, object, object]:
394-
if isinstance(item, TestCaseFunction) and "twisted.trial.unittest" in sys.modules:
395-
ut: Any = sys.modules["twisted.python.failure"]
396-
global classImplements_has_run
397-
Failure__init__ = ut.Failure.__init__
398-
if not classImplements_has_run:
399-
from twisted.trial.itrial import IReporter
400-
from zope.interface import classImplements
401-
402-
classImplements(TestCaseFunction, IReporter)
403-
classImplements_has_run = True
404-
405-
def excstore(
445+
def pytest_runtest_protocol(item: Item) -> Iterator[None]:
446+
if _get_twisted_version() is TwistedVersion.Version24:
447+
import twisted.python.failure as ut
448+
449+
# Monkeypatch `Failure.__init__` to store the raw exception info.
450+
original__init__ = ut.Failure.__init__
451+
452+
def store_raw_exception_info(
406453
self, exc_value=None, exc_type=None, exc_tb=None, captureVars=None
407-
):
454+
): # pragma: no cover
408455
if exc_value is None:
409-
self._rawexcinfo = sys.exc_info()
456+
raw_exc_info = sys.exc_info()
410457
else:
411458
if exc_type is None:
412459
exc_type = type(exc_value)
413-
self._rawexcinfo = (exc_type, exc_value, exc_tb)
460+
if exc_tb is None:
461+
exc_tb = sys.exc_info()[2]
462+
raw_exc_info = (exc_type, exc_value, exc_tb)
463+
setattr(self, TWISTED_RAW_EXCINFO_ATTR, tuple(raw_exc_info))
414464
try:
415-
Failure__init__(
465+
original__init__(
416466
self, exc_value, exc_type, exc_tb, captureVars=captureVars
417467
)
418-
except TypeError:
419-
Failure__init__(self, exc_value, exc_type, exc_tb)
468+
except TypeError: # pragma: no cover
469+
original__init__(self, exc_value, exc_type, exc_tb)
420470

421-
ut.Failure.__init__ = excstore
422-
try:
423-
res = yield
424-
finally:
425-
ut.Failure.__init__ = Failure__init__
471+
with MonkeyPatch.context() as patcher:
472+
patcher.setattr(ut.Failure, "__init__", store_raw_exception_info)
473+
return (yield)
426474
else:
427-
res = yield
428-
return res
429-
430-
431-
def _is_skipped(obj) -> bool:
432-
"""Return True if the given object has been marked with @unittest.skip."""
433-
return bool(getattr(obj, "__unittest_skip__", False))
475+
return (yield)
476+
477+
478+
def _handle_twisted_exc_info(
479+
rawexcinfo: _SysExcInfoType | BaseException,
480+
) -> _SysExcInfoType:
481+
"""
482+
Twisted passes a custom Failure instance to `addError()` instead of using `sys.exc_info()`.
483+
Therefore, if `rawexcinfo` is a `Failure` instance, convert it into the equivalent `sys.exc_info()` tuple
484+
as expected by pytest.
485+
"""
486+
twisted_version = _get_twisted_version()
487+
if twisted_version is TwistedVersion.NotInstalled:
488+
# Unfortunately, because we cannot import `twisted.python.failure` at the top of the file
489+
# and use it in the signature, we need to use `type:ignore` here because we cannot narrow
490+
# the type properly in the `if` statement above.
491+
return rawexcinfo # type:ignore[return-value]
492+
elif twisted_version is TwistedVersion.Version24:
493+
# Twisted calls addError() passing its own classes (like `twisted.python.Failure`), which violates
494+
# the `addError()` signature, so we extract the original `sys.exc_info()` tuple which is stored
495+
# in the object.
496+
if hasattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR):
497+
saved_exc_info = getattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR)
498+
# Delete the attribute from the original object to avoid leaks.
499+
delattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR)
500+
return saved_exc_info # type:ignore[no-any-return]
501+
return rawexcinfo # type:ignore[return-value]
502+
elif twisted_version is TwistedVersion.Version25:
503+
if isinstance(rawexcinfo, BaseException):
504+
import twisted.python.failure
505+
506+
if isinstance(rawexcinfo, twisted.python.failure.Failure):
507+
tb = rawexcinfo.__traceback__
508+
if tb is None:
509+
tb = sys.exc_info()[2]
510+
return type(rawexcinfo.value), rawexcinfo.value, tb
511+
512+
return rawexcinfo # type:ignore[return-value]
513+
else:
514+
# Ideally we would use assert_never() here, but it is not available in all Python versions
515+
# we support, plus we do not require `type_extensions` currently.
516+
assert False, f"Unexpected Twisted version: {twisted_version}"

tox.ini

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ envlist =
1111
py313
1212
py314
1313
pypy3
14-
py39-{pexpect,xdist,unittestextras,numpy,pluggymain,pylib}
14+
py39-{pexpect,xdist,twisted24,twisted25,asynctest,numpy,pluggymain,pylib}
1515
doctesting
1616
doctesting-coverage
1717
plugins
@@ -36,7 +36,9 @@ description =
3636
pexpect: against `pexpect`
3737
pluggymain: against the bleeding edge `pluggy` from Git
3838
pylib: against `py` lib
39-
unittestextras: against the unit test extras
39+
twisted24: against the unit test extras with twisted prior to 24.0
40+
twisted25: against the unit test extras with twisted 25.0 or later
41+
asynctest: against the unit test extras with asynctest
4042
xdist: with pytest in parallel mode
4143
under `{basepython}`
4244
doctesting: including doctests
@@ -51,7 +53,7 @@ passenv =
5153
TERM
5254
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST
5355
setenv =
54-
_PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:}
56+
_PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_FILES:}
5557

5658
# See https://docs.python.org/3/library/io.html#io-encoding-warning
5759
# If we don't enable this, neither can any of our downstream users!
@@ -66,6 +68,12 @@ setenv =
6668

6769
doctesting: _PYTEST_TOX_POSARGS_DOCTESTING=doc/en
6870

71+
# The configurations below are related only to standard unittest support.
72+
# Run only tests from test_unittest.py.
73+
asynctest: _PYTEST_FILES=testing/test_unittest.py
74+
twisted24: _PYTEST_FILES=testing/test_unittest.py
75+
twisted25: _PYTEST_FILES=testing/test_unittest.py
76+
6977
nobyte: PYTHONDONTWRITEBYTECODE=1
7078

7179
lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof
@@ -79,8 +87,9 @@ deps =
7987
pexpect: pexpect>=4.8.0
8088
pluggymain: pluggy @ git+https://github.com/pytest-dev/pluggy.git
8189
pylib: py>=1.8.2
82-
unittestextras: twisted
83-
unittestextras: asynctest
90+
twisted24: twisted<25
91+
twisted25: twisted>=25
92+
asynctest: asynctest
8493
xdist: pytest-xdist>=2.1.0
8594
xdist: -e .
8695
{env:_PYTEST_TOX_EXTRA_DEP:}

0 commit comments

Comments
 (0)