Handle exceptions in set_tlsext_servername_callback callbacks (#1478)

When the servername callback raises an exception, call sys.excepthook
with the exception info and return SSL_TLSEXT_ERR_ALERT_FATAL to abort
the handshake. Previously, exceptions would propagate uncaught through
the CFFI callback boundary.

https://claude.ai/code/session_01P7y1XmWkdtC5UcmZwGDvGi

Co-authored-by: Claude <noreply@anthropic.com>

Author: alex

CHANGELOG.rst

@@ -20,6 +20,7 @@ Changes:
 ^^^^^^^^
 
 - Added ``OpenSSL.SSL.Connection.get_group_name`` to determine which group name was negotiated.
+- ``Context.set_tlsext_servername_callback`` now handles exceptions raised in the callback by calling ``sys.excepthook`` and returning a fatal TLS alert. Previously, exceptions were silently swallowed and the handshake would proceed as if the callback had succeeded.
 
 25.3.0 (2025-09-16)
 -------------------

src/OpenSSL/SSL.py

@@ -2,6 +2,7 @@
 
 import os
 import socket
+import sys
 import typing
 import warnings
 from collections.abc import Sequence
@@ -1752,7 +1753,11 @@ def set_tlsext_servername_callback(
 
         @wraps(callback)
         def wrapper(ssl, alert, arg):  # type: ignore[no-untyped-def]
-            callback(Connection._reverse_mapping[ssl])
+            try:
+                callback(Connection._reverse_mapping[ssl])
+            except Exception:
+                sys.excepthook(*sys.exc_info())
+                return _lib.SSL_TLSEXT_ERR_ALERT_FATAL
             return 0
 
         self._tlsext_servername_callback = _ffi.callback(

tests/test_ssl.py

@@ -2034,6 +2034,56 @@ def servername(conn: Connection) -> None:
 
         assert args == [(server, b"foo1.example.com")]
 
+    def test_servername_callback_exception(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """
+        When the callback passed to `Context.set_tlsext_servername_callback`
+        raises an exception, ``sys.excepthook`` is called with the exception
+        and the handshake fails with an ``Error``.
+        """
+        exc = TypeError("server name callback failed")
+
+        def servername(conn: Connection) -> None:
+            raise exc
+
+        excepthook_calls: list[
+            tuple[type[BaseException], BaseException, object]
+        ] = []
+
+        def custom_excepthook(
+            exc_type: type[BaseException],
+            exc_value: BaseException,
+            exc_tb: object,
+        ) -> None:
+            excepthook_calls.append((exc_type, exc_value, exc_tb))
+
+        context = Context(SSLv23_METHOD)
+        context.set_tlsext_servername_callback(servername)
+
+        # Necessary to actually accept the connection
+        context.use_privatekey(load_privatekey(FILETYPE_PEM, server_key_pem))
+        context.use_certificate(
+            load_certificate(FILETYPE_PEM, server_cert_pem)
+        )
+
+        # Do a little connection to trigger the logic
+        server = Connection(context, None)
+        server.set_accept_state()
+
+        client = Connection(Context(SSLv23_METHOD), None)
+        client.set_connect_state()
+        client.set_tlsext_host_name(b"foo1.example.com")
+
+        monkeypatch.setattr(sys, "excepthook", custom_excepthook)
+        with pytest.raises(Error):
+            interact_in_memory(server, client)
+
+        assert len(excepthook_calls) == 1
+        assert excepthook_calls[0][0] is TypeError
+        assert excepthook_calls[0][1] is exc
+        assert excepthook_calls[0][2] is not None
+
 
 class TestApplicationLayerProtoNegotiation:
     """