DEVPROD-3638 fix(tests): add HTTP retry logic for transient connectio…

tests/rptest/services/provider_clients/rpcloud_client.py

@@ -1,6 +1,8 @@
 from typing import Any, Literal, Union, overload
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 
 class RpCloudApiClient(object):
@@ -9,6 +11,26 @@ def __init__(self, config, log):
         self._token = None
         self._logger = log
         self.lasterror = None
+        self._session = self._create_session()
+
+    def _create_session(self) -> requests.Session:
+        """Create a session with retry for transient connection errors."""
+        retry = Retry(
+            total=3,
+            connect=3,
+            read=3,
+            backoff_factor=0.5,
+            status_forcelist=[
+                requests.codes.bad_gateway,
+                requests.codes.service_unavailable,
+                requests.codes.gateway_timeout,
+            ],
+            allowed_methods=["GET", "POST", "PATCH", "DELETE", "HEAD"],
+        )
+        session = requests.Session()
+        session.mount("https://", HTTPAdapter(max_retries=retry))
+        session.mount("http://", HTTPAdapter(max_retries=retry))
+        return session
 
     def _handle_error(self, response: requests.Response, quite=False):
         try:
@@ -38,7 +60,7 @@ def _get_token(self):
                 "client_secret": f"{self._config.oauth_client_secret}",
                 "audience": f"{self._config.oauth_audience}",
             }
-            resp = requests.post(
+            resp = self._session.post(
                 f"{self._config.oauth_url}", headers=headers, data=data
             )
             _r = self._handle_error(resp)
@@ -86,7 +108,7 @@ def _http_get(
             "Accept": "application/json",
         }
         _base = base_url if base_url else self._config.api_url
-        resp = requests.get(f"{_base}{endpoint}", headers=headers, **kwargs)
+        resp = self._session.get(f"{_base}{endpoint}", headers=headers, **kwargs)
         _r = self._handle_error(resp, quite=quite)
         if text_response:
             return _r.text
@@ -101,7 +123,7 @@ def _http_post(self, base_url=None, endpoint="", override_headers={}, **kwargs):
             "Accept": "application/json",
         } | override_headers
         _base = base_url if base_url else self._config.api_url
-        resp = requests.post(f"{_base}{endpoint}", headers=headers, **kwargs)
+        resp = self._session.post(f"{_base}{endpoint}", headers=headers, **kwargs)
         _r = self._handle_error(resp)
         return _r if _r is None else _r.json()
 
@@ -123,15 +145,15 @@ def _http_patch(
         } | override_headers
 
         _base = base_url if base_url else self._config.api_url
-        resp = requests.patch(f"{_base}{endpoint}", headers=headers, **kwargs)
+        resp = self._session.patch(f"{_base}{endpoint}", headers=headers, **kwargs)
         _r = self._handle_error(resp)
         return _r if _r is None else _r.json()
 
     def _http_delete(self, base_url=None, endpoint="", **kwargs):
         token = self._get_token()
         headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
         _base = base_url if base_url else self._config.api_url
-        resp = requests.delete(f"{_base}{endpoint}", headers=headers, **kwargs)
+        resp = self._session.delete(f"{_base}{endpoint}", headers=headers, **kwargs)
         _r = self._handle_error(resp)
         return _r if _r is None else _r.json()
 

tests/rptest/services/redpanda_installer.py

@@ -15,11 +15,12 @@
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timedelta, timezone
 from functools import lru_cache
-from time import sleep
 from typing import Any, Iterable, Literal, TYPE_CHECKING
 
 import concurrent.futures
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 from ducktape.utils.util import wait_until
 from ducktape.cluster.cluster import ClusterNode
 
@@ -199,6 +200,7 @@ def __init__(self, redpanda: "RedpandaService"):
         """
         self._started = False
         self._redpanda = redpanda
+        self._http_session = self._create_http_session()
 
         # Keep track if the original install path is /opt/redpanda, as is the
         # case for package-deployed clusters. Since the installer uses this
@@ -228,6 +230,21 @@ def __init__(self, redpanda: "RedpandaService"):
         self._arch_lock = threading.Lock()
         self._arch = None
 
+    def _create_http_session(self) -> requests.Session:
+        """Create a session with retry for transient errors (connection and HTTP 5xx)."""
+        retry = Retry(
+            total=3,
+            connect=3,
+            read=3,
+            status=3,
+            backoff_factor=1.0,
+            allowed_methods=["HEAD", "GET"],
+            status_forcelist=[500, 502, 503, 504],
+        )
+        session = requests.Session()
+        session.mount("https://", HTTPAdapter(max_retries=retry))
+        return session
+
     def installed_version(self, node: ClusterNode) -> RedpandaVersion:
         assert node in self._installed_versions, (
             f"Node {node} not in installed versions dictionary {self._installed_versions}"
@@ -472,18 +489,11 @@ def _avail_for_download(self, version: tuple[int, int, int]):
         validate that it is really downloadable: this avoids tests being upset by ongoing releases
         which might exist in github but not yet fave all their artifacts
         """
-        r = requests.head(self._version_package_url(version))
+        # Session with retry handles transient connection errors automatically
+        r = self._http_session.head(self._version_package_url(version))
+
         # allow 403 ClientError, it usually indicates Unauthorized get and can happen on S3 while dealing with old releases
         allowed = (200, 403, 404)
-        if r.status_code not in allowed:
-            num_retries = 3
-            while num_retries > 0:
-                sleep(5.0 ** (4 - num_retries))
-                r = requests.head(self._version_package_url(version))
-                if r.status_code in allowed:
-                    break
-                num_retries -= 1
-
         if r.status_code not in allowed:
             r.raise_for_status()