documents standard http session settings, sets shorter timeouts in telemetry (#3074)

Author: rudolfix

dlt/common/runtime/anon_tracker.py

@@ -37,9 +37,10 @@ def init_anon_tracker(config: RuntimeConfiguration) -> None:
 
     # lazily import requests to avoid binding config before initialization
     global requests
-    from dlt.sources.helpers import requests as r_
+    from dlt.sources.helpers.requests import Client
 
-    requests = r_  # type: ignore[assignment]
+    # fail fast, don't block user
+    requests = Client(request_timeout=_REQUEST_TIMEOUT, request_max_attempts=0)  # type: ignore[assignment]
 
     global _WRITE_KEY, _ANON_TRACKER_ENDPOINT, _THREAD_POOL
     # start the pool

dlt/pipeline/platform.py

@@ -29,9 +29,10 @@ class TPipelineSyncPayload(TypedDict):
 def init_platform_tracker() -> None:
     # lazily import requests to avoid binding config before initialization
     global requests
-    from dlt.sources.helpers import requests as r_
+    from dlt.sources.helpers.requests import Client
 
-    requests = r_  # type: ignore[assignment]
+    # fail fast, don't block user
+    requests = Client(request_timeout=(2, 10), request_max_attempts=0)  # type: ignore[assignment]
 
     global _THREAD_POOL
     if _THREAD_POOL is None:

docs/website/docs/dlt-ecosystem/verified-sources/rest_api/advanced.md

@@ -241,3 +241,22 @@ source_config = {
 
 In this example, the resource will set the correct encoding for all responses. More callables can be added to the list of response_actions.
 
+
+### Setup timeouts and retry strategies
+`rest_api` uses `dlt` [custom sessions](../../../general-usage/http/requests.md) and [`RESTClient`](../../../general-usage/http/rest-client.md) to access http(s) endpoints. You can use them to configure timeout, retries and other aspects. For example:
+```py
+from dlt.sources.helpers import requests
+
+source_config: RESTAPIConfig = {
+    "client": {
+        "session": requests.Client(request_timeout=(1.0, 1.0), request_max_attempts=0)
+    },
+}
+```
+will set-up all endpoints to use a short connect and read timeouts with no retries.
+Most settings [can be configured](../../../general-usage/http/requests.md#customizing-retry-settings) using `toml` files or environment variables.
+
+:::note
+By default, we set connection timeout and read timeout to 60 seconds, with
+5 retry attempts without backoff.
+:::

docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md

@@ -240,6 +240,7 @@ The `client` configuration is used to connect to the API's endpoints. It include
 - `auth` (optional): Authentication configuration. This can be a simple token, an `AuthConfigBase` object, or a more complex authentication method.
 - `session` (requests.Session, optional): A custom session object. When provided, this session will be used for all HTTP requests instead of the default session. Can be used, for example, with [requests-oauthlib](https://github.com/requests/requests-oauthlib) for OAuth authentication.
 - `paginator` (optional): Configuration for the default pagination used for resources that support pagination. Refer to the [pagination](#pagination) section for more details.
+- `session` (optional): Custom `requests` session to setup custom [timeouts and retry strategies.](advanced.md#setup-timeouts-and-retry-strategies)
 
 #### `resource_defaults` (optional)
 

docs/website/docs/general-usage/http/requests.md

@@ -63,6 +63,18 @@ request_timeout = 120  # Timeout in seconds
 request_max_retry_delay = 30  # Cap exponential delay to 30 seconds
 ```
 
+:::note
+Default session retires as follows:
+
+```toml
+[runtime]
+request_timeout=60
+request_max_attempts = 5
+request_backoff_factor = 1
+request_max_retry_delay = 300
+```
+:::
+
 For more control, you can create your own instance of `dlt.sources.requests.Client` and use that instead of the global client.
 
 This lets you customize which status codes and exceptions to retry on:
@@ -98,6 +110,12 @@ http_client = Client(
     retry_condition=retry_if_error_key
 )
 ```
+
+:::tip
+`requests.Client` is thread safe. We recommend to share sessions across threads for better performance.
+:::
+
+
 ## Handling API Rate Limits
 
 HTTP 429 errors indicate you've hit API rate limits. The dlt requests client retries these automatically and respects `Retry-After` headers. If rate limits persist, consider additional mitigation strategies.

docs/website/docs/general-usage/http/rest-client.md

@@ -620,6 +620,7 @@ Unfortunately, most OAuth 2.0 implementations vary, and thus you might need to s
 - `client_secret`: Client credential to obtain authorization. Usually issued via a developer portal.
 - `access_token_request_data`: A dictionary with data required by the authorization server apart from the `client_id`, `client_secret`, and `"grant_type": "client_credentials"`. Defaults to `None`.
 - `default_token_expiration`: The time in seconds after which the temporary access token expires. Defaults to 3600.
+- `session`: Custom `requests` session where you can configure default timeouts and retry strategies
 
 **Example:**
 
@@ -729,6 +730,30 @@ request_timeout = 120  # Timeout in seconds
 request_max_retry_delay = 30  # Cap exponential delay to 30 seconds
 ```
 
+:::note
+`RESTClient` retries by default:
+
+```toml
+[runtime]
+request_timeout=60
+request_max_attempts = 5
+request_backoff_factor = 1
+request_max_retry_delay = 300
+```
+:::
+
+### Use custom session
+You can pass custom `requests` `Session` to `RESTClient`. `dlt` [provides own implementation](requests.md#customizing-retry-settings) where you can easily configure
+retry strategies, timeouts and other factors. For example:
+```py
+from dlt.sources.helpers import requests
+client = RESTClient(
+    base_url="https://api.example.com",
+    session=requests.Client(request_timeout=(1.0, 1.0), request_max_attempts=0).session
+)
+```
+will set-up the client for a short connect and read timeouts with no retries.
+
 ### URL sanitization and secret protection
 
 The RESTClient automatically sanitizes URLs in logs and error messages to prevent exposure of sensitive information. Query parameters with the following names are automatically redacted:
@@ -820,4 +845,4 @@ for page in client.paginate(
     hooks={"response": [response_hook]}
 ):
     print(page)
-```
+```

tests/common/runtime/test_telemetry.py

@@ -137,19 +137,17 @@ def test_telemetry_endpoint_exceptions(
 def test_track_anon_event(
     mocker: MockerFixture, disable_temporary_telemetry: RuntimeConfiguration
 ) -> None:
-    from dlt.sources.helpers import requests
     from dlt.common.runtime import anon_tracker
 
     mock_github_env(os.environ)
     mock_pod_env(os.environ)
     SENT_ITEMS.clear()
     config = SentryLoggerConfiguration()
 
-    requests_post = mocker.spy(requests, "post")
-
     props = {"destination_name": "duckdb", "elapsed_time": 712.23123, "success": True}
     with patch("dlt.common.runtime.anon_tracker.before_send", _mock_before_send):
         start_test_telemetry(config)
+        requests_post = mocker.spy(anon_tracker.requests, "post")
         track("pipeline", "run", props)
         # this will send stuff
         disable_anon_tracker()