feat(exporter-otlp): Add max retry limit

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpRetry.cs

@@ -46,13 +46,21 @@ internal static class OtlpRetry
     public const int InitialBackoffMilliseconds = 1000;
     private const int MaxBackoffMilliseconds = 5000;
     private const double BackoffMultiplier = 1.5;
+    private const int MaxRetryAttempts = 5;
 
 #if !NET
     private static readonly Random Random = new();
 #endif
 
-    public static bool TryGetHttpRetryResult(ExportClientHttpResponse response, int retryDelayInMilliSeconds, out RetryResult retryResult)
+    public static bool TryGetHttpRetryResult(ExportClientHttpResponse response, int retryDelayInMilliSeconds, int attempt, out RetryResult retryResult)
     {
+        // Fail fast check
+        if (attempt >= MaxRetryAttempts)
+        {
+            retryResult = default;
+            return false;
+        }
+
         if (response.StatusCode.HasValue)
         {
             return TryGetRetryResult(response.StatusCode.Value, IsHttpStatusCodeRetryable, response.DeadlineUtc, response.Headers, TryGetHttpRetryDelay, retryDelayInMilliSeconds, out retryResult);
@@ -80,10 +88,16 @@ public static bool ShouldHandleHttpRequestException(Exception? exception)
         return true;
     }
 
-    public static bool TryGetGrpcRetryResult(ExportClientGrpcResponse response, int retryDelayMilliseconds, out RetryResult retryResult)
+    public static bool TryGetGrpcRetryResult(ExportClientGrpcResponse response, int retryDelayMilliseconds, int attempt, out RetryResult retryResult)
     {
         retryResult = default;
 
+        // Fail fast check
+        if (attempt >= MaxRetryAttempts)
+        {
+            return false;
+        }
+
         if (response.Status != null)
         {
             var nextRetryDelayMilliseconds = retryDelayMilliseconds;

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/Transmission/OtlpExporterPersistentStorageTransmissionHandler.cs

@@ -51,7 +51,7 @@ internal bool InitiateAndWaitForRetryProcess(int timeOutMilliseconds)
     protected override bool OnSubmitRequestFailure(byte[] request, int contentLength, ExportClientResponse response)
     {
         Debug.Assert(request != null, "request was null");
-        return RetryHelper.ShouldRetryRequest(response, OtlpRetry.InitialBackoffMilliseconds, out _) && this.persistentBlobProvider.TryCreateBlob(request!, out _);
+        return RetryHelper.ShouldRetryRequest(response, OtlpRetry.InitialBackoffMilliseconds, 0, out _) && this.persistentBlobProvider.TryCreateBlob(request!, out _);
     }
 
     protected override void OnShutdown(int timeoutMilliseconds)
@@ -127,7 +127,7 @@ private void RetryStoredRequests()
                     if (blob.TryLease((int)this.TimeoutMilliseconds) && blob.TryRead(out var data))
                     {
                         var deadlineUtc = DateTime.UtcNow.AddMilliseconds(this.TimeoutMilliseconds);
-                        if (this.TryRetryRequest(data, data.Length, deadlineUtc, out var response) || !RetryHelper.ShouldRetryRequest(response, OtlpRetry.InitialBackoffMilliseconds, out var retryInfo))
+                        if (this.TryRetryRequest(data, data.Length, deadlineUtc, out var response) || !RetryHelper.ShouldRetryRequest(response, OtlpRetry.InitialBackoffMilliseconds, 0, out var retryInfo))
                         {
                             blob.TryDelete();
                         }

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/Transmission/OtlpExporterRetryTransmissionHandler.cs

@@ -15,7 +15,8 @@ internal OtlpExporterRetryTransmissionHandler(IExportClient exportClient, double
     protected override bool OnSubmitRequestFailure(byte[] request, int contentLength, ExportClientResponse response)
     {
         var nextRetryDelayMilliseconds = OtlpRetry.InitialBackoffMilliseconds;
-        while (RetryHelper.ShouldRetryRequest(response, nextRetryDelayMilliseconds, out var retryResult))
+        int attempt = 0;
+        while (RetryHelper.ShouldRetryRequest(response, nextRetryDelayMilliseconds, attempt, out var retryResult))
         {
             // Note: This delay cannot exceed the configured timeout period for otlp exporter.
             // If the backend responds with `RetryAfter` duration that would result in exceeding the configured timeout period
@@ -28,6 +29,7 @@ protected override bool OnSubmitRequestFailure(byte[] request, int contentLength
             }
 
             nextRetryDelayMilliseconds = retryResult.NextRetryDelayMilliseconds;
+            attempt++;
         }
 
         return false;

src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/Transmission/RetryHelper.cs

@@ -7,18 +7,18 @@ namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmissi
 
 internal static class RetryHelper
 {
-    internal static bool ShouldRetryRequest(ExportClientResponse response, int retryDelayMilliseconds, out OtlpRetry.RetryResult retryResult)
+    internal static bool ShouldRetryRequest(ExportClientResponse response, int retryDelayMilliseconds, int attempt, out OtlpRetry.RetryResult retryResult)
     {
         if (response is ExportClientGrpcResponse grpcResponse)
         {
-            if (OtlpRetry.TryGetGrpcRetryResult(grpcResponse, retryDelayMilliseconds, out retryResult))
+            if (OtlpRetry.TryGetGrpcRetryResult(grpcResponse, retryDelayMilliseconds, attempt, out retryResult))
             {
                 return true;
             }
         }
         else if (response is ExportClientHttpResponse httpResponse)
         {
-            if (OtlpRetry.TryGetHttpRetryResult(httpResponse, retryDelayMilliseconds, out retryResult))
+            if (OtlpRetry.TryGetHttpRetryResult(httpResponse, retryDelayMilliseconds, attempt, out retryResult))
             {
                 return true;
             }

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/GrpcRetryTestCase.cs

@@ -76,13 +76,26 @@ public static TheoryData<GrpcRetryTestCase> GetGrpcTestCases()
                     new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 2250),
                     new(StatusCode.Unavailable, throttleDelay: GetThrottleDelayString(Duration.FromTimeSpan(TimeSpan.FromMilliseconds(500))), expectedNextRetryDelayMilliseconds: 750),
                     new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 1125),
-                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 1688),
-                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 2532),
-                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 3798),
+                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 1688)
+                ],
+                expectedRetryAttempts: 5),
+
+            // Verify max retry limit (5 attempts pass, 6th fails)
+            new(
+                "Max retries exceeded (fail on 6th attempt)",
+                [
+
+                    // Attempts 1-5 should pass with correct backoff values
+                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 1500),
+                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 2250),
+                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 3375),
                     new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 5000),
-                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 5000)
+                    new(StatusCode.Unavailable, expectedNextRetryDelayMilliseconds: 5000),
+
+                    // Attempt 6 should fail, next delay is irrelevant
+                    new(StatusCode.Unavailable, expectedSuccess: false)
                 ],
-                expectedRetryAttempts: 9),
+                expectedRetryAttempts: 6),
         ];
     }
 

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/HttpRetryTestCase.cs

@@ -61,6 +61,23 @@ public static TheoryData<HttpRetryTestCase> GetHttpTestCases()
                 [
                     new(statusCode: HttpStatusCode.ServiceUnavailable, isDeadlineExceeded: true, expectedSuccess: false)
                 ]),
+
+            // Verify max retry limit (5 attempts pass, 6th fails)
+            new(
+                "Max retries exceeded (fail on 6th attempt)",
+                [
+
+                    // Attempts 1-5 should pass with correct backoff values
+                    new(statusCode: HttpStatusCode.ServiceUnavailable, expectedNextRetryDelayMilliseconds: 1500),
+                    new(statusCode: HttpStatusCode.ServiceUnavailable, expectedNextRetryDelayMilliseconds: 2250),
+                    new(statusCode: HttpStatusCode.ServiceUnavailable, expectedNextRetryDelayMilliseconds: 3375),
+                    new(statusCode: HttpStatusCode.ServiceUnavailable, expectedNextRetryDelayMilliseconds: 5000),
+                    new(statusCode: HttpStatusCode.ServiceUnavailable, expectedNextRetryDelayMilliseconds: 5000),
+
+                    // Attempt 6 should fail, next delay is irrelevant
+                    new(statusCode: HttpStatusCode.ServiceUnavailable, expectedSuccess: false)
+                ],
+                expectedRetryAttempts: 6),
         ];
 
         // TODO: Add more cases.

test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpRetryTests.cs

@@ -33,7 +33,7 @@ public void TryGetGrpcRetryResultTest(GrpcRetryTestCase testCase)
             var statusCode = retryAttempt.Response.Status.Value.StatusCode;
             var deadline = retryAttempt.Response.DeadlineUtc;
             var trailers = retryAttempt.Response.GrpcStatusDetailsHeader;
-            var success = OtlpRetry.TryGetGrpcRetryResult(retryAttempt.Response, nextRetryDelayMilliseconds, out var retryResult);
+            var success = OtlpRetry.TryGetGrpcRetryResult(retryAttempt.Response, nextRetryDelayMilliseconds, attempts - 1, out var retryResult);
 
             Assert.Equal(retryAttempt.ExpectedSuccess, success);
 
@@ -82,7 +82,7 @@ public void TryGetHttpRetryResultTest(HttpRetryTestCase testCase)
             var statusCode = retryAttempt.Response.StatusCode;
             var deadline = retryAttempt.Response.DeadlineUtc;
             var headers = retryAttempt.Response.Headers;
-            var success = OtlpRetry.TryGetHttpRetryResult(retryAttempt.Response, nextRetryDelayMilliseconds, out var retryResult);
+            var success = OtlpRetry.TryGetHttpRetryResult(retryAttempt.Response, nextRetryDelayMilliseconds, attempts - 1, out var retryResult);
 
             Assert.Equal(retryAttempt.ExpectedSuccess, success);