Skip to content

Commit ff01c14

Browse files
NaluTripicianNaluTripician
authored
ClientRetryPolicy: Fixes behavior to Meta-data write operations in multimaster accounts (#3466)
* Changes behavior to LocationCache and ClientRetryPolicy for metadata write operations * Changes behavior to LocationCache and ClientRetryPolicy for metadata write operations * Suggested changes * fixed added bugs * Was not checking to see if request was actually metadata request. * suggested changes and tests * Cleaning up tests and removing global variable by adding functionality into retryContext * remove unneccary code * fixed IsMetadataWriteREquestOnMultimasterAccount * added missed case to IsMetadataWriteREquestOnMultimasterAccount * suggested changes Co-authored-by: Nalu Tripician <[email protected]>
1 parent f1f4544 commit ff01c14

File tree

4 files changed

+142
-15
lines changed

4 files changed

+142
-15
lines changed

‎Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,15 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
137137

138138
if (this.retryContext != null)
139139
{
140-
// set location-based routing directive based on request retry context
141-
request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
140+
if (this.retryContext.RouteToHub)
141+
{
142+
request.RequestContext.RouteToLocation(this.globalEndpointManager.GetHubUri());
143+
}
144+
else
145+
{
146+
// set location-based routing directive based on request retry context
147+
request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
148+
}
142149
}
143150

144151
// Resolve the endpoint for the request and pin the resolution to the resolved endpoint
@@ -185,6 +192,31 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
185192
this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
186193
this.documentServiceRequest?.ResourceAddress ?? string.Empty);
187194

195+
if (this.globalEndpointManager.IsMultimasterMetadataWriteRequest(this.documentServiceRequest))
196+
{
197+
bool forceRefresh = false;
198+
199+
if (this.retryContext != null && this.retryContext.RouteToHub)
200+
{
201+
forceRefresh = true;
202+
203+
}
204+
205+
ShouldRetryResult retryResult = await this.ShouldRetryOnEndpointFailureAsync(
206+
isReadRequest: false,
207+
markBothReadAndWriteAsUnavailable: false,
208+
forceRefresh: forceRefresh,
209+
retryOnPreferredLocations: false,
210+
overwriteEndpointDiscovery: true);
211+
212+
if (retryResult.ShouldRetry)
213+
{
214+
this.retryContext.RouteToHub = true;
215+
}
216+
217+
return retryResult;
218+
}
219+
188220
return await this.ShouldRetryOnEndpointFailureAsync(
189221
isReadRequest: false,
190222
markBothReadAndWriteAsUnavailable: false,
@@ -243,9 +275,10 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
243275
bool isReadRequest,
244276
bool markBothReadAndWriteAsUnavailable,
245277
bool forceRefresh,
246-
bool retryOnPreferredLocations)
278+
bool retryOnPreferredLocations,
279+
bool overwriteEndpointDiscovery = false)
247280
{
248-
if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
281+
if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !overwriteEndpointDiscovery))
249282
{
250283
DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}",
251284
this.failoverRetryCount,
@@ -255,7 +288,7 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
255288

256289
this.failoverRetryCount++;
257290

258-
if (this.locationEndpoint != null)
291+
if (this.locationEndpoint != null && !overwriteEndpointDiscovery)
259292
{
260293
if (isReadRequest || markBothReadAndWriteAsUnavailable)
261294
{
@@ -400,6 +433,8 @@ private sealed class RetryContext
400433
{
401434
public int RetryLocationIndex { get; set; }
402435
public bool RetryRequestOnPreferredLocations { get; set; }
436+
437+
public bool RouteToHub { get; set; }
403438
}
404439
}
405440
}

‎Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,16 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
9595

9696
public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
9797

98+
public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request)
99+
{
100+
return this.locationCache.IsMultimasterMetadataWriteRequest(request);
101+
}
102+
103+
public Uri GetHubUri()
104+
{
105+
return this.locationCache.GetHubUri();
106+
}
107+
98108
/// <summary>
99109
/// This will get the account information.
100110
/// It will try the global endpoint first.
@@ -541,7 +551,6 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
541551
}
542552
}
543553
}
544-
545554
internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh = false)
546555
{
547556
#nullable disable // Needed because AsyncCache does not have nullable enabled

‎Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ namespace Microsoft.Azure.Cosmos.Routing
88
using System.Collections.Concurrent;
99
using System.Collections.Generic;
1010
using System.Collections.ObjectModel;
11+
using System.Globalization;
1112
using System.Linq;
1213
using System.Net;
14+
using global::Azure.Core;
1315
using Microsoft.Azure.Cosmos.Core.Trace;
1416
using Microsoft.Azure.Documents;
1517

@@ -206,6 +208,28 @@ public void OnLocationPreferenceChanged(ReadOnlyCollection<string> preferredLoca
206208
preferenceList: preferredLocations);
207209
}
208210

211+
public bool IsMetaData(DocumentServiceRequest request)
212+
{
213+
return (request.OperationType != Documents.OperationType.ExecuteJavaScript && request.ResourceType == ResourceType.StoredProcedure) ||
214+
request.ResourceType != ResourceType.Document;
215+
216+
}
217+
public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request)
218+
{
219+
return !request.IsReadOnlyRequest && this.locationInfo.AvailableWriteLocations.Count > 1
220+
&& this.IsMetaData(request)
221+
&& this.CanUseMultipleWriteLocations();
222+
223+
}
224+
225+
public Uri GetHubUri()
226+
{
227+
DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
228+
string writeLocation = currentLocationInfo.AvailableWriteLocations[0];
229+
Uri locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
230+
return locationEndpointToRoute;
231+
}
232+
209233
/// <summary>
210234
/// Resolves request to service endpoint.
211235
/// 1. If this is a write request

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ClientRetryPolicyTests.cs‎

Lines changed: 68 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,18 @@
11
namespace Microsoft.Azure.Cosmos.Client.Tests
22
{
33
using System;
4-
using Microsoft.VisualStudio.TestTools.UnitTesting;
54
using Microsoft.Azure.Cosmos.Routing;
6-
using Moq;
75
using Microsoft.Azure.Documents;
6+
using Microsoft.VisualStudio.TestTools.UnitTesting;
7+
using Moq;
88
using System.Collections.Generic;
99
using System.Collections.ObjectModel;
1010
using System.Globalization;
1111
using System.Linq;
1212
using System.Net;
13-
using System.Net.Http;
1413
using System.Threading;
1514
using System.Threading.Tasks;
16-
using Microsoft.Azure.Cosmos.Core.Trace;
1715
using Microsoft.Azure.Documents.Collections;
18-
using Microsoft.Azure.Documents.Routing;
19-
using System.Net.WebSockets;
20-
using System.Net.Http.Headers;
21-
using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities;
22-
using System.Collections.Specialized;
2316
using Microsoft.Azure.Documents.Client;
2417
using Microsoft.Azure.Cosmos.Common;
2518

@@ -37,6 +30,60 @@ public sealed class ClientRetryPolicyTests
3730
private GlobalPartitionEndpointManager partitionKeyRangeLocationCache;
3831
private Mock<IDocumentClientInternal> mockedClient;
3932

33+
/// <summary>
34+
/// Tests behavior of Multimaster Accounts on metadata writes where the default location is not the hub region
35+
/// </summary>
36+
[TestMethod]
37+
public void MultimasterMetadataWriteRetryTest()
38+
{
39+
const bool enableEndpointDiscovery = false;
40+
41+
//Creates GlobalEndpointManager where enableEndpointDiscovery is False and
42+
//Default location is false
43+
using GlobalEndpointManager endpointManager = this.Initialize(
44+
useMultipleWriteLocations: true,
45+
enableEndpointDiscovery: enableEndpointDiscovery,
46+
isPreferredLocationsListEmpty: true,
47+
multimasterMetadataWriteRetryTest: true);
48+
49+
50+
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(endpointManager, this.partitionKeyRangeLocationCache, enableEndpointDiscovery, new RetryOptions());
51+
52+
//Creates a metadata write request
53+
DocumentServiceRequest request = this.CreateRequest(false, true);
54+
55+
Assert.IsTrue(endpointManager.IsMultimasterMetadataWriteRequest(request));
56+
57+
//On first attempt should get incorrect (default/non hub) location
58+
retryPolicy.OnBeforeSendRequest(request);
59+
Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, ClientRetryPolicyTests.Location2Endpoint);
60+
61+
//Creation of 403.3 Error
62+
HttpStatusCode forbidden = HttpStatusCode.Forbidden;
63+
SubStatusCodes writeForbidden = SubStatusCodes.WriteForbidden;
64+
Exception forbiddenWriteFail = new Exception();
65+
Mock<INameValueCollection> nameValueCollection = new Mock<INameValueCollection>();
66+
67+
DocumentClientException documentClientException = new DocumentClientException(
68+
message: "Multimaster Metadata Write Fail",
69+
innerException: forbiddenWriteFail,
70+
statusCode: forbidden,
71+
substatusCode: writeForbidden,
72+
requestUri: request.RequestContext.LocationEndpointToRoute,
73+
responseHeaders: nameValueCollection.Object);
74+
75+
CancellationToken cancellationToken = new CancellationToken();
76+
77+
//Tests behavior of should retry
78+
Task<ShouldRetryResult> shouldRetry = retryPolicy.ShouldRetryAsync(documentClientException, cancellationToken);
79+
80+
Assert.IsTrue(shouldRetry.Result.ShouldRetry);
81+
82+
//Now since the retry context is not null, should route to the hub region
83+
retryPolicy.OnBeforeSendRequest(request);
84+
Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, ClientRetryPolicyTests.Location1Endpoint);
85+
}
86+
4087
/// <summary>
4188
/// Tests to see if different 503 substatus codes are handeled correctly
4289
/// </summary>
@@ -338,6 +385,18 @@ private GlobalEndpointManager Initialize(
338385
return endpointManager;
339386
}
340387

388+
private DocumentServiceRequest CreateRequest(bool isReadRequest, bool isMasterResourceType)
389+
{
390+
if (isReadRequest)
391+
{
392+
return DocumentServiceRequest.Create(OperationType.Read, isMasterResourceType ? ResourceType.Database : ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
393+
}
394+
else
395+
{
396+
return DocumentServiceRequest.Create(OperationType.Create, isMasterResourceType ? ResourceType.Database : ResourceType.Document, AuthorizationTokenType.PrimaryMasterKey);
397+
}
398+
}
399+
341400
private MockDocumentClientContext InitializeMockedDocumentClient(
342401
bool useMultipleWriteLocations,
343402
bool isPreferredLocationsListEmpty)

0 commit comments

Comments
 (0)