ClientRetryPolicy: Fixes behavior to Meta-data write operations in multimaster accounts

Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs

@@ -34,6 +34,7 @@ internal sealed class ClientRetryPolicy : IDocumentClientRetryPolicy
         private int serviceUnavailableRetryCount;
         private bool isReadRequest;
         private bool canUseMultipleWriteLocations;
+        private bool retryMetadataMultiMasterWrite403dot3;
         private Uri locationEndpoint;
         private RetryContext retryContext;
         private DocumentServiceRequest documentServiceRequest;
@@ -139,6 +140,11 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
             {
                 // set location-based routing directive based on request retry context
                 request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
+
+                if (this.retryMetadataMultiMasterWrite403dot3)
+                {
+                    request.RequestContext.RouteToLocation(this.globalEndpointManager.GetHubUri());
+                }
             }
 
             // Resolve the endpoint for the request and pin the resolution to the resolved endpoint
@@ -185,6 +191,20 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
                     this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
                     this.documentServiceRequest?.ResourceAddress ?? string.Empty);
 
+                if (this.globalEndpointManager.IsMetadataWriteRequestMultimaster(this.documentServiceRequest))
+                {
+                    Task<ShouldRetryResult> retryResult = this.ShouldRetryOnEndpointFailureAsync(
+                        isReadRequest: false,
+                        markBothReadAndWriteAsUnavailable: false,
+                        forceRefresh: true,
+                        retryOnPreferredLocations: false,
+                        overwriteEndpointDiscovery: true);
+
+                    this.retryMetadataMultiMasterWrite403dot3 = true;
+
+                    return await retryResult;
+                }
+
                 return await this.ShouldRetryOnEndpointFailureAsync(
                     isReadRequest: false,
                     markBothReadAndWriteAsUnavailable: false,
@@ -237,9 +257,10 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
             bool isReadRequest,
             bool markBothReadAndWriteAsUnavailable,
             bool forceRefresh,
-            bool retryOnPreferredLocations)
+            bool retryOnPreferredLocations,
+            bool overwriteEndpointDiscovery = false)
         {
-            if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
+            if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !overwriteEndpointDiscovery))
             {
                 DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}", 
                     this.failoverRetryCount,
@@ -249,7 +270,7 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
 
             this.failoverRetryCount++;
 
-            if (this.locationEndpoint != null)
+            if (this.locationEndpoint != null && !overwriteEndpointDiscovery)
             {
                 if (isReadRequest || markBothReadAndWriteAsUnavailable)
                 {

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -95,6 +95,16 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
 
         public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
 
+        public bool IsMetadataWriteRequestMultimaster(DocumentServiceRequest request)
+        {
+            return this.locationCache.IsMetadataWriteRequestOnMultimasterAccount(request);
+        }
+
+        public Uri GetHubUri()
+        {
+            return this.locationCache.GetHubUri();
+        }
+
         /// <summary>
         /// This will get the account information.
         /// It will try the global endpoint first. 
@@ -541,7 +551,6 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
                 }
             }
         }
-
         internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh = false)
         {
 #nullable disable  // Needed because AsyncCache does not have nullable enabled

Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs

@@ -206,6 +206,20 @@ public void OnLocationPreferenceChanged(ReadOnlyCollection<string> preferredLoca
                 preferenceList: preferredLocations);
         }
 
+        public bool IsMetadataWriteRequestOnMultimasterAccount(DocumentServiceRequest request)
+        {
+            return !request.IsReadOnlyRequest && this.locationInfo.AvailableWriteLocations.Count > 1 
+                && !this.CanUseMultipleWriteLocations(request) && request.ResourceType != ResourceType.Document;
+        }
+
+        public Uri GetHubUri()
+        {
+            DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
+            string writeLocation = currentLocationInfo.AvailableWriteLocations[0];
+            Uri locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
+            return locationEndpointToRoute;
+        }
+
         /// <summary>
         /// Resolves request to service endpoint. 
         /// 1. If this is a write request