ClientRetryPolicy: Fixes behavior to Meta-data write operations in multimaster accounts

Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs

@@ -139,6 +139,11 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
             {
                 // set location-based routing directive based on request retry context
                 request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
+
+                if (!this.canUseMultipleWriteLocations && this.globalEndpointManager.GetTotalAvailableWriteLocations > 1 && !this.isReadRequest)
+                {
+                    request.RequestContext.IsRetry = true;
+                }
             }
 
             // Resolve the endpoint for the request and pin the resolution to the resolved endpoint
@@ -185,6 +190,16 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
                     this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
                     this.documentServiceRequest?.ResourceAddress ?? string.Empty);
 
+                if (!this.canUseMultipleWriteLocations && this.globalEndpointManager.GetTotalAvailableWriteLocations > 1 && !this.isReadRequest)
+                {
+                    return await this.ShouldRetryOnEndpointFailureAsync(
+                    isReadRequest: false,
+                    markBothReadAndWriteAsUnavailable: false,
+                    forceRefresh: true,
+                    retryOnPreferredLocations: false,
+                    isMetadataWrite: true);
+                }
+
                 return await this.ShouldRetryOnEndpointFailureAsync(
                     isReadRequest: false,
                     markBothReadAndWriteAsUnavailable: false,
@@ -237,9 +252,10 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
             bool isReadRequest,
             bool markBothReadAndWriteAsUnavailable,
             bool forceRefresh,
-            bool retryOnPreferredLocations)
+            bool retryOnPreferredLocations,
+            bool isMetadataWrite = false)
         {
-            if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
+            if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !isMetadataWrite))
             {
                 DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}", 
                     this.failoverRetryCount,

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -95,6 +95,11 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
 
         public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
 
+        /// <summary>
+        /// Gets total availible write locations from the current location information for a Cosmos account
+        /// </summary>
+        public int GetTotalAvailableWriteLocations => this.locationCache.GetTotalAvailableWriteLocations;
+
         /// <summary>
         /// This will get the account information.
         /// It will try the global endpoint first. 
@@ -541,7 +546,6 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
                 }
             }
         }
-
         internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh = false)
         {
 #nullable disable  // Needed because AsyncCache does not have nullable enabled

Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs

@@ -78,7 +78,10 @@ public LocationCache(
 #endif  
 #endif
         }
-
+        /// <summary>
+        /// Gets total availible write locations from the current location information for a Cosmos account
+        /// </summary>
+        public int GetTotalAvailableWriteLocations => this.locationInfo.AvailableWriteLocations.Count;
         /// <summary>
         /// Gets list of read endpoints ordered by
         /// 1. Preferred location
@@ -249,6 +252,12 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request)
                     string writeLocation = currentLocationInfo.AvailableWriteLocations[locationIndex];
                     locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
                 }
+                else if (request.RequestContext.IsRetry && currentLocationInfo.AvailableWriteLocations.Count > 1 && request.ResourceType != ResourceType.Document)
+                {
+                    Console.WriteLine(currentLocationInfo);
+                    string writeLocation = currentLocationInfo.AvailableWriteLocations[0]; //Always want to rout to hub region, the first one in the list of available regions
+                    locationEndpointToRoute = currentLocationInfo.AvailableWriteEndpointByLocation[writeLocation];
+                }
             }
             else
             {