Backend/azure: subscription_id infer from Azure CLI & a way to skip *unnecessary* management plane API call (#36623)

* backend/azure: Infer `subscription_id` from Azure CLI if not specified

* Avoid management plane call if not necessary

There are four auth scenarios of the blob/container:
1. User specifies the SAS token
2. User specifies the shared access key
3. User specified to use AAD auth (and credential provided)
4. None of the above, management plane API call needed to list the shared access key and use it to auth

For 1, 2 and 3, the management plane API can be skipped in most of the cases, except the target storage account is using private DNS zone. The blob/container data plane client requires the user to specify the base URI, which is not deterministic if the storage account opt in the private DNS zone. In this case, an additional management plane `GET` is required against the storage account, to retrieve the blob endpoint. While if private DNS zone is not used, the base URI can be composed in a fixed pattern, with the storage account name and container&blob name. Hence no management plane API call is needed.

The user is expected to use the `subscription_id` and `resource_group_name` to indicate the above intent: Only if both are specified, the additional `GET` call will be invoked to get the accurate blob endpoint.

(NOTE: the `subscription_id` can be inferred from the Azure CLI if unspecified)

* No need for AAD auth if shared key or sas is specified

* update

* Add test

* update comments

* changelog

Author: magodo

.changes/v1.11/BUG FIXES-20250311-104640.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: 'Backend/azure: `subscription_id` be optional & skip *unnecessary* management plane API call in some setup'
+time: 2025-03-11T10:46:40.000000+11:00
+custom:
+    Issue: "36595"

internal/backend/remote-state/azure/api_client.go

@@ -23,21 +23,21 @@ import (
 )
 
 type Client struct {
-	// These Clients are only initialized if an Access Key isn't provided
+	environment        environments.Environment
+	storageAccountName string
+
+	// Storage ARM client is used for looking up the blob endpoint, or/and listing access key (if not specified).
 	storageAccountsClient *storageaccounts.StorageAccountsClient
+	// This is only non-nil if the config has specified to lookup the blob endpoint
+	accountDetail *AccountDetails
 
 	// Caching
 	containersClient *containers.Client
 	blobsClient      *blobs.Client
 
-	environment        environments.Environment
-	storageAccountName string
-
-	accountDetail *AccountDetails
-
-	accessKey string
-	sasToken  string
-	// azureAdStorageAuth is only here if we're using AzureAD Authentication but is an Authorizer for Storage
+	// Only one of them shall be specified
+	accessKey          string
+	sasToken           string
 	azureAdStorageAuth auth.Authorizer
 }
 
@@ -47,54 +47,71 @@ func buildClient(ctx context.Context, config BackendConfig) (*Client, error) {
 		storageAccountName: config.StorageAccountName,
 	}
 
-	// if we have an Access Key - we don't need the other clients
-	if config.AccessKey != "" {
+	var armAuthRequired bool
+	switch {
+	case config.AccessKey != "":
 		client.accessKey = config.AccessKey
-		return &client, nil
-	}
-
-	// likewise with a SAS token
-	if config.SasToken != "" {
+	case config.SasToken != "":
 		sasToken := config.SasToken
 		if strings.TrimSpace(sasToken) == "" {
 			return nil, fmt.Errorf("sasToken cannot be empty")
 		}
 		client.sasToken = strings.TrimPrefix(sasToken, "?")
-
-		return &client, nil
-	}
-
-	if config.UseAzureADAuthentication {
+	case config.UseAzureADAuthentication:
 		var err error
 		client.azureAdStorageAuth, err = auth.NewAuthorizerFromCredentials(ctx, *config.AuthConfig, config.AuthConfig.Environment.Storage)
 		if err != nil {
 			return nil, fmt.Errorf("unable to build authorizer for Storage API: %+v", err)
 		}
+	default:
+		// AAD authentication (ARM scope) is required only when no auth method is specified, which falls back to listing the access key via ARM API.
+		armAuthRequired = true
 	}
 
-	resourceManagerAuth, err := auth.NewAuthorizerFromCredentials(ctx, *config.AuthConfig, config.AuthConfig.Environment.ResourceManager)
-	if err != nil {
-		return nil, fmt.Errorf("unable to build authorizer for Resource Manager API: %+v", err)
+	// If `config.LookupBlobEndpoint` is true, we need to authenticate with ARM to lookup the blob endpoint
+	if config.LookupBlobEndpoint {
+		armAuthRequired = true
 	}
 
-	client.storageAccountsClient, err = storageaccounts.NewStorageAccountsClientWithBaseURI(config.AuthConfig.Environment.ResourceManager)
-	if err != nil {
-		return nil, fmt.Errorf("building Storage Accounts client: %+v", err)
-	}
-	client.configureClient(client.storageAccountsClient.Client, resourceManagerAuth)
+	if armAuthRequired {
+		resourceManagerAuth, err := auth.NewAuthorizerFromCredentials(ctx, *config.AuthConfig, config.AuthConfig.Environment.ResourceManager)
+		if err != nil {
+			return nil, fmt.Errorf("unable to build authorizer for Resource Manager API: %+v", err)
+		}
 
-	// Populating the storage account detail
-	storageAccountId := commonids.NewStorageAccountID(config.SubscriptionID, config.ResourceGroupName, client.storageAccountName)
-	resp, err := client.storageAccountsClient.GetProperties(ctx, storageAccountId, storageaccounts.DefaultGetPropertiesOperationOptions())
-	if err != nil {
-		return nil, fmt.Errorf("retrieving %s: %+v", storageAccountId, err)
-	}
-	if resp.Model == nil {
-		return nil, fmt.Errorf("retrieving %s: model was nil", storageAccountId)
-	}
-	client.accountDetail, err = populateAccountDetails(storageAccountId, *resp.Model)
-	if err != nil {
-		return nil, fmt.Errorf("populating details for %s: %+v", storageAccountId, err)
+		// When using Azure CLI to auth, the user can leave the "subscription_id" unspecified. In this case the subscription id is inferred from
+		// the Azure CLI default subscription.
+		if config.SubscriptionID == "" {
+			if cachedAuth, ok := resourceManagerAuth.(*auth.CachedAuthorizer); ok {
+				if cliAuth, ok := cachedAuth.Source.(*auth.AzureCliAuthorizer); ok && cliAuth.DefaultSubscriptionID != "" {
+					config.SubscriptionID = cliAuth.DefaultSubscriptionID
+				}
+			}
+		}
+		if config.SubscriptionID == "" {
+			return nil, fmt.Errorf("subscription id not specified")
+		}
+
+		// Setup the SA client.
+		client.storageAccountsClient, err = storageaccounts.NewStorageAccountsClientWithBaseURI(config.AuthConfig.Environment.ResourceManager)
+		if err != nil {
+			return nil, fmt.Errorf("building Storage Accounts client: %+v", err)
+		}
+		client.configureClient(client.storageAccountsClient.Client, resourceManagerAuth)
+
+		// Populating the storage account detail
+		storageAccountId := commonids.NewStorageAccountID(config.SubscriptionID, config.ResourceGroupName, client.storageAccountName)
+		resp, err := client.storageAccountsClient.GetProperties(ctx, storageAccountId, storageaccounts.DefaultGetPropertiesOperationOptions())
+		if err != nil {
+			return nil, fmt.Errorf("retrieving %s: %+v", storageAccountId, err)
+		}
+		if resp.Model == nil {
+			return nil, fmt.Errorf("retrieving %s: model was nil", storageAccountId)
+		}
+		client.accountDetail, err = populateAccountDetails(storageAccountId, *resp.Model)
+		if err != nil {
+			return nil, fmt.Errorf("populating details for %s: %+v", storageAccountId, err)
+		}
 	}
 
 	return &client, nil
@@ -111,16 +128,29 @@ func (c *Client) getBlobClient(ctx context.Context) (bc *blobs.Client, err error
 		}
 	}()
 
-	if c.sasToken != "" {
-		log.Printf("[DEBUG] Building the Blob Client from a SAS Token")
-		baseURL, err := naiveStorageAccountBlobBaseURL(c.environment, c.storageAccountName)
+	var baseUri string
+	if c.accountDetail != nil {
+		// Use the actual blob endpoint if available
+		pBaseUri, err := c.accountDetail.DataPlaneEndpoint(EndpointTypeBlob)
 		if err != nil {
-			return nil, fmt.Errorf("build storage account blob base URL: %v", err)
+			return nil, err
 		}
-		blobsClient, err := blobs.NewWithBaseUri(baseURL)
+		baseUri = *pBaseUri
+	} else {
+		baseUri, err = naiveStorageAccountBlobBaseURL(c.environment, c.storageAccountName)
 		if err != nil {
-			return nil, fmt.Errorf("new blob client: %v", err)
+			return nil, err
 		}
+	}
+
+	blobsClient, err := blobs.NewWithBaseUri(baseUri)
+	if err != nil {
+		return nil, fmt.Errorf("new blob client: %v", err)
+	}
+
+	switch {
+	case c.sasToken != "":
+		log.Printf("[DEBUG] Building the Blob Client from a SAS Token")
 		c.configureClient(blobsClient.Client, nil)
 		blobsClient.Client.AppendRequestMiddleware(func(r *http.Request) (*http.Request, error) {
 			if r.URL.RawQuery == "" {
@@ -131,59 +161,35 @@ func (c *Client) getBlobClient(ctx context.Context) (bc *blobs.Client, err error
 			return r, nil
 		})
 		return blobsClient, nil
-	}
 
-	if c.accessKey != "" {
+	case c.accessKey != "":
 		log.Printf("[DEBUG] Building the Blob Client from an Access Key")
-		baseURL, err := naiveStorageAccountBlobBaseURL(c.environment, c.storageAccountName)
-		if err != nil {
-			return nil, fmt.Errorf("build storage account blob base URL: %v", err)
-		}
-		blobsClient, err := blobs.NewWithBaseUri(baseURL)
-		if err != nil {
-			return nil, fmt.Errorf("new blob client: %v", err)
-		}
-		c.configureClient(blobsClient.Client, nil)
-
 		authorizer, err := auth.NewSharedKeyAuthorizer(c.storageAccountName, c.accessKey, auth.SharedKey)
 		if err != nil {
 			return nil, fmt.Errorf("new shared key authorizer: %v", err)
 		}
 		c.configureClient(blobsClient.Client, authorizer)
-
 		return blobsClient, nil
-	}
 
-	// Neither shared access key nor sas token specified, then we have the storage account details populated.
-	// This detail can be used to get the "most" correct blob endpoint comparing to the naive construction.
-	baseUri, err := c.accountDetail.DataPlaneEndpoint(EndpointTypeBlob)
-	if err != nil {
-		return nil, err
-	}
-	blobsClient, err := blobs.NewWithBaseUri(*baseUri)
-	if err != nil {
-		return nil, fmt.Errorf("new blob client: %v", err)
-	}
-
-	if c.azureAdStorageAuth != nil {
+	case c.azureAdStorageAuth != nil:
 		log.Printf("[DEBUG] Building the Blob Client from AAD auth")
 		c.configureClient(blobsClient.Client, c.azureAdStorageAuth)
 		return blobsClient, nil
-	}
-
-	log.Printf("[DEBUG] Building the Blob Client from an Access Token (using user credentials)")
-	key, err := c.accountDetail.AccountKey(ctx, c.storageAccountsClient)
-	if err != nil {
-		return nil, fmt.Errorf("retrieving key for Storage Account %q: %s", c.storageAccountName, err)
-	}
 
-	authorizer, err := auth.NewSharedKeyAuthorizer(c.storageAccountName, *key, auth.SharedKey)
-	if err != nil {
-		return nil, fmt.Errorf("new shared key authorizer: %v", err)
+	default:
+		// Neither shared access key, sas token, or AAD Auth were specified so we have to call the management plane API to get the key.
+		log.Printf("[DEBUG] Building the Blob Client from an Access Key (key is listed using client credentials)")
+		key, err := c.accountDetail.AccountKey(ctx, c.storageAccountsClient)
+		if err != nil {
+			return nil, fmt.Errorf("retrieving key for Storage Account %q: %s", c.storageAccountName, err)
+		}
+		authorizer, err := auth.NewSharedKeyAuthorizer(c.storageAccountName, *key, auth.SharedKey)
+		if err != nil {
+			return nil, fmt.Errorf("new shared key authorizer: %v", err)
+		}
+		c.configureClient(blobsClient.Client, authorizer)
+		return blobsClient, nil
 	}
-	c.configureClient(blobsClient.Client, authorizer)
-
-	return blobsClient, nil
 }
 
 func (c *Client) getContainersClient(ctx context.Context) (cc *containers.Client, err error) {
@@ -197,16 +203,29 @@ func (c *Client) getContainersClient(ctx context.Context) (cc *containers.Client
 		}
 	}()
 
-	if c.sasToken != "" {
-		log.Printf("[DEBUG] Building the Container Client from a SAS Token")
-		baseURL, err := naiveStorageAccountBlobBaseURL(c.environment, c.storageAccountName)
+	var baseUri string
+	if c.accountDetail != nil {
+		// Use the actual blob endpoint if available
+		pBaseUri, err := c.accountDetail.DataPlaneEndpoint(EndpointTypeBlob)
 		if err != nil {
-			return nil, fmt.Errorf("build storage account blob base URL: %v", err)
+			return nil, err
 		}
-		containersClient, err := containers.NewWithBaseUri(baseURL)
+		baseUri = *pBaseUri
+	} else {
+		baseUri, err = naiveStorageAccountBlobBaseURL(c.environment, c.storageAccountName)
 		if err != nil {
-			return nil, fmt.Errorf("new container client: %v", err)
+			return nil, err
 		}
+	}
+
+	containersClient, err := containers.NewWithBaseUri(baseUri)
+	if err != nil {
+		return nil, fmt.Errorf("new container client: %v", err)
+	}
+
+	switch {
+	case c.sasToken != "":
+		log.Printf("[DEBUG] Building the Container Client from a SAS Token")
 		c.configureClient(containersClient.Client, nil)
 		containersClient.Client.AppendRequestMiddleware(func(r *http.Request) (*http.Request, error) {
 			if r.URL.RawQuery == "" {
@@ -217,59 +236,35 @@ func (c *Client) getContainersClient(ctx context.Context) (cc *containers.Client
 			return r, nil
 		})
 		return containersClient, nil
-	}
 
-	if c.accessKey != "" {
+	case c.accessKey != "":
 		log.Printf("[DEBUG] Building the Container Client from an Access Key")
-		baseURL, err := naiveStorageAccountBlobBaseURL(c.environment, c.storageAccountName)
-		if err != nil {
-			return nil, fmt.Errorf("build storage account blob base URL: %v", err)
-		}
-		containersClient, err := containers.NewWithBaseUri(baseURL)
-		if err != nil {
-			return nil, fmt.Errorf("new container client: %v", err)
-		}
-		c.configureClient(containersClient.Client, nil)
-
 		authorizer, err := auth.NewSharedKeyAuthorizer(c.storageAccountName, c.accessKey, auth.SharedKey)
 		if err != nil {
 			return nil, fmt.Errorf("new shared key authorizer: %v", err)
 		}
 		c.configureClient(containersClient.Client, authorizer)
-
 		return containersClient, nil
-	}
-
-	// Neither shared access key nor sas token specified, then we have the storage account details populated.
-	// This detail can be used to get the "most" correct blob endpoint comparing to the naive construction.
-	baseUri, err := c.accountDetail.DataPlaneEndpoint(EndpointTypeBlob)
-	if err != nil {
-		return nil, err
-	}
-	containersClient, err := containers.NewWithBaseUri(*baseUri)
-	if err != nil {
-		return nil, fmt.Errorf("new container client: %v", err)
-	}
 
-	if c.azureAdStorageAuth != nil {
+	case c.azureAdStorageAuth != nil:
 		log.Printf("[DEBUG] Building the Container Client from AAD auth")
 		c.configureClient(containersClient.Client, c.azureAdStorageAuth)
 		return containersClient, nil
-	}
-
-	log.Printf("[DEBUG] Building the Container Client from an Access Token (using user credentials)")
-	key, err := c.accountDetail.AccountKey(ctx, c.storageAccountsClient)
-	if err != nil {
-		return nil, fmt.Errorf("retrieving key for Storage Account %q: %s", c.storageAccountName, err)
-	}
 
-	authorizer, err := auth.NewSharedKeyAuthorizer(c.storageAccountName, *key, auth.SharedKey)
-	if err != nil {
-		return nil, fmt.Errorf("new shared key authorizer: %v", err)
+	default:
+		// Neither shared access key, sas token, or AAD Auth were specified so we have to call the management plane API to get the key.
+		log.Printf("[DEBUG] Building the Container Client from an Access Key (key is listed using user credentials)")
+		key, err := c.accountDetail.AccountKey(ctx, c.storageAccountsClient)
+		if err != nil {
+			return nil, fmt.Errorf("retrieving key for Storage Account %q: %s", c.storageAccountName, err)
+		}
+		authorizer, err := auth.NewSharedKeyAuthorizer(c.storageAccountName, *key, auth.SharedKey)
+		if err != nil {
+			return nil, fmt.Errorf("new shared key authorizer: %v", err)
+		}
+		c.configureClient(containersClient.Client, authorizer)
+		return containersClient, nil
 	}
-	c.configureClient(containersClient.Client, authorizer)
-
-	return containersClient, nil
 }
 
 func (c *Client) configureClient(client client.BaseClient, authorizer auth.Authorizer) {