Merge pull request #36982 from hashicorp/f-cloudformation-stack-set

cloudformation/stack_set: Add retry on create

Author: YakDriver

.changelog/36982.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+resource/aws_cloudformation_stack_set: Add retry when creating to potentially help with eventual consistency problems
+```

internal/service/cloudformation/stack_set.go

@@ -266,7 +266,57 @@ func resourceStackSetCreate(ctx context.Context, d *schema.ResourceData, meta in
 		input.TemplateURL = aws.String(v.(string))
 	}
 
-	_, err := conn.CreateStackSetWithContext(ctx, input)
+	_, err := tfresource.RetryWhen(ctx, propagationTimeout,
+		func() (interface{}, error) {
+			output, err := conn.CreateStackSetWithContext(ctx, input)
+			if err != nil {
+				return nil, err
+			}
+
+			operation, err := WaitStackSetCreated(ctx, conn, name, d.Get("call_as").(string), d.Timeout(schema.TimeoutCreate))
+			if err != nil {
+				return nil, fmt.Errorf("waiting for completion (%s): %w", aws.StringValue(output.StackSetId), err)
+			}
+			return operation, nil
+		},
+		func(err error) (bool, error) {
+			if err == nil {
+				return false, nil
+			}
+
+			message := err.Error()
+
+			// IAM eventual consistency
+			if strings.Contains(message, "AccountGate check failed") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			// User: XXX is not authorized to perform: cloudformation:CreateStack on resource: YYY
+			if strings.Contains(message, "is not authorized") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			// XXX role has insufficient YYY permissions
+			if strings.Contains(message, "role has insufficient") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			// Account XXX should have YYY role with trust relationship to Role ZZZ
+			if strings.Contains(message, "role with trust relationship") {
+				return true, err
+			}
+
+			// IAM eventual consistency
+			if strings.Contains(message, "The security token included in the request is invalid") {
+				return true, err
+			}
+
+			return false, err
+		},
+	)
 
 	if err != nil {
 		return sdkdiag.AppendErrorf(diags, "creating CloudFormation StackSet (%s): %s", name, err)

internal/service/cloudformation/status.go

@@ -28,6 +28,22 @@ func StatusChangeSet(ctx context.Context, conn *cloudformation.CloudFormation, s
 	}
 }
 
+func StatusStackSet(ctx context.Context, conn *cloudformation.CloudFormation, name, callAs string) retry.StateRefreshFunc {
+	return func() (interface{}, string, error) {
+		output, err := FindStackSetByName(ctx, conn, name, callAs)
+
+		if tfresource.NotFound(err) {
+			return nil, "", nil
+		}
+
+		if err != nil {
+			return nil, "", err
+		}
+
+		return output, aws.StringValue(output.Status), nil
+	}
+}
+
 func StatusStackSetOperation(ctx context.Context, conn *cloudformation.CloudFormation, stackSetName, operationID, callAs string) retry.StateRefreshFunc {
 	return func() (interface{}, string, error) {
 		output, err := FindStackSetOperationByStackSetNameAndOperationID(ctx, conn, stackSetName, operationID, callAs)

internal/service/cloudformation/wait.go

@@ -40,6 +40,28 @@ func WaitChangeSetCreated(ctx context.Context, conn *cloudformation.CloudFormati
 	return nil, err
 }
 
+func WaitStackSetCreated(ctx context.Context, conn *cloudformation.CloudFormation, name, callAs string, timeout time.Duration) (*cloudformation.StackSet, error) {
+	stateConf := retry.StateChangeConf{
+		Pending: []string{},
+		Target:  []string{cloudformation.StackSetStatusActive},
+		Timeout: ChangeSetCreatedTimeout,
+		Refresh: StatusStackSet(ctx, conn, name, callAs),
+		Delay:   15 * time.Second,
+	}
+
+	outputRaw, err := stateConf.WaitForStateContext(ctx)
+
+	if output, ok := outputRaw.(*cloudformation.StackSet); ok {
+		if status := aws.StringValue(output.Status); status == cloudformation.ChangeSetStatusFailed {
+			tfresource.SetLastError(err, fmt.Errorf("describing CloudFormation Stack Set (%s) results: %w", name, err))
+		}
+
+		return output, err
+	}
+
+	return nil, err
+}
+
 const (
 	stackSetOperationDelay = 5 * time.Second
 )