fix: wait for AWS OpsManager VM to be running (#702)

PR #696 added a wait for the AWS OpsManager VM to start running.
This builds on top of that change, updating the expected output
from the AWS CLI so that it works.

Tests were also updated. Note that the tests are quite fragile
and would benefit from re-work. Specifically the
`runner.ExecuteWithEnvVarsReturnsOnCall()` test fake is problematic
because (a) it's possible to write tests that pass but do not match
the actual implmentation of the AWS CLI, and (b) adding/removing/changing
the order of calls requires the tests to be re-written.

Author: blgm

vmlifecycle/vmmanagers/aws.go

@@ -132,8 +132,8 @@ func (a *AWSVMManager) CreateVM() (Status, StateInfo, error) {
 	}
 	latestState.ID = instanceID
 
-	if err := a.waitTillVmRunning(instanceID); err != nil {
-		return Incomplete, latestState, fmt.Errorf("aws error creating the vm: %s", err)
+	if err := a.waitUntilVMRunning(instanceID); err != nil {
+		return Incomplete, latestState, fmt.Errorf("aws error waiting for VM to enter running state: %w", err)
 	}
 
 	if iaasConfig.PublicIP != "" {
@@ -273,28 +273,26 @@ func (a *AWSVMManager) createVM(ami string) (string, error) {
 	return cleanupString(instanceID.String()), nil
 }
 
-func (a *AWSVMManager) waitTillVmRunning(instanceID string) error {
-	// describe state and wait till running
-	for i := 0; i <= 200; i++ {
-		state, _, err := a.ExecuteWithInstanceProfile(a.addEnvVars(),
-			[]interface{}{
+func (a *AWSVMManager) waitUntilVMRunning(instanceID string) error {
+	for range 200 {
+		stdout, _, err := a.ExecuteWithInstanceProfile(a.addEnvVars(),
+			[]any{
 				"ec2", "describe-instances",
 				"--instance-ids", instanceID,
-				"--query", "Reservations[*].Instances[*].State.Name",
-			})
+				"--query", "Reservations[0].Instances[0].State.Name",
+			},
+		)
 		if err != nil {
-			return fmt.Errorf("could not check the instance state for %s", instanceID)
+			return fmt.Errorf("could not check the instance state for %s: %w", instanceID, err)
 		}
-		if cleanupString(state.String()) == "running" {
+		if cleanupString(stdout.String()) == "running" {
 			return nil
 		}
-		if i == 200 {
-			return fmt.Errorf("timeout exceeded in running status")
-		}
 		time.Sleep(a.pollingInterval)
 	}
-	return nil
+	return fmt.Errorf("timeout exceeded waiting for the VM to enter the running state")
 }
+
 func (a *AWSVMManager) getIPAddressID() (ipAddress string, err error) {
 	allocationID, _, err := a.ExecuteWithInstanceProfile(a.addEnvVars(),
 		[]interface{}{

vmlifecycle/vmmanagers/aws_test.go

@@ -60,11 +60,10 @@ opsman-configuration:
 `
 
 		command, runner := createCommand(fmt.Sprintf(configStrTemplate, accessKey, secretAccessKey, assumeRole, region, publicIP, privateIP))
-		// Set default return value for all calls (for waitTillVmRunning)
-		runner.ExecuteWithEnvVarsReturns(bytes.NewBufferString("\"running\"\r\n"), nil, nil)
 		// Override specific calls with their expected return values
-		runner.ExecuteWithEnvVarsReturnsOnCall(0, bytes.NewBufferString(fmt.Sprintf("\"%s\"\r\n", vmID)), nil, nil)
-		runner.ExecuteWithEnvVarsReturnsOnCall(2, bytes.NewBufferString("\"eipalloc-18643c24\"\r\n"), nil, nil)
+		runner.ExecuteWithEnvVarsReturnsOnCall(0, bytes.NewBufferString(fmt.Sprintf("\"%s\"\r\n", vmID)), nil, nil) // ec2 run-instances
+		runner.ExecuteWithEnvVarsReturnsOnCall(1, bytes.NewBufferString("\"running\"\r\n"), nil, nil)               // ec2 describe-instances
+		runner.ExecuteWithEnvVarsReturnsOnCall(2, bytes.NewBufferString("\"eipalloc-18643c24\"\r\n"), nil, nil)     // ec2 describe-addresses
 		runner.ExecuteWithEnvVarsReturnsOnCall(5, bytes.NewBufferString("\"stopping\"\r\n"), nil, nil)
 		runner.ExecuteWithEnvVarsReturnsOnCall(6, bytes.NewBufferString("\"stopped\"\r\n"), nil, nil)
 		runner.ExecuteWithEnvVarsReturnsOnCall(7, bytes.NewBufferString("\"stopped\"\r\n"), nil, nil)
@@ -98,7 +97,7 @@ opsman-configuration:
 					{
 						`ec2`, `describe-instances`,
 						`--instance-ids`, `i-0016d0fe3a11c73c2`,
-						`--query`, `Reservations[*].Instances[*].State.Name`,
+						`--query`, `Reservations[0].Instances[0].State.Name`,
 					},
 					{
 						`ec2`, `describe-addresses`,
@@ -258,7 +257,9 @@ credential_source = Ec2InstanceMetadata`)), comment)
 				It("execute as if no vm has been deployed", func() {
 					command, runner := createValidCommand("1.2.3.4", "10.10.10.10", "us-west-2")
 					runner.ExecuteWithEnvVarsReturnsOnCall(0, bytes.NewBufferString("terminated\r\n"), nil, nil)
-					runner.ExecuteWithEnvVarsReturnsOnCall(1, bytes.NewBufferString(fmt.Sprintf("\"%s\"\r\n", vmID)), nil, nil)
+					runner.ExecuteWithEnvVarsReturnsOnCall(1, bytes.NewBufferString(fmt.Sprintf("\"%s\"\r\n", vmID)), nil, nil) // ec2 run-instances
+					runner.ExecuteWithEnvVarsReturnsOnCall(2, bytes.NewBufferString("\"running\"\r\n"), nil, nil)               // ec2 describe-instances
+					runner.ExecuteWithEnvVarsReturnsOnCall(3, bytes.NewBufferString(fmt.Sprintf("\"%s\"\r\n", vmID)), nil, nil)
 
 					command.State = vmmanagers.StateInfo{
 						IAAS: "aws",
@@ -278,6 +279,7 @@ credential_source = Ec2InstanceMetadata`)), comment)
 					command, runner := createValidCommand("1.2.3.4", "10.10.10.10", "us-west-2")
 					runner.ExecuteWithEnvVarsReturnsOnCall(0, bytes.NewBufferString("[]\n"), nil, nil)
 					runner.ExecuteWithEnvVarsReturnsOnCall(1, bytes.NewBufferString(fmt.Sprintf("\"%s\"\r\n", vmID)), nil, nil)
+					runner.ExecuteWithEnvVarsReturnsOnCall(2, bytes.NewBufferString("\"running\"\r\n"), nil, nil) // ec2 describe-instances
 
 					command.State = vmmanagers.StateInfo{
 						IAAS: "aws",
@@ -393,11 +395,12 @@ credential_source = Ec2InstanceMetadata`)), comment)
 						Expect(err.Error()).To(ContainSubstring("aws error "))
 					},
 						Entry("create vm", 0, vmmanagers.Unknown),
-						Entry("find root device", 1, vmmanagers.Incomplete),
-						Entry("modify root device size", 2, vmmanagers.Incomplete),
-						Entry("find public ip address", 3, vmmanagers.Incomplete),
-						Entry("assign public ip address", 4, vmmanagers.Incomplete),
-						Entry("reboot vm", 5, vmmanagers.Incomplete),
+						Entry("wait for running", 1, vmmanagers.Incomplete),
+						Entry("find root device", 2, vmmanagers.Incomplete),
+						Entry("modify root device size", 3, vmmanagers.Incomplete),
+						Entry("find public ip address", 4, vmmanagers.Incomplete),
+						Entry("assign public ip address", 5, vmmanagers.Incomplete),
+						Entry("reboot vm", 6, vmmanagers.Incomplete),
 					)
 				})
 
@@ -551,8 +554,7 @@ opsman-configuration:
     private_ip: 1.2.3.4
 `)
 			runner.ExecuteWithEnvVarsReturnsOnCall(4, bytes.NewBufferString("some-id\r\n"), nil, nil)
-			// Add return values for waitTillVmRunning calls
-			runner.ExecuteWithEnvVarsReturnsOnCall(5, bytes.NewBufferString("running\r\n"), nil, nil)
+			runner.ExecuteWithEnvVarsReturnsOnCall(5, bytes.NewBufferString("\"running\"\r\n"), nil, nil) // waitUntilVMRunning()
 			runner.ExecuteWithEnvVarsReturnsOnCall(11, bytes.NewBufferString("stopped\r\n"), nil, nil)
 
 			_, _, err := command.CreateVM()