testing framework: allow providers to reference run blocks

internal/backend/local/test.go

@@ -62,6 +62,13 @@ type TestSuiteRunner struct {
 
 	// Verbose tells the runner to print out plan files during each test run.
 	Verbose bool
+
+	// configProviders is a cache of config keys mapped to all the providers
+	// referenced by the given config.
+	//
+	// The config keys are globally unique across an entire test suite, so we
+	// store this at the suite runner level to get maximum efficiency.
+	configProviders map[string]map[string]bool
 }
 
 func (runner *TestSuiteRunner) Stop() {
@@ -75,6 +82,9 @@ func (runner *TestSuiteRunner) Cancel() {
 func (runner *TestSuiteRunner) Test() (moduletest.Status, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
+	// First thing, initialise the config providers map.
+	runner.configProviders = make(map[string]map[string]bool)
+
 	suite, suiteDiags := runner.collectTests()
 	diags = diags.Append(suiteDiags)
 	if suiteDiags.HasErrors() {
@@ -349,7 +359,13 @@ func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, st
 		return state, false
 	}
 
-	resetConfig, configDiags := configtest.TransformConfigForTest(config, run, file, runner.globalVariables)
+	key := MainStateIdentifier
+	if run.Config.ConfigUnderTest != nil {
+		key = run.Config.Module.Source.String()
+	}
+	runner.gatherProviders(key, config)
+
+	resetConfig, configDiags := configtest.TransformConfigForTest(config, run, file, runner.globalVariables, runner.PriorStates, runner.Suite.configProviders[key])
 	defer resetConfig()
 
 	run.Diagnostics = run.Diagnostics.Append(configDiags)
@@ -372,7 +388,7 @@ func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, st
 		return state, false
 	}
 
-	variables, variableDiags := runner.GetVariables(config, run, file, references)
+	variables, variableDiags := runner.GetVariables(config, run, references)
 	run.Diagnostics = run.Diagnostics.Append(variableDiags)
 	if variableDiags.HasErrors() {
 		run.Status = moduletest.Error
@@ -563,7 +579,7 @@ func (runner *TestFileRunner) destroy(config *configs.Config, state *states.Stat
 
 	var diags tfdiags.Diagnostics
 
-	variables, variableDiags := runner.GetVariables(config, run, file, nil)
+	variables, variableDiags := runner.GetVariables(config, run, nil)
 	diags = diags.Append(variableDiags)
 
 	if diags.HasErrors() {
@@ -845,7 +861,7 @@ func (runner *TestFileRunner) cleanup(file *moduletest.File) {
 			diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Inconsistent state", fmt.Sprintf("Found inconsistent state while cleaning up %s. This is a bug in Terraform - please report it", file.Name)))
 		}
 	} else {
-		reset, configDiags := configtest.TransformConfigForTest(runner.Suite.Config, main.Run, file, runner.globalVariables)
+		reset, configDiags := configtest.TransformConfigForTest(runner.Suite.Config, main.Run, file, runner.globalVariables, runner.PriorStates, runner.Suite.configProviders[MainStateIdentifier])
 		diags = diags.Append(configDiags)
 
 		if !configDiags.HasErrors() {
@@ -920,7 +936,7 @@ func (runner *TestFileRunner) cleanup(file *moduletest.File) {
 
 		var diags tfdiags.Diagnostics
 
-		reset, configDiags := configtest.TransformConfigForTest(state.Run.Config.ConfigUnderTest, state.Run, file, runner.globalVariables)
+		reset, configDiags := configtest.TransformConfigForTest(state.Run.Config.ConfigUnderTest, state.Run, file, runner.globalVariables, runner.PriorStates, runner.Suite.configProviders[state.Run.Config.Module.Source.String()])
 		diags = diags.Append(configDiags)
 
 		updated := state.State
@@ -951,7 +967,7 @@ func (runner *TestFileRunner) cleanup(file *moduletest.File) {
 // more variables than are required by the config. FilterVariablesToConfig
 // should be called before trying to use these variables within a Terraform
 // plan, apply, or destroy operation.
-func (runner *TestFileRunner) GetVariables(config *configs.Config, run *moduletest.Run, file *moduletest.File, references []*addrs.Reference) (terraform.InputValues, tfdiags.Diagnostics) {
+func (runner *TestFileRunner) GetVariables(config *configs.Config, run *moduletest.Run, references []*addrs.Reference) (terraform.InputValues, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	// relevantVariables contains the variables that are of interest to this
@@ -1039,7 +1055,7 @@ func (runner *TestFileRunner) GetVariables(config *configs.Config, run *modulete
 		variables[name] = value.Value
 	}
 
-	ctx, ctxDiags := hcltest.EvalContext(exprs, variables, runner.PriorStates)
+	ctx, ctxDiags := hcltest.EvalContext(hcltest.TargetRunBlock, exprs, variables, runner.PriorStates)
 	diags = diags.Append(ctxDiags)
 
 	var failedContext bool
@@ -1188,3 +1204,51 @@ func (runner *TestFileRunner) initVariables(file *moduletest.File) {
 		runner.globalVariables[name] = unparsedTestVariableValue{expr}
 	}
 }
+
+func (runner *TestFileRunner) gatherProviders(key string, config *configs.Config) {
+	if _, exists := runner.Suite.configProviders[key]; exists {
+		// Then we've processed this key before, so skip it.
+		return
+	}
+
+	providers := make(map[string]bool)
+
+	// First, let's look at the required providers first.
+	for _, provider := range config.Module.ProviderRequirements.RequiredProviders {
+		providers[provider.Name] = true
+		for _, alias := range provider.Aliases {
+			providers[alias.StringCompact()] = true
+		}
+	}
+
+	// Second, we look at the defined provider configs.
+	for _, provider := range config.Module.ProviderConfigs {
+		providers[provider.Addr().StringCompact()] = true
+	}
+
+	// Third, we look at the resources and data sources.
+	for _, resource := range config.Module.ManagedResources {
+		if resource.ProviderConfigRef != nil {
+			providers[resource.ProviderConfigRef.String()] = true
+			continue
+		}
+		providers[resource.Provider.Type] = true
+	}
+	for _, datasource := range config.Module.DataResources {
+		if datasource.ProviderConfigRef != nil {
+			providers[datasource.ProviderConfigRef.String()] = true
+			continue
+		}
+		providers[datasource.Provider.Type] = true
+	}
+
+	// Finally, we look at any module calls to see if any providers are used
+	// in there.
+	for _, module := range config.Module.ModuleCalls {
+		for _, provider := range module.Providers {
+			providers[provider.InParent.String()] = true
+		}
+	}
+
+	runner.Suite.configProviders[key] = providers
+}

internal/command/test_test.go

@@ -1146,8 +1146,12 @@ is declared in run block "test".
   run "finalise"... skip
 main.tftest.hcl... tearing down
 main.tftest.hcl... fail
+providers.tftest.hcl... in progress
+  run "test"... fail
+providers.tftest.hcl... tearing down
+providers.tftest.hcl... fail
 
-Failure! 1 passed, 1 failed, 1 skipped.
+Failure! 1 passed, 2 failed, 1 skipped.
 `
 	actualOut := output.Stdout()
 	if diff := cmp.Diff(actualOut, expectedOut); len(diff) > 0 {
@@ -1169,9 +1173,9 @@ Error: Reference to unavailable run block
   on main.tftest.hcl line 16, in run "test":
   16:     input_two = run.finalise.response
 
-The run block "finalise" is not available to the current run block. You can
-only reference run blocks that are in the same test file and will execute
-before the current run block.
+The run block "finalise" has not executed yet. You can only reference run
+blocks that are in the same test file and will execute before the current run
+block.
 
 Error: Reference to unknown run block
 
@@ -1181,6 +1185,15 @@ Error: Reference to unknown run block
 The run block "madeup" does not exist within this test file. You can only
 reference run blocks that are in the same test file and will execute before
 the current run block.
+
+Error: Reference to unavailable variable
+
+  on providers.tftest.hcl line 3, in provider "test":
+   3:   resource_prefix = var.default
+
+The input variable "default" is not available to the current run block. You
+can only reference variables defined at the file or global levels when
+populating the variables block within a run block.
 `
 	actualErr := output.Stderr()
 	if diff := cmp.Diff(actualErr, expectedErr); len(diff) > 0 {
@@ -1690,3 +1703,156 @@ func TestTest_LongRunningTestJSON(t *testing.T) {
 		t.Errorf("unexpected output\n\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", strings.Join(expected, "\n"), strings.Join(messages, "\n"), diff)
 	}
 }
+
+func TestTest_RunBlocksInProviders(t *testing.T) {
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "provider_runs")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+
+	providerSource, close := newMockProviderSource(t, map[string][]string{
+		"test": {"1.0.0"},
+	})
+	defer close()
+
+	streams, done := terminal.StreamsForTesting(t)
+	view := views.NewView(streams)
+	ui := new(cli.MockUi)
+
+	meta := Meta{
+		testingOverrides: metaOverridesForProvider(provider.Provider),
+		Ui:               ui,
+		View:             view,
+		Streams:          streams,
+		ProviderSource:   providerSource,
+	}
+
+	init := &InitCommand{
+		Meta: meta,
+	}
+
+	if code := init.Run(nil); code != 0 {
+		t.Fatalf("expected status code 0 but got %d: %s", code, ui.ErrorWriter)
+	}
+
+	test := &TestCommand{
+		Meta: meta,
+	}
+
+	code := test.Run([]string{"-no-color"})
+	output := done(t)
+
+	if code != 0 {
+		t.Errorf("expected status code 0 but got %d", code)
+	}
+
+	expected := `main.tftest.hcl... in progress
+  run "setup"... pass
+  run "main"... pass
+main.tftest.hcl... tearing down
+main.tftest.hcl... pass
+
+Success! 2 passed, 0 failed.
+`
+	actual := output.All()
+	if diff := cmp.Diff(actual, expected); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}
+
+func TestTest_RunBlocksInProviders_BadReferences(t *testing.T) {
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "provider_runs_invalid")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+
+	providerSource, close := newMockProviderSource(t, map[string][]string{
+		"test": {"1.0.0"},
+	})
+	defer close()
+
+	streams, done := terminal.StreamsForTesting(t)
+	view := views.NewView(streams)
+	ui := new(cli.MockUi)
+
+	meta := Meta{
+		testingOverrides: metaOverridesForProvider(provider.Provider),
+		Ui:               ui,
+		View:             view,
+		Streams:          streams,
+		ProviderSource:   providerSource,
+	}
+
+	init := &InitCommand{
+		Meta: meta,
+	}
+
+	if code := init.Run(nil); code != 0 {
+		t.Fatalf("expected status code 0 but got %d: %s", code, ui.ErrorWriter)
+	}
+
+	test := &TestCommand{
+		Meta: meta,
+	}
+
+	code := test.Run([]string{"-no-color"})
+	output := done(t)
+
+	if code != 1 {
+		t.Errorf("expected status code 1 but got %d", code)
+	}
+
+	expectedOut := `missing_run_block.tftest.hcl... in progress
+  run "main"... fail
+missing_run_block.tftest.hcl... tearing down
+missing_run_block.tftest.hcl... fail
+unavailable_run_block.tftest.hcl... in progress
+  run "main"... fail
+unavailable_run_block.tftest.hcl... tearing down
+unavailable_run_block.tftest.hcl... fail
+unused_provider.tftest.hcl... in progress
+  run "main"... pass
+unused_provider.tftest.hcl... tearing down
+unused_provider.tftest.hcl... pass
+
+Failure! 1 passed, 2 failed.
+`
+	actualOut := output.Stdout()
+	if diff := cmp.Diff(actualOut, expectedOut); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expectedOut, actualOut, diff)
+	}
+
+	expectedErr := `
+Error: Reference to unknown run block
+
+  on missing_run_block.tftest.hcl line 2, in provider "test":
+   2:   resource_prefix = run.missing.resource_directory
+
+The run block "missing" does not exist within this test file. You can only
+reference run blocks that are in the same test file and will execute before
+the provider is required.
+
+Error: Reference to unavailable run block
+
+  on unavailable_run_block.tftest.hcl line 2, in provider "test":
+   2:   resource_prefix = run.main.resource_directory
+
+The run block "main" has not executed yet. You can only reference run blocks
+that are in the same test file and will execute before the provider is
+required.
+`
+	actualErr := output.Stderr()
+	if diff := cmp.Diff(actualErr, expectedErr); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expectedErr, actualErr, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}

internal/command/testdata/test/bad-references/providers.tftest.hcl

@@ -0,0 +1,6 @@
+
+provider "test" {
+  resource_prefix = var.default
+}
+
+run "test" {}

internal/command/testdata/test/provider_runs/main.tf

@@ -0,0 +1 @@
+resource "test_resource" "foo" {}

internal/command/testdata/test/provider_runs/main.tftest.hcl

@@ -0,0 +1,24 @@
+variables {
+  resource_directory = "resources"
+}
+
+provider "test" {
+  alias = "setup"
+  resource_prefix = var.resource_directory
+}
+
+run "setup" {
+  module {
+    source = "./setup"
+  }
+
+  providers = {
+    test = test.setup
+  }
+}
+
+provider "test" {
+  resource_prefix = run.setup.resource_directory
+}
+
+run "main" {}

internal/command/testdata/test/provider_runs/setup/main.tf

@@ -0,0 +1,11 @@
+variable "resource_directory" {
+  type = string
+}
+
+resource "test_resource" "foo" {
+  value = var.resource_directory
+}
+
+output "resource_directory" {
+  value = test_resource.foo.value
+}