internal/providercache: Handle built-in providers

Built-in providers are special providers that are distributed as part of
Terraform CLI itself, rather than being installed separately. They always
live in the terraform.io/builtin/... namespace so it's easier to see that
they are special, and currently there is only one built-in provider named
"terraform".

Previous commits established the addressing scheme for built-in providers.
This commit makes the installer aware of them to the extent that it knows
not to try to install them the usual way and it's able to report an error
if the user requests a built-in provider that doesn't exist or tries to
impose a particular version constraint for a built-in provider.

For the moment the tests for this are the ones in the "command" package
because that's where the existing testing infrastructure for this
functionality lives. A later commit should add some more focused unit
tests here in the internal/providercache package, too.

Author: apparentlymart

command/init.go

@@ -467,6 +467,16 @@ func (c *InitCommand) getProviders(earlyConfig *earlyconfig.Config, state *state
 		ProviderAlreadyInstalled: func(provider addrs.Provider, selectedVersion getproviders.Version) {
 			c.Ui.Info(fmt.Sprintf("- Using previously-installed %s v%s", provider, selectedVersion))
 		},
+		BuiltInProviderAvailable: func(provider addrs.Provider) {
+			c.Ui.Info(fmt.Sprintf("- %s is built in to Terraform", provider))
+		},
+		BuiltInProviderFailure: func(provider addrs.Provider, err error) {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Invalid dependency on built-in provider",
+				fmt.Sprintf("Cannot use %s: %s.", provider, err),
+			))
+		},
 		QueryPackagesBegin: func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints) {
 			if len(versionConstraints) > 0 {
 				c.Ui.Info(fmt.Sprintf("- Finding %s versions matching %q...", provider, getproviders.VersionConstraintsString(versionConstraints)))

command/init_test.go

@@ -1396,7 +1396,7 @@ func TestInit_pluginDirProvidersDoesNotGet(t *testing.T) {
 }
 
 // Verify that plugin-dir doesn't prevent discovery of internal providers
-func TestInit_pluginWithInternal(t *testing.T) {
+func TestInit_pluginDirWithBuiltIn(t *testing.T) {
 	td := tempDir(t)
 	copy.CopyDir(testFixturePath("init-internal"), td)
 	defer os.RemoveAll(td)
@@ -1406,7 +1406,7 @@ func TestInit_pluginWithInternal(t *testing.T) {
 	providerSource, close := newMockProviderSource(t, nil)
 	defer close()
 
-	ui := new(cli.MockUi)
+	ui := cli.NewMockUi()
 	m := Meta{
 		testingOverrides: metaOverridesForProvider(testProvider()),
 		Ui:               ui,
@@ -1418,10 +1418,53 @@ func TestInit_pluginWithInternal(t *testing.T) {
 	}
 
 	args := []string{"-plugin-dir", "./"}
-	//args := []string{}
 	if code := c.Run(args); code != 0 {
 		t.Fatalf("error: %s", ui.ErrorWriter)
 	}
+
+	outputStr := ui.OutputWriter.String()
+	if subStr := "terraform.io/builtin/terraform is built in to Terraform"; !strings.Contains(outputStr, subStr) {
+		t.Errorf("output should mention the terraform provider\nwant substr: %s\ngot:\n%s", subStr, outputStr)
+	}
+}
+
+func TestInit_invalidBuiltInProviders(t *testing.T) {
+	// This test fixture includes two invalid provider dependencies:
+	// - an implied dependency on terraform.io/builtin/terraform with an
+	//   explicit version number, which is not allowed because it's builtin.
+	// - an explicit dependency on terraform.io/builtin/nonexist, which does
+	//   not exist at all.
+	td := tempDir(t)
+	copy.CopyDir(testFixturePath("init-internal-invalid"), td)
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	// An empty provider source
+	providerSource, close := newMockProviderSource(t, nil)
+	defer close()
+
+	ui := cli.NewMockUi()
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+		ProviderSource:   providerSource,
+	}
+
+	c := &InitCommand{
+		Meta: m,
+	}
+
+	if code := c.Run(nil); code == 0 {
+		t.Fatalf("succeeded, but was expecting error\nstdout:\n%s\nstderr:\n%s", ui.OutputWriter, ui.ErrorWriter)
+	}
+
+	errStr := ui.ErrorWriter.String()
+	if subStr := "Cannot use terraform.io/builtin/terraform: built-in"; !strings.Contains(errStr, subStr) {
+		t.Errorf("error output should mention the terraform provider\nwant substr: %s\ngot:\n%s", subStr, errStr)
+	}
+	if subStr := "Cannot use terraform.io/builtin/nonexist: this Terraform release"; !strings.Contains(errStr, subStr) {
+		t.Errorf("error output should mention the 'nonexist' provider\nwant substr: %s\ngot:\n%s", subStr, errStr)
+	}
 }
 
 // The module in this test uses terraform 0.11-style syntax. We expect that the

command/meta_providers.go

@@ -58,6 +58,11 @@ func (m *Meta) providerInstallerCustomSource(source getproviders.Source) *provid
 	if globalCacheDir != nil {
 		inst.SetGlobalCacheDir(globalCacheDir)
 	}
+	var builtinProviderTypes []string
+	for ty := range m.internalProviders() {
+		builtinProviderTypes = append(builtinProviderTypes, ty)
+	}
+	inst.SetBuiltInProviderTypes(builtinProviderTypes)
 	return inst
 }
 

command/testdata/init-internal-invalid/main.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    nonexist = {
+      source = "terraform.io/builtin/nonexist"
+    }
+    terraform = {
+      version = "1.2.0"
+    }
+  }
+}

internal/providercache/installer.go

@@ -35,6 +35,12 @@ type Installer struct {
 	// both the disk space and the download time for a particular provider
 	// version between different configurations on the same system.
 	globalCacheDir *Dir
+
+	// builtInProviderTypes is an optional set of types that should be
+	// considered valid to appear in the special terraform.io/builtin/...
+	// namespace, which we use for providers that are built in to Terraform
+	// and thus do not need any separate installation step.
+	builtInProviderTypes []string
 }
 
 // NewInstaller constructs and returns a new installer with the given target
@@ -70,6 +76,24 @@ func (i *Installer) SetGlobalCacheDir(cacheDir *Dir) {
 	i.globalCacheDir = cacheDir
 }
 
+// SetBuiltInProviderTypes tells the receiver to consider the type names in the
+// given slice to be valid as providers in the special special
+// terraform.io/builtin/... namespace that we use for providers that are
+// built in to Terraform and thus do not need a separate installation step.
+//
+// If a caller requests installation of a provider in that namespace, the
+// installer will treat it as a no-op if its name exists in this list, but
+// will produce an error if it does not.
+//
+// The default, if this method isn't called, is for there to be no valid
+// builtin providers.
+//
+// Do not modify the buffer under the given slice after passing it to this
+// method.
+func (i *Installer) SetBuiltInProviderTypes(types []string) {
+	i.builtInProviderTypes = types
+}
+
 // EnsureProviderVersions compares the given provider requirements with what
 // is already available in the installer's target directory and then takes
 // appropriate installation actions to ensure that suitable packages
@@ -113,6 +137,42 @@ func (i *Installer) EnsureProviderVersions(ctx context.Context, reqs getprovider
 	mightNeed := map[addrs.Provider]getproviders.VersionSet{}
 MightNeedProvider:
 	for provider, versionConstraints := range reqs {
+		if provider.IsBuiltIn() {
+			// Built in providers do not require installation but we'll still
+			// verify that the requested provider name is valid.
+			valid := false
+			for _, name := range i.builtInProviderTypes {
+				if name == provider.Type {
+					valid = true
+					break
+				}
+			}
+			var err error
+			if valid {
+				if len(versionConstraints) == 0 {
+					// Other than reporting an event for the outcome of this
+					// provider, we'll do nothing else with it: it's just
+					// automatically available for use.
+					if cb := evts.BuiltInProviderAvailable; cb != nil {
+						cb(provider)
+					}
+				} else {
+					// A built-in provider is not permitted to have an explicit
+					// version constraint, because we can only use the version
+					// that is built in to the current Terraform release.
+					err = fmt.Errorf("built-in providers do not support explicit version constraints")
+				}
+			} else {
+				err = fmt.Errorf("this Terraform release has no built-in provider named %q", provider.Type)
+			}
+			if err != nil {
+				errs[provider] = err
+				if cb := evts.BuiltInProviderFailure; cb != nil {
+					cb(provider, err)
+				}
+			}
+			continue
+		}
 		acceptableVersions := versions.MeetingConstraints(versionConstraints)
 		if mode.forceQueryAllProviders() {
 			// If our mode calls for us to look for newer versions regardless

internal/providercache/installer_events.go

@@ -40,6 +40,19 @@ type InstallerEvents struct {
 	// available version.
 	ProviderAlreadyInstalled func(provider addrs.Provider, selectedVersion getproviders.Version)
 
+	// The BuiltInProvider... family of events describe the outcome for any
+	// requested providers that are built in to Terraform. Only one of these
+	// methods will be called for each such provider, and no other method
+	// will be called for them except that they are included in the
+	// aggregate PendingProviders map.
+	//
+	// The "Available" event reports that the requested builtin provider is
+	// available in this release of Terraform. The "Failure" event reports
+	// either that the provider is unavailable or that the request for it
+	// is invalid somehow.
+	BuiltInProviderAvailable func(provider addrs.Provider)
+	BuiltInProviderFailure   func(provider addrs.Provider, err error)
+
 	// The QueryPackages... family of events delimit the operation of querying
 	// a provider source for information about available packages matching
 	// a particular version constraint, prior to selecting a single version