tools/terraform-bundle: refactor to use new provider installer and provider directory layouts (#24629)

* tools/terraform-bundle: refactor to use new provider installer and
provider directory layouts

terraform-bundle now supports a "source" attribute for providers,
uses the new provider installer, and the archive it creates preserves
the new (required) directory hierarchy for providers, under a "plugins"
directory.

This is a breaking change in many ways: source is required for any
non-HashiCorp provider, locally-installed providers must be given a
source (can be arbitrary, see docs) and placed in the expected directory
hierarchy, and the unzipped archive is no longer flat; there is a new
"plugins" directory created with providers in the new directory layout.

This PR also extends the existing test to check the contents of the zip
file.

TODO: Re-enable e2e tests (currently suppressed with a t.Skip)
This commit includes an update to our travis configuration, so the terraform-bundle e2e tests run. It also turns off the e2e tests, which will fail until we have a terraform 0.13.* release under releases.hashicorp.com. We decided it was better to merge this now instead of waiting when we started seeing issues opened from users who built terraform-bundle from 0.13 and found it didn't work with 0.12 - better that they get an immediate error message from the binary directing them to build from the appropriate release.

Author: mildwonkey

.circleci/config.yml

@@ -81,10 +81,7 @@ jobs:
       - run:
           name: Run Go E2E Tests
           command: |
-            PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
-            echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
-            echo $PACKAGE_NAMES
-            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./command/e2etest
+            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./command/e2etest ./tools/terraform-bundle/e2etest
 
       # save coverage report parts
       - persist_to_workspace:

e2e/e2e.go

@@ -270,3 +270,8 @@ func GoBuild(pkgPath, tmpPrefix string) string {
 
 	return tmpFilename
 }
+
+// WorkDir() returns the binary workdir
+func (b *binary) WorkDir() string {
+	return b.workDir
+}

internal/providercache/dir.go

@@ -55,13 +55,13 @@ func NewDir(baseDir string) *Dir {
 	}
 }
 
-// newDirWithPlatform is a variant of NewDir that allows selecting a specific
+// NewDirWithPlatform is a variant of NewDir that allows selecting a specific
 // target platform, rather than taking the current one where this code is
 // running.
 //
 // This is primarily intended for portable unit testing and not particularly
-// useful in "real" callers.
-func newDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
+// useful in "real" callers, with the exception of terraform-bundle.
+func NewDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
 	return &Dir{
 		baseDir:        baseDir,
 		targetPlatform: platform,

internal/providercache/dir_modify_test.go

@@ -84,8 +84,8 @@ func TestLinkFromOtherCache(t *testing.T) {
 		addrs.DefaultRegistryHost, "hashicorp", "null",
 	)
 
-	srcDir := newDirWithPlatform(srcDirPath, windowsPlatform)
-	tmpDir := newDirWithPlatform(tmpDirPath, windowsPlatform)
+	srcDir := NewDirWithPlatform(srcDirPath, windowsPlatform)
+	tmpDir := NewDirWithPlatform(tmpDirPath, windowsPlatform)
 
 	// First we'll check our preconditions: srcDir should have only the
 	// null provider version 2.0.0 in it, because we're faking that we're on

internal/providercache/dir_test.go

@@ -40,7 +40,7 @@ func TestDirReading(t *testing.T) {
 
 	t.Run("ProviderLatestVersion", func(t *testing.T) {
 		t.Run("exists", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			got := dir.ProviderLatestVersion(nullProvider)
 			want := &CachedProvider{
@@ -59,7 +59,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no package for current platform", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// random provider is only cached for linux_amd64 in our fixtures dir
 			got := dir.ProviderLatestVersion(randomProvider)
@@ -70,7 +70,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no versions available at all", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// nonexist provider is not present in our fixtures dir at all
 			got := dir.ProviderLatestVersion(nonExistProvider)
@@ -84,7 +84,7 @@ func TestDirReading(t *testing.T) {
 
 	t.Run("ProviderVersion", func(t *testing.T) {
 		t.Run("exists", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("2.0.0"))
 			want := &CachedProvider{
@@ -100,7 +100,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("specified version is not cached", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// there is no v5.0.0 package in our fixtures dir
 			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("5.0.0"))
@@ -111,7 +111,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no package for current platform", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// random provider 1.2.0 is only cached for linux_amd64 in our fixtures dir
 			got := dir.ProviderVersion(randomProvider, versions.MustParseVersion("1.2.0"))
@@ -122,7 +122,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no versions available at all", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// nonexist provider is not present in our fixtures dir at all
 			got := dir.ProviderVersion(nonExistProvider, versions.MustParseVersion("1.0.0"))
@@ -135,7 +135,7 @@ func TestDirReading(t *testing.T) {
 	})
 
 	t.Run("AllAvailablePackages", func(t *testing.T) {
-		dir := newDirWithPlatform(testDir, linuxPlatform)
+		dir := NewDirWithPlatform(testDir, linuxPlatform)
 
 		got := dir.AllAvailablePackages()
 		want := map[addrs.Provider][]CachedProvider{

tools/terraform-bundle/README.md

@@ -34,12 +34,8 @@ the rest of this README to be in `PATH`.
 
 `terraform-bundle` is a repackaging of the module installation functionality
 from Terraform itself, so for best results you should build from the tag
-relating to the version of Terraform you plan to use. There is some slack in
-this requirement due to the fact that the module installation behavior changes
-rarely, but please note that in particular bundles for versions of
-Terraform before v0.12 must be built from a `terraform-bundle` built against
-a Terraform v0.11 tag at the latest, since Terraform v0.12 installs plugins
-in a different way that is not compatible.
+relating to the version of Terraform you plan to use. For example, use the v0.12
+tag to build a version of terraform-bundle compatible with Terraform v0.12*.
 
 ## Usage
 
@@ -59,19 +55,26 @@ terraform {
 # Define which provider plugins are to be included
 providers {
   # Include the newest "aws" provider version in the 1.0 series.
-  aws = ["~> 1.0"]
+  aws = {
+    versions = ["~> 1.0"]
+  }
 
   # Include both the newest 1.0 and 2.0 versions of the "google" provider.
   # Each item in these lists allows a distinct version to be added. If the
   # two expressions match different versions then _both_ are included in
   # the bundle archive.
-  google = ["~> 1.0", "~> 2.0"]
+  google = {
+    versions = ["~> 1.0", "~> 2.0"]
+  }
 
   # Include a custom plugin to the bundle. Will search for the plugin in the
-  # plugins directory, and package it with the bundle archive. Plugin must have
-  # a name of the form: terraform-provider-*, and must be build with the operating
-  # system and architecture that terraform enterprise is running, e.g. linux and amd64
-  customplugin = ["0.1"]
+  # plugins directory and package it with the bundle archive. Plugin must have
+  # a name of the form: terraform-provider-*, and must be built with the operating
+  # system and architecture that terraform enterprise is running, e.g. linux and amd64.
+  customplugin = {
+    versions = ["0.1"]
+    source = "myorg/customplugin"
+  }
 }
 
 ```
@@ -80,10 +83,10 @@ The `terraform` block defines which version of Terraform will be included
 in the bundle. An exact version is required here.
 
 The `providers` block defines zero or more providers to include in the bundle
-along with core Terraform. Each attribute in this block is a provider name,
-and its value is a list of version constraints. For each given constraint,
-`terraform-bundle` will find the newest available version matching the
-constraint and include it in the bundle.
+along with core Terraform. Each attribute is a provider name, and its value is a
+block with the list of version constraints and (optional) source. For each given
+constraint, `terraform-bundle` will find the newest available version matching
+the constraint and include it in the bundle.
 
 It is allowed to specify multiple constraints for the same provider, in which
 case multiple versions can be included in the resulting bundle. Each constraint
@@ -119,13 +122,44 @@ this composite version number so that bundle archives can be easily
 distinguished from official release archives and from each other when multiple
 bundles contain the same core Terraform version.
 
-To include custom plugins in the bundle file, create a local directory "./plugins"
-and put all the plugins you want to include there. Optionally, you can use the
-`-plugin-dir` flag to specify a location where to find the plugins. To be recognized
-as a valid plugin, the file must have a name of the form
-`terraform-provider-<NAME>_v<VERSION>`. In
-addition, ensure that the plugin is built using the same operating system and
-architecture used for Terraform Enterprise. Typically this will be `linux` and `amd64`.
+## Custom Plugins
+To include custom plugins in the bundle file, create a local directory
+"./plugins" and put all the plugins you want to include there, under the
+required [sub directory](#plugins-directory-layout). Optionally, you can use the
+`-plugin-dir` flag to specify a location where to find the plugins. To be
+recognized as a valid plugin, the file must have a name of the form
+`terraform-provider-<NAME>`. In addition, ensure that the plugin is built using
+the same operating system and architecture used for Terraform Enterprise.
+Typically this will be `linux` and `amd64`.
+
+### Plugins Directory Layout
+To include custom plugins in the bundle file, you must specify a "source"
+attribute in the configuration and place the plugin in the appropriate
+subdirectory under "./plugins". The directory must have the following layout:
+
+```
+./plugins/$SOURCEHOST/$SOURCENAMESPACE/$NAME/$VERSION/$OS_$ARCH/
+```
+
+When installing custom plugins, you may choose any arbitrary identifier for the
+$SOURCEHOST and $SOURCENAMESPACE subdirectories. 
+
+For example, given the following configuration and a plugin built for Terraform Enterprise:
+
+```
+providers {
+  customplugin = {
+    versions = ["0.1"]
+    source = "example.com/myorg/customplugin"
+  }
+}
+```
+
+The binary must be placed in the following directory:
+
+```
+./plugins/example.com/myorg/customplugin/0.1/linux_amd64/
+```
 
 ## Provider Resolution Behavior
 

tools/terraform-bundle/config.go

@@ -5,20 +5,27 @@ import (
 	"io/ioutil"
 
 	"github.com/hashicorp/hcl"
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 	"github.com/hashicorp/terraform/plugin/discovery"
 )
 
-var zeroTwelve = discovery.ConstraintStr(">= 0.12.0").MustParse()
+var zeroThirteen = discovery.ConstraintStr(">= 0.13.0").MustParse()
 
 type Config struct {
-	Terraform TerraformConfig                      `hcl:"terraform"`
-	Providers map[string][]discovery.ConstraintStr `hcl:"providers"`
+	Terraform TerraformConfig           `hcl:"terraform"`
+	Providers map[string]ProviderConfig `hcl:"providers"`
 }
 
 type TerraformConfig struct {
 	Version discovery.VersionStr `hcl:"version"`
 }
 
+type ProviderConfig struct {
+	Versions []string `hcl:"versions"`
+	Source   string   `hcl:"source"`
+}
+
 func LoadConfig(src []byte, filename string) (*Config, error) {
 	config := &Config{}
 	err := hcl.Decode(config, string(src))
@@ -49,17 +56,23 @@ func (c *Config) validate() error {
 	if v, err = c.Terraform.Version.Parse(); err != nil {
 		return fmt.Errorf("terraform.version: %s", err)
 	}
-	if !zeroTwelve.Allows(v) {
-		return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.12 and later; build terraform-bundle from the v0.11 branch or a v0.11.* tag to construct bundles for earlier versions")
+	if !zeroThirteen.Allows(v) {
+		return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.13 and later; build terraform-bundle from a release tag (such as v0.12.*) to construct bundles for earlier versions")
 	}
 
 	if c.Providers == nil {
-		c.Providers = map[string][]discovery.ConstraintStr{}
+		c.Providers = map[string]ProviderConfig{}
 	}
 
 	for k, cs := range c.Providers {
-		for _, c := range cs {
-			if _, err := c.Parse(); err != nil {
+		if cs.Source != "" {
+			_, diags := addrs.ParseProviderSourceString(cs.Source)
+			if diags.HasErrors() {
+				return fmt.Errorf("providers.%s: %s", k, diags.Err().Error())
+			}
+		}
+		for _, c := range cs.Versions {
+			if _, err := getproviders.ParseVersionConstraints(c); err != nil {
 				return fmt.Errorf("providers.%s: %s", k, err)
 			}
 		}