Skip to content

Commit 5127f1e

Browse files
author
Paddy
authored
command: Unmanaged providers
This adds supports for "unmanaged" providers, or providers with process lifecycles not controlled by Terraform. These providers are assumed to be started before Terraform is launched, and are assumed to shut themselves down after Terraform has finished running. To do this, we must update the go-plugin dependency to v1.3.0, which added support for the "test mode" plugin serving that powers all this. As a side-effect of not needing to manage the process lifecycle anymore, Terraform also no longer needs to worry about the provider's binary, as it won't be used for anything anymore. Because of this, we can disable the init behavior that concerns itself with downloading that provider's binary, checking its version, and otherwise managing the binary. This is all managed on a per-provider basis, so managed providers that Terraform downloads, starts, and stops can be used in the same commands as unmanaged providers. The TF_REATTACH_PROVIDERS environment variable is added, and is a JSON encoding of the provider's address to the information we need to connect to it. This change enables two benefits: first, delve and other debuggers can now be attached to provider server processes, and Terraform can connect. This allows for attaching debuggers to provider processes, which before was difficult to impossible. Second, it allows the SDK test framework to host the provider in the same process as the test driver, while running a production Terraform binary against the provider. This allows for Go's built-in race detector and test coverage tooling to work as expected in provider tests. Unmanaged providers are expected to work in the exact same way as managed providers, with one caveat: Terraform kills provider processes and restarts them once per graph walk, meaning multiple times during most Terraform CLI commands. As unmanaged providers can't be killed by Terraform, and have no visibility into graph walks, unmanaged providers are likely to have differences in how their global mutable state behaves when compared to managed providers. Namely, unmanaged providers are likely to retain global state when managed providers would have reset it. Developers relying on global state should be aware of this.
1 parent 5d0b75d commit 5127f1e

File tree

105 files changed

+5923
-1811
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+5923
-1811
lines changed

‎builtin/providers/test/provider.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ func Provider() terraform.ResourceProvider {
4242
"test_resource_config_mode": testResourceConfigMode(),
4343
"test_resource_nested_id": testResourceNestedId(),
4444
"test_resource_provider_meta": testResourceProviderMeta(),
45+
"test_resource_signal": testResourceSignal(),
4546
"test_undeleteable": testResourceUndeleteable(),
4647
"test_resource_required_min": testResourceRequiredMin(),
4748
},

‎builtin/providers/test/resource_signal.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package test
2+
3+
import (
4+
"github.com/hashicorp/terraform/helper/schema"
5+
)
6+
7+
func testResourceSignal() *schema.Resource {
8+
return &schema.Resource{
9+
Create: testResourceSignalCreate,
10+
Read: testResourceSignalRead,
11+
Update: testResourceSignalUpdate,
12+
Delete: testResourceSignalDelete,
13+
14+
Importer: &schema.ResourceImporter{
15+
State: schema.ImportStatePassthrough,
16+
},
17+
18+
Schema: map[string]*schema.Schema{
19+
"optional": {
20+
Type: schema.TypeString,
21+
Optional: true,
22+
},
23+
},
24+
}
25+
}
26+
27+
func testResourceSignalCreate(d *schema.ResourceData, meta interface{}) error {
28+
d.SetId("testId")
29+
30+
return testResourceSignalRead(d, meta)
31+
}
32+
33+
func testResourceSignalRead(d *schema.ResourceData, meta interface{}) error {
34+
return nil
35+
}
36+
37+
func testResourceSignalUpdate(d *schema.ResourceData, meta interface{}) error {
38+
return testResourceSignalRead(d, meta)
39+
}
40+
41+
func testResourceSignalDelete(d *schema.ResourceData, meta interface{}) error {
42+
return nil
43+
}

‎command/e2etest/testdata/test-provider/main.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
provider "test" {
2+
3+
}
4+
5+
resource "test_resource_signal" "test" {
6+
}

‎command/e2etest/unmanaged_test.go‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
package e2etest
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"io/ioutil"
7+
"path/filepath"
8+
"strings"
9+
"testing"
10+
11+
"github.com/hashicorp/go-hclog"
12+
"github.com/hashicorp/go-plugin"
13+
"github.com/hashicorp/terraform/builtin/providers/test"
14+
"github.com/hashicorp/terraform/e2e"
15+
grpcplugin "github.com/hashicorp/terraform/helper/plugin"
16+
proto "github.com/hashicorp/terraform/internal/tfplugin5"
17+
tfplugin "github.com/hashicorp/terraform/plugin"
18+
)
19+
20+
// The tests in this file are for the "unmanaged provider workflow", which
21+
// includes variants of the following sequence, with different details:
22+
// terraform init
23+
// terraform plan
24+
// terraform apply
25+
//
26+
// These tests are run against an in-process server, and checked to make sure
27+
// they're not trying to control the lifecycle of the binary. They are not
28+
// checked for correctness of the operations themselves.
29+
30+
type reattachConfig struct {
31+
Protocol string
32+
Pid int
33+
Test bool
34+
Addr reattachConfigAddr
35+
}
36+
37+
type reattachConfigAddr struct {
38+
Network string
39+
String string
40+
}
41+
42+
type providerServer struct {
43+
*grpcplugin.GRPCProviderServer
44+
planResourceChangeCalled bool
45+
applyResourceChangeCalled bool
46+
}
47+
48+
func (p *providerServer) PlanResourceChange(ctx context.Context, req *proto.PlanResourceChange_Request) (*proto.PlanResourceChange_Response, error) {
49+
p.planResourceChangeCalled = true
50+
return p.GRPCProviderServer.PlanResourceChange(ctx, req)
51+
}
52+
53+
func (p *providerServer) ApplyResourceChange(ctx context.Context, req *proto.ApplyResourceChange_Request) (*proto.ApplyResourceChange_Response, error) {
54+
p.applyResourceChangeCalled = true
55+
return p.GRPCProviderServer.ApplyResourceChange(ctx, req)
56+
}
57+
58+
func TestUnmanagedSeparatePlan(t *testing.T) {
59+
t.Parallel()
60+
61+
fixturePath := filepath.Join("testdata", "test-provider")
62+
tf := e2e.NewBinary(terraformBin, fixturePath)
63+
defer tf.Close()
64+
65+
reattachCh := make(chan *plugin.ReattachConfig)
66+
closeCh := make(chan struct{})
67+
provider := &providerServer{
68+
GRPCProviderServer: grpcplugin.NewGRPCProviderServerShim(test.Provider()),
69+
}
70+
ctx, cancel := context.WithCancel(context.Background())
71+
defer cancel()
72+
go plugin.Serve(&plugin.ServeConfig{
73+
Logger: hclog.New(&hclog.LoggerOptions{
74+
Name: "plugintest",
75+
Level: hclog.Trace,
76+
Output: ioutil.Discard,
77+
}),
78+
Test: &plugin.ServeTestConfig{
79+
Context: ctx,
80+
ReattachConfigCh: reattachCh,
81+
CloseCh: closeCh,
82+
},
83+
GRPCServer: plugin.DefaultGRPCServer,
84+
VersionedPlugins: map[int]plugin.PluginSet{
85+
5: plugin.PluginSet{
86+
"provider": &tfplugin.GRPCProviderPlugin{
87+
GRPCProvider: func() proto.ProviderServer {
88+
return provider
89+
},
90+
},
91+
},
92+
},
93+
})
94+
config := <-reattachCh
95+
if config == nil {
96+
t.Fatalf("no reattach config received")
97+
}
98+
reattachStr, err := json.Marshal(map[string]reattachConfig{
99+
"hashicorp/test": reattachConfig{
100+
Protocol: string(config.Protocol),
101+
Pid: config.Pid,
102+
Test: true,
103+
Addr: reattachConfigAddr{
104+
Network: config.Addr.Network(),
105+
String: config.Addr.String(),
106+
},
107+
},
108+
})
109+
tf.AddEnv("TF_REATTACH_PROVIDERS=" + string(reattachStr))
110+
tf.AddEnv("PLUGIN_PROTOCOL_VERSION=5")
111+
112+
//// INIT
113+
stdout, stderr, err := tf.Run("init")
114+
if err != nil {
115+
t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr)
116+
}
117+
118+
// Make sure we didn't download the binary
119+
if strings.Contains(stdout, "Installing hashicorp/test v") {
120+
t.Errorf("test provider download message is present in init output:\n%s", stdout)
121+
}
122+
if tf.FileExists(filepath.Join(".terraform", "plugins", "registry.terraform.io", "hashicorp", "test")) {
123+
t.Errorf("test provider binary found in .terraform dir")
124+
}
125+
126+
//// PLAN
127+
_, stderr, err = tf.Run("plan", "-out=tfplan")
128+
if err != nil {
129+
t.Fatalf("unexpected plan error: %s\nstderr:\n%s", err, stderr)
130+
}
131+
132+
if !provider.planResourceChangeCalled {
133+
t.Error("PlanResourceChange not called on in-process provider")
134+
}
135+
136+
//// APPLY
137+
_, stderr, err = tf.Run("apply", "tfplan")
138+
if err != nil {
139+
t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr)
140+
}
141+
142+
if !provider.applyResourceChangeCalled {
143+
t.Error("ApplyResourceChange not called on in-process provider")
144+
}
145+
provider.applyResourceChangeCalled = false
146+
147+
//// DESTROY
148+
_, stderr, err = tf.Run("destroy", "-auto-approve")
149+
if err != nil {
150+
t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr)
151+
}
152+
153+
if !provider.applyResourceChangeCalled {
154+
t.Error("ApplyResourceChange (destroy) not called on in-process provider")
155+
}
156+
cancel()
157+
<-closeCh
158+
}

‎command/meta.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"strings"
1515
"time"
1616

17+
plugin "github.com/hashicorp/go-plugin"
1718
"github.com/hashicorp/terraform-svchost/disco"
1819
"github.com/hashicorp/terraform/addrs"
1920
"github.com/hashicorp/terraform/backend"
@@ -92,6 +93,9 @@ type Meta struct {
9293
// When this channel is closed, the command will be cancelled.
9394
ShutdownCh <-chan struct{}
9495

96+
// UnmanagedProviders are a set of providers that exist as processes predating Terraform, which Terraform should use but not worry about the lifecycle of.
97+
UnmanagedProviders map[addrs.Provider]*plugin.ReattachConfig
98+
9599
//----------------------------------------------------------
96100
// Protected: commands can set these
97101
//----------------------------------------------------------

‎command/meta_providers.go‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,11 @@ func (m *Meta) providerInstallerCustomSource(source getproviders.Source) *provid
6363
builtinProviderTypes = append(builtinProviderTypes, ty)
6464
}
6565
inst.SetBuiltInProviderTypes(builtinProviderTypes)
66+
unmanagedProviderTypes := make(map[addrs.Provider]struct{}, len(m.UnmanagedProviders))
67+
for ty := range m.UnmanagedProviders {
68+
unmanagedProviderTypes[ty] = struct{}{}
69+
}
70+
inst.SetUnmanagedProviderTypes(unmanagedProviderTypes)
6671
return inst
6772
}
6873

@@ -172,10 +177,13 @@ func (m *Meta) providerFactories() (map[addrs.Provider]providers.Factory, error)
172177
// and they'll just be ignored if not used.
173178
internalFactories := m.internalProviders()
174179

175-
factories := make(map[addrs.Provider]providers.Factory, len(selected)+len(internalFactories))
180+
factories := make(map[addrs.Provider]providers.Factory, len(selected)+len(internalFactories)+len(m.UnmanagedProviders))
176181
for name, factory := range internalFactories {
177182
factories[addrs.NewBuiltInProvider(name)] = factory
178183
}
184+
for provider, reattach := range m.UnmanagedProviders {
185+
factories[provider] = unmanagedProviderFactory(provider, reattach)
186+
}
179187
for provider, cached := range selected {
180188
factories[provider] = providerFactory(cached)
181189
}
@@ -202,14 +210,15 @@ func providerFactory(meta *providercache.CachedProvider) providers.Factory {
202210
})
203211

204212
config := &plugin.ClientConfig{
205-
Cmd: exec.Command(meta.ExecutableFile),
206213
HandshakeConfig: tfplugin.Handshake,
207-
VersionedPlugins: tfplugin.VersionedPlugins,
208-
Managed: true,
209214
Logger: logger,
210215
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
216+
Managed: true,
217+
Cmd: exec.Command(meta.ExecutableFile),
211218
AutoMTLS: enableProviderAutoMTLS,
219+
VersionedPlugins: tfplugin.VersionedPlugins,
212220
}
221+
213222
client := plugin.NewClient(config)
214223
rpcClient, err := client.Client()
215224
if err != nil {
@@ -224,6 +233,51 @@ func providerFactory(meta *providercache.CachedProvider) providers.Factory {
224233
// store the client so that the plugin can kill the child process
225234
p := raw.(*tfplugin.GRPCProvider)
226235
p.PluginClient = client
236+
237+
return p, nil
238+
}
239+
}
240+
241+
// unmanagedProviderFactory produces a provider factory that uses the passed
242+
// reattach information to connect to go-plugin processes that are already
243+
// running, and implements providers.Interface against it.
244+
func unmanagedProviderFactory(provider addrs.Provider, reattach *plugin.ReattachConfig) providers.Factory {
245+
return func() (providers.Interface, error) {
246+
logger := hclog.New(&hclog.LoggerOptions{
247+
Name: "unmanaged-plugin",
248+
Level: hclog.Trace,
249+
Output: os.Stderr,
250+
})
251+
252+
config := &plugin.ClientConfig{
253+
HandshakeConfig: tfplugin.Handshake,
254+
Logger: logger,
255+
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
256+
Managed: false,
257+
Reattach: reattach,
258+
}
259+
// TODO: we probably shouldn't hardcode the protocol version
260+
// here, but it'll do for now, because only one protocol
261+
// version is supported. Eventually, we'll probably want to
262+
// sneak it into the JSON ReattachConfigs.
263+
if plugins, ok := tfplugin.VersionedPlugins[5]; !ok {
264+
return nil, fmt.Errorf("no supported plugins for protocol 5")
265+
} else {
266+
config.Plugins = plugins
267+
}
268+
269+
client := plugin.NewClient(config)
270+
rpcClient, err := client.Client()
271+
if err != nil {
272+
return nil, err
273+
}
274+
275+
raw, err := rpcClient.Dispense(tfplugin.ProviderPluginName)
276+
if err != nil {
277+
return nil, err
278+
}
279+
280+
p := raw.(*tfplugin.GRPCProvider)
227281
return p, nil
228282
}
229283
}

‎commands.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ import (
66

77
"github.com/mitchellh/cli"
88

9+
"github.com/hashicorp/go-plugin"
910
svchost "github.com/hashicorp/terraform-svchost"
1011
"github.com/hashicorp/terraform-svchost/auth"
1112
"github.com/hashicorp/terraform-svchost/disco"
13+
"github.com/hashicorp/terraform/addrs"
1214
"github.com/hashicorp/terraform/command"
1315
"github.com/hashicorp/terraform/command/cliconfig"
1416
"github.com/hashicorp/terraform/command/webbrowser"
@@ -38,7 +40,7 @@ const (
3840
OutputPrefix = "o:"
3941
)
4042

41-
func initCommands(config *cliconfig.Config, services *disco.Disco, providerSrc getproviders.Source) {
43+
func initCommands(config *cliconfig.Config, services *disco.Disco, providerSrc getproviders.Source, unmanagedProviders map[addrs.Provider]*plugin.ReattachConfig) {
4244
var inAutomation bool
4345
if v := os.Getenv(runningInAutomationEnvName); v != "" {
4446
inAutomation = true
@@ -76,7 +78,8 @@ func initCommands(config *cliconfig.Config, services *disco.Disco, providerSrc g
7678
PluginCacheDir: config.PluginCacheDir,
7779
OverrideDataDir: dataDir,
7880

79-
ShutdownCh: makeShutdownCh(),
81+
ShutdownCh: makeShutdownCh(),
82+
UnmanagedProviders: unmanagedProviders,
8083
}
8184

8285
// The command list is included in the terraform -help

0 commit comments

Comments
 (0)