Skip to content

Commit 75d5061

Browse files
xiaozhu36xiaozhu36
committed
add new backend OSS
1 parent 03ddb91 commit 75d5061

45 files changed

Lines changed: 10561 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎backend/init/init.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
backendGCS "github.com/hashicorp/terraform/backend/remote-state/gcs"
1818
backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem"
1919
backendManta "github.com/hashicorp/terraform/backend/remote-state/manta"
20+
backendOSS "github.com/hashicorp/terraform/backend/remote-state/oss"
2021
backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3"
2122
backendSwift "github.com/hashicorp/terraform/backend/remote-state/swift"
2223
)
@@ -51,6 +52,7 @@ func init() {
5152
"etcdv3": func() backend.Backend { return backendetcdv3.New() },
5253
"gcs": func() backend.Backend { return backendGCS.New() },
5354
"manta": func() backend.Backend { return backendManta.New() },
55+
"oss": func() backend.Backend { return backendOSS.New() },
5456
}
5557

5658
// Add the legacy remote backends that haven't yet been convertd to

‎backend/remote-state/oss/backend.go‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
package oss
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"github.com/aliyun/aliyun-oss-go-sdk/oss"
7+
"github.com/denverdino/aliyungo/common"
8+
"github.com/denverdino/aliyungo/location"
9+
"github.com/hashicorp/terraform/backend"
10+
"github.com/hashicorp/terraform/helper/schema"
11+
"github.com/hashicorp/terraform/terraform"
12+
"os"
13+
"strings"
14+
15+
"log"
16+
)
17+
18+
// New creates a new backend for OSS remote state.
19+
func New() backend.Backend {
20+
s := &schema.Backend{
21+
Schema: map[string]*schema.Schema{
22+
"bucket": {
23+
Type: schema.TypeString,
24+
Required: true,
25+
Description: "The name of the OSS bucket",
26+
},
27+
28+
"key": {
29+
Type: schema.TypeString,
30+
Required: true,
31+
Description: "The path to the state file inside the bucket",
32+
ValidateFunc: func(v interface{}, s string) ([]string, []error) {
33+
// oss will strip leading slashes from an object, so while this will
34+
// technically be accepted by oss, it will break our workspace hierarchy.
35+
if strings.HasPrefix(v.(string), "/") {
36+
return nil, []error{fmt.Errorf("key must not start with '/'")}
37+
}
38+
return nil, nil
39+
},
40+
},
41+
"access_key": {
42+
Type: schema.TypeString,
43+
Optional: true,
44+
Description: "Alibaba Cloud Access Key ID",
45+
DefaultFunc: schema.EnvDefaultFunc("ALICLOUD_ACCESS_KEY", ""),
46+
},
47+
48+
"secret_key": {
49+
Type: schema.TypeString,
50+
Optional: true,
51+
Description: "Alibaba Cloud Access Secret Key",
52+
DefaultFunc: schema.EnvDefaultFunc("ALICLOUD_SECRET_KEY", ""),
53+
},
54+
55+
"security_token": {
56+
Type: schema.TypeString,
57+
Optional: true,
58+
Description: "Alibaba Cloud Security Token",
59+
DefaultFunc: schema.EnvDefaultFunc("ALICLOUD_SECURITY_TOKEN", os.Getenv("SECURITY_TOKEN")),
60+
},
61+
62+
"region": {
63+
Type: schema.TypeString,
64+
Required: true,
65+
Description: "The region of the OSS bucket. It will be ignored when 'endpoint' is specified.",
66+
DefaultFunc: schema.EnvDefaultFunc("ALICLOUD_REGION", "cn-beijing"),
67+
},
68+
69+
"endpoint": {
70+
Type: schema.TypeString,
71+
Optional: true,
72+
Description: "A custom endpoint for the OSS API",
73+
DefaultFunc: schema.EnvDefaultFunc("ALICLOUD_OSS_ENDPOINT", ""),
74+
},
75+
76+
"encrypt": {
77+
Type: schema.TypeBool,
78+
Optional: true,
79+
Description: "Whether to enable server side encryption of the state file",
80+
Default: false,
81+
},
82+
83+
"acl": {
84+
Type: schema.TypeString,
85+
Optional: true,
86+
Description: "Object ACL to be applied to the state file",
87+
Default: "",
88+
},
89+
90+
"workspace_key_prefix": {
91+
Type: schema.TypeString,
92+
Optional: true,
93+
Description: "The prefix applied to the non-default state path inside the bucket",
94+
Default: "workspaces",
95+
},
96+
},
97+
}
98+
99+
result := &Backend{Backend: s}
100+
result.Backend.ConfigureFunc = result.configure
101+
return result
102+
}
103+
104+
type Backend struct {
105+
*schema.Backend
106+
107+
// The fields below are set from configure
108+
ossClient *oss.Client
109+
110+
bucketName string
111+
keyName string
112+
serverSideEncryption bool
113+
acl string
114+
security_token string
115+
endpoint string
116+
workspaceKeyPrefix string
117+
}
118+
119+
func (b *Backend) configure(ctx context.Context) error {
120+
if b.ossClient != nil {
121+
return nil
122+
}
123+
124+
// Grab the resource data
125+
data := schema.FromContextBackendConfig(ctx)
126+
127+
b.bucketName = data.Get("bucket").(string)
128+
b.keyName = data.Get("key").(string)
129+
b.serverSideEncryption = data.Get("encrypt").(bool)
130+
b.acl = data.Get("acl").(string)
131+
b.workspaceKeyPrefix = data.Get("workspace_key_prefix").(string)
132+
access_key := data.Get("access_key").(string)
133+
secret_key := data.Get("secret_key").(string)
134+
security_token := data.Get("security_token").(string)
135+
endpoint := data.Get("endpoint").(string)
136+
if endpoint == "" {
137+
region := common.Region(data.Get("region").(string))
138+
if end, err := b.getOSSEndpointByRegion(access_key, secret_key, region); err != nil {
139+
return err
140+
} else {
141+
endpoint = end
142+
}
143+
}
144+
145+
log.Printf("[DEBUG] Instantiate OSS client using endpoint: %#v", endpoint)
146+
var options []oss.ClientOption
147+
if security_token != "" {
148+
options = append(options, oss.SecurityToken(security_token))
149+
}
150+
options = append(options, oss.UserAgent(fmt.Sprintf("HashiCorp-Terraform-v%s", terraform.VersionString())))
151+
152+
if client, err := oss.New(fmt.Sprintf("http://%s", endpoint), access_key, secret_key, options...); err != nil {
153+
return err
154+
} else {
155+
b.ossClient = client
156+
}
157+
158+
return nil
159+
}
160+
161+
func (b *Backend) getOSSEndpointByRegion(access_key, secret_key string, region common.Region) (string, error) {
162+
163+
endpoints, err := location.NewClient(access_key, secret_key).DescribeEndpoints(&location.DescribeEndpointsArgs{
164+
Id: region,
165+
ServiceCode: "oss",
166+
Type: "openAPI",
167+
})
168+
if err != nil {
169+
return "", fmt.Errorf("Describe endpoint using region: %#v got an error: %#v.", region, err)
170+
}
171+
endpointItem := endpoints.Endpoints.Endpoint
172+
endpoint := ""
173+
if endpointItem != nil && len(endpointItem) > 0 {
174+
endpoint = endpointItem[0].Endpoint
175+
}
176+
177+
return endpoint, nil
178+
}

‎backend/remote-state/oss/backend_state.go‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
package oss
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"sort"
7+
"strings"
8+
9+
"github.com/aliyun/aliyun-oss-go-sdk/oss"
10+
"github.com/hashicorp/terraform/backend"
11+
"github.com/hashicorp/terraform/state"
12+
"github.com/hashicorp/terraform/state/remote"
13+
"github.com/hashicorp/terraform/terraform"
14+
"log"
15+
"path"
16+
)
17+
18+
const (
19+
lockFileSuffix = ".tflock"
20+
)
21+
22+
// get a remote client configured for this state
23+
func (b *Backend) remoteClient(name string) (*RemoteClient, error) {
24+
if name == "" {
25+
return nil, errors.New("missing state name")
26+
}
27+
28+
client := &RemoteClient{
29+
ossClient: b.ossClient,
30+
bucketName: b.bucketName,
31+
statePath: b.statePath(name),
32+
lockPath: b.lockPath(name),
33+
serverSideEncryption: b.serverSideEncryption,
34+
acl: b.acl,
35+
}
36+
37+
return client, nil
38+
}
39+
40+
func (b *Backend) State(name string) (state.State, error) {
41+
client, err := b.remoteClient(name)
42+
if err != nil {
43+
return nil, err
44+
}
45+
46+
stateMgr := &remote.State{Client: client}
47+
48+
// Check to see if this state already exists.
49+
existing, err := b.States()
50+
if err != nil {
51+
return nil, err
52+
}
53+
log.Printf("Current state name: %s. All States:%#v", name, existing)
54+
55+
exists := false
56+
for _, s := range existing {
57+
if s == name {
58+
exists = true
59+
break
60+
}
61+
}
62+
// We need to create the object so it's listed by States.
63+
if !exists {
64+
// take a lock on this state while we write it
65+
lockInfo := state.NewLockInfo()
66+
lockInfo.Operation = "init"
67+
lockId, err := client.Lock(lockInfo)
68+
if err != nil {
69+
return nil, fmt.Errorf("Failed to lock OSS state: %s", err)
70+
}
71+
72+
// Local helper function so we can call it multiple places
73+
lockUnlock := func(e error) error {
74+
if err := stateMgr.Unlock(lockId); err != nil {
75+
return fmt.Errorf(strings.TrimSpace(stateUnlockError), lockId, err)
76+
}
77+
return e
78+
}
79+
80+
// Grab the value
81+
// This is to ensure that no one beat us to writing a state between
82+
// the `exists` check and taking the lock.
83+
if err := stateMgr.RefreshState(); err != nil {
84+
err = lockUnlock(err)
85+
return nil, err
86+
}
87+
88+
// If we have no state, we have to create an empty state
89+
if v := stateMgr.State(); v == nil {
90+
if err := stateMgr.WriteState(terraform.NewState()); err != nil {
91+
err = lockUnlock(err)
92+
return nil, err
93+
}
94+
if err := stateMgr.PersistState(); err != nil {
95+
err = lockUnlock(err)
96+
return nil, err
97+
}
98+
}
99+
100+
// Unlock, the state should now be initialized
101+
if err := lockUnlock(nil); err != nil {
102+
return nil, err
103+
}
104+
105+
}
106+
return stateMgr, nil
107+
}
108+
109+
func (b *Backend) States() ([]string, error) {
110+
bucket, err := b.ossClient.Bucket(b.bucketName)
111+
if err != nil {
112+
return []string{""}, fmt.Errorf("Error getting bucket: %#v", err)
113+
}
114+
115+
var options []oss.Option
116+
options = append(options, oss.Prefix(b.workspaceKeyPrefix))
117+
resp, err := bucket.ListObjects(options...)
118+
119+
if err != nil {
120+
return nil, err
121+
}
122+
123+
workspaces := []string{backend.DefaultStateName}
124+
for _, obj := range resp.Objects {
125+
workspace := b.keyEnv(obj.Key)
126+
if workspace != "" {
127+
workspaces = append(workspaces, workspace)
128+
}
129+
}
130+
131+
sort.Strings(workspaces[1:])
132+
return workspaces, nil
133+
}
134+
135+
func (b *Backend) DeleteState(name string) error {
136+
if name == backend.DefaultStateName || name == "" {
137+
return fmt.Errorf("can't delete default state")
138+
}
139+
140+
client, err := b.remoteClient(name)
141+
if err != nil {
142+
return err
143+
}
144+
log.Printf("Delete state %s ...", name)
145+
return client.Delete()
146+
}
147+
148+
// extract the object name from the OSS key
149+
func (b *Backend) keyEnv(key string) string {
150+
// we have 3 parts, the workspace key prefix, the workspace name, and the state key name
151+
parts := strings.SplitN(key, "/", 3)
152+
if len(parts) < 3 {
153+
// no workspace prefix here
154+
return ""
155+
}
156+
157+
// shouldn't happen since we listed by prefix
158+
if parts[0] != b.workspaceKeyPrefix {
159+
return ""
160+
}
161+
162+
// not our key, so don't include it in our listing
163+
if parts[2] != b.keyName {
164+
return ""
165+
}
166+
167+
return parts[1]
168+
}
169+
170+
func (b *Backend) statePath(name string) string {
171+
if name == backend.DefaultStateName && b.keyName != "" {
172+
return b.keyName
173+
}
174+
return path.Join(b.workspaceKeyPrefix, name, b.keyName)
175+
}
176+
177+
func (b *Backend) lockPath(name string) string {
178+
if name == backend.DefaultStateName && b.keyName != "" {
179+
return b.keyName + lockFileSuffix
180+
}
181+
return path.Join(b.workspaceKeyPrefix, name, b.keyName+lockFileSuffix)
182+
}
183+
184+
const stateUnlockError = `
185+
Error unlocking Alicloud OSS state file:
186+
187+
Lock ID: %s
188+
Error message: %#v
189+
190+
You may have to force-unlock this state in order to use it again.
191+
The Alibaba Cloud backend acquires a lock during initialization to ensure the initial state file is created.
192+
`

0 commit comments

Comments
 (0)