aws cloudstorage bucket refactor

api/v1alpha1/cloud_storage_types.go

@@ -19,11 +19,13 @@ type CloudStorage struct {
 type CloudStorageProvider string
 
 const (
-	AWSBucketProvider CloudStorageProvider = "aws"
+	AWSBucketProvider   CloudStorageProvider = "aws"
+	AzureBucketProvider CloudStorageProvider = "azure"
+	GCPBucketProvider   CloudStorageProvider = "gcp"
 )
 
 type CloudStorageSpec struct {
-	// Name is the name requested for the bucket
+	// Name is the name requested for the bucket (aws, gcp) or container (azure)
 	Name string `json:"name"`
 	// CreationSecret is the secret that is needed to be used while creating the bucket.
 	CreationSecret corev1.SecretKeySelector `json:"creationSecret"`
@@ -36,6 +38,14 @@ type CloudStorageSpec struct {
 	Region string `json:"region,omitempty"`
 	// +kubebuilder:validation:Enum=aws
 	Provider CloudStorageProvider `json:"provider"`
+
+	// https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/[email protected]#section-readme
+	// azure blob primary endpoint
+	// az storage account show -g <resource-group> -n <storage-account>
+	// need storage account name and key to create azure container
+	// az storage container create -n <container-name> --account-name <storage-account-name> --account-key <storage-account-key>
+	// azure account key will use CreationSecret to store key and account name
+
 }
 
 type CloudStorageStatus struct {

bundle/manifests/oadp.openshift.io_cloudstorages.yaml

@@ -55,7 +55,8 @@ spec:
                   for AWS Buckets
                 type: boolean
               name:
-                description: Name is the name requested for the bucket
+                description: Name is the name requested for the bucket (aws, gcp)
+                  or container (azure)
                 type: string
               provider:
                 enum:

config/crd/bases/oadp.openshift.io_cloudstorages.yaml

@@ -57,7 +57,8 @@ spec:
                   for AWS Buckets
                 type: boolean
               name:
-                description: Name is the name requested for the bucket
+                description: Name is the name requested for the bucket (aws, gcp)
+                  or container (azure)
                 type: string
               provider:
                 enum:

controllers/bsl.go

@@ -134,10 +134,19 @@ func (r *DPAReconciler) ReconcileBackupStorageLocations(log logr.Logger) (bool,
 				}
 				bsl.Spec.Credential = bslSpec.CloudStorage.Credential
 				bsl.Spec.Default = bslSpec.CloudStorage.Default
-				bsl.Spec.Provider = AWSProvider
 				bsl.Spec.ObjectStorage = &velerov1.ObjectStorageLocation{
 					Bucket: bucket.Spec.Name,
 				}
+				switch bucket.Spec.Provider {
+				case oadpv1alpha1.AWSBucketProvider:
+					bsl.Spec.Provider = AWSProvider
+				case oadpv1alpha1.AzureBucketProvider:
+					return fmt.Errorf("azure provider not yet supported")
+				case oadpv1alpha1.GCPBucketProvider:
+					return fmt.Errorf("gcp provider not yet supported")
+				default:
+					return fmt.Errorf("invalid provider")
+				}
 			}
 			return nil
 		})

go.mod

@@ -3,6 +3,10 @@ module github.com/openshift/oadp-operator
 go 1.16
 
 require (
+	github.com/Azure-Samples/azure-sdk-for-go-samples v0.0.0-20210506191746-b49c4162aa1d
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0
+	github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc
 	github.com/aws/aws-sdk-go v1.28.2
 	github.com/go-logr/logr v0.4.0
 	github.com/google/uuid v1.1.2

pkg/bucket/aws.go

@@ -0,0 +1,172 @@
+package bucket
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/openshift/oadp-operator/api/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type awsBucketClient struct {
+	bucket v1alpha1.CloudStorage
+	client client.Client
+}
+
+func (a awsBucketClient) Exists() (bool, error) {
+	s3Client, err := a.getS3Client()
+	if err != nil {
+		return false, err
+	}
+	input := &s3.HeadBucketInput{
+		Bucket: aws.String(a.bucket.Spec.Name),
+	}
+	_, err = s3Client.HeadBucket(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			// This is supposed to say "NoSuchBucket", but actually emits "NotFound"
+			// https://github.com/aws/aws-sdk-go/issues/2593
+			case s3.ErrCodeNoSuchBucket, "NotFound":
+				return false, nil
+			default:
+				// Return true, because we are unable to detemine if bucket exists or not
+				return true, fmt.Errorf("unable to determine bucket %v status: %v", a.bucket.Spec.Name, aerr.Error())
+			}
+		} else {
+			// Return true, because we are unable to detemine if bucket exists or not
+			return true, fmt.Errorf("unable to determine bucket %v status: %v", a.bucket.Spec.Name, aerr.Error())
+		}
+	}
+
+	err = a.tagBucket()
+	if err != nil {
+		return true, err
+	}
+
+	return true, nil
+}
+
+func (a awsBucketClient) Create() (bool, error) {
+	s3Client, err := a.getS3Client()
+	if err != nil {
+		return false, err
+	}
+	createBucketInput := &s3.CreateBucketInput{
+		ACL:    aws.String(s3.BucketCannedACLPrivate),
+		Bucket: aws.String(a.bucket.Spec.Name),
+	}
+	if a.bucket.Spec.Region != "us-east-1" {
+		createBucketConfiguration := &s3.CreateBucketConfiguration{
+			LocationConstraint: &a.bucket.Spec.Region,
+		}
+		createBucketInput.SetCreateBucketConfiguration(createBucketConfiguration)
+	}
+	if err := createBucketInput.Validate(); err != nil {
+		return false, fmt.Errorf("unable to validate %v bucket creation configuration: %v", a.bucket.Spec.Name, err)
+	}
+
+	_, err = s3Client.CreateBucket(createBucketInput)
+	if err != nil {
+		return false, err
+	}
+
+	// tag Bucket.
+	err = a.tagBucket()
+	if err != nil {
+		return true, err
+	}
+
+	return true, nil
+}
+
+func (a awsBucketClient) tagBucket() error {
+	s3Client, err := a.getS3Client()
+	// Clear bucket tags.
+	if err != nil {
+		return err
+	}
+	deleteInput := &s3.DeleteBucketTaggingInput{Bucket: aws.String(a.bucket.Spec.Name)}
+	_, err = s3Client.DeleteBucketTagging(deleteInput)
+	if err != nil {
+		return err
+	}
+	input := CreateBucketTaggingInput(a.bucket.Spec.Name, a.bucket.Spec.Tags)
+
+	_, err = s3Client.PutBucketTagging(input)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// CreateBucketTaggingInput creates an S3 PutBucketTaggingInput object,
+// which is used to associate a list of tags with a bucket.
+func CreateBucketTaggingInput(bucketname string, tags map[string]string) *s3.PutBucketTaggingInput {
+	putInput := &s3.PutBucketTaggingInput{
+		Bucket: aws.String(bucketname),
+		Tagging: &s3.Tagging{
+			TagSet: []*s3.Tag{},
+		},
+	}
+	for key, value := range tags {
+		newTag := s3.Tag{
+			Key:   aws.String(key),
+			Value: aws.String(value),
+		}
+		putInput.Tagging.TagSet = append(putInput.Tagging.TagSet, &newTag)
+	}
+	return putInput
+}
+
+func (a awsBucketClient) getS3Client() (s3iface.S3API, error) {
+	awsConfig := &aws.Config{Region: &a.bucket.Spec.Region}
+	cred, err := getCredentialFromCloudStorageSecret(a.client, a.bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := session.Options{
+		Config:            *awsConfig,
+		SharedConfigFiles: []string{cred},
+	}
+
+	if a.bucket.Spec.EnableSharedConfig != nil && *a.bucket.Spec.EnableSharedConfig {
+		opts.SharedConfigState = session.SharedConfigEnable
+	}
+
+	s, err := session.NewSessionWithOptions(opts)
+	if err != nil {
+		return nil, err
+	}
+	return s3.New(s), nil
+}
+
+func (a awsBucketClient) ForceCredentialRefresh() error {
+	return fmt.Errorf("force credential refresh is not yet implemented")
+}
+
+func (a awsBucketClient) Delete() (bool, error) {
+	s3Client, err := a.getS3Client()
+	if err != nil {
+		return false, err
+	}
+	deleteBucketInput := &s3.DeleteBucketInput{
+		Bucket: aws.String(a.bucket.Spec.Name),
+	}
+
+	if err := deleteBucketInput.Validate(); err != nil {
+		return false, fmt.Errorf("unable to validate %v bucket deletion configuration: %v", a.bucket.Spec.Name, err)
+	}
+
+	_, err = s3Client.DeleteBucket(deleteBucketInput)
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}