Merge pull request #170 from sfdc-pcg/refactor_and_test

Add GCS Getter

Author: azr

README.md

@@ -71,6 +71,7 @@ can be augmented at runtime by implementing the `Getter` interface.
   * Mercurial
   * HTTP
   * Amazon S3
+  * Google GCP
 
 In addition to the above protocols, go-getter has what are called "detectors."
 These take a URL and attempt to automatically choose the best protocol for
@@ -334,3 +335,14 @@ Some examples for these addressing schemes:
 - bucket.s3-eu-west-1.amazonaws.com/foo/bar
 - "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=KEYID&aws_access_key_secret=SECRETKEY&region=us-east-2"
 
+### GCS (`gcs`)
+
+#### GCS Authentication
+
+In order to access to GCS, authentication credentials should be provided. More information can be found [here](https://cloud.google.com/docs/authentication/getting-started)
+
+#### GCS Bucket Examples
+
+- gcs::https://www.googleapis.com/storage/v1/bucket
+- gcs::https://www.googleapis.com/storage/v1/bucket/foo.zip
+- www.googleapis.com/storage/v1/bucket/foo

detect.go

@@ -26,6 +26,7 @@ func init() {
 		new(GitDetector),
 		new(BitBucketDetector),
 		new(S3Detector),
+		new(GCSDetector),
 		new(FileDetector),
 	}
 }

detect_gcs.go

@@ -0,0 +1,43 @@
+package getter
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// GCSDetector implements Detector to detect GCS URLs and turn
+// them into URLs that the GCSGetter can understand.
+type GCSDetector struct{}
+
+func (d *GCSDetector) Detect(src, _ string) (string, bool, error) {
+	if len(src) == 0 {
+		return "", false, nil
+	}
+
+	if strings.Contains(src, "googleapis.com/") {
+		return d.detectHTTP(src)
+	}
+
+	return "", false, nil
+}
+
+func (d *GCSDetector) detectHTTP(src string) (string, bool, error) {
+
+	parts := strings.Split(src, "/")
+	if len(parts) < 5 {
+		return "", false, fmt.Errorf(
+			"URL is not a valid GCS URL")
+	}
+	version := parts[2]
+	bucket := parts[3]
+	object := strings.Join(parts[4:], "/")
+
+	url, err := url.Parse(fmt.Sprintf("https://www.googleapis.com/storage/%s/%s/%s",
+		version, bucket, object))
+	if err != nil {
+		return "", false, fmt.Errorf("error parsing GCS URL: %s", err)
+	}
+
+	return "gcs::" + url.String(), true, nil
+}

detect_gcs_test.go

@@ -0,0 +1,41 @@
+package getter
+
+import (
+	"testing"
+)
+
+func TestGCSDetector(t *testing.T) {
+	cases := []struct {
+		Input  string
+		Output string
+	}{
+		{
+			"www.googleapis.com/storage/v1/bucket/foo",
+			"gcs::https://www.googleapis.com/storage/v1/bucket/foo",
+		},
+		{
+			"www.googleapis.com/storage/v1/bucket/foo/bar",
+			"gcs::https://www.googleapis.com/storage/v1/bucket/foo/bar",
+		},
+		{
+			"www.googleapis.com/storage/v1/foo/bar.baz",
+			"gcs::https://www.googleapis.com/storage/v1/foo/bar.baz",
+		},
+	}
+
+	pwd := "/pwd"
+	f := new(GCSDetector)
+	for i, tc := range cases {
+		output, ok, err := f.Detect(tc.Input, pwd)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		if !ok {
+			t.Fatal("not ok")
+		}
+
+		if output != tc.Output {
+			t.Fatalf("%d: bad: %#v", i, output)
+		}
+	}
+}

get.go

@@ -67,6 +67,7 @@ func init() {
 	Getters = map[string]Getter{
 		"file":  new(FileGetter),
 		"git":   new(GitGetter),
+		"gcs":   new(GCSGetter),
 		"hg":    new(HgGetter),
 		"s3":    new(S3Getter),
 		"http":  httpGetter,

get_gcs.go

@@ -0,0 +1,172 @@
+package getter
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"google.golang.org/api/iterator"
+)
+
+// GCSGetter is a Getter implementation that will download a module from
+// a GCS bucket.
+type GCSGetter struct {
+	getter
+}
+
+func (g *GCSGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	ctx := g.Context()
+
+	// Parse URL
+	bucket, object, err := g.parseURL(u)
+	if err != nil {
+		return 0, err
+	}
+
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return 0, err
+	}
+	iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object})
+	for {
+		obj, err := iter.Next()
+		if err != nil && err != iterator.Done {
+			return 0, err
+		}
+
+		if err == iterator.Done {
+			break
+		}
+		if strings.HasSuffix(obj.Name, "/") {
+			// A directory matched the prefix search, so this must be a directory
+			return ClientModeDir, nil
+		} else if obj.Name != object {
+			// A file matched the prefix search and doesn't have the same name
+			// as the query, so this must be a directory
+			return ClientModeDir, nil
+		}
+	}
+	// There are no directories or subdirectories, and if a match was returned,
+	// it was exactly equal to the prefix search. So return File mode
+	return ClientModeFile, nil
+}
+
+func (g *GCSGetter) Get(dst string, u *url.URL) error {
+	ctx := g.Context()
+
+	// Parse URL
+	bucket, object, err := g.parseURL(u)
+	if err != nil {
+		return err
+	}
+
+	// Remove destination if it already exists
+	_, err = os.Stat(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if err == nil {
+		// Remove the destination
+		if err := os.RemoveAll(dst); err != nil {
+			return err
+		}
+	}
+
+	// Create all the parent directories
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Iterate through all matching objects.
+	iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object})
+	for {
+		obj, err := iter.Next()
+		if err != nil && err != iterator.Done {
+			return err
+		}
+		if err == iterator.Done {
+			break
+		}
+
+		if !strings.HasSuffix(obj.Name, "/") {
+			// Get the object destination path
+			objDst, err := filepath.Rel(object, obj.Name)
+			if err != nil {
+				return err
+			}
+			objDst = filepath.Join(dst, objDst)
+			// Download the matching object.
+			err = g.getObject(ctx, client, objDst, bucket, obj.Name)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (g *GCSGetter) GetFile(dst string, u *url.URL) error {
+	ctx := g.Context()
+
+	// Parse URL
+	bucket, object, err := g.parseURL(u)
+	if err != nil {
+		return err
+	}
+
+	client, err := storage.NewClient(ctx)
+	if err != nil {
+		return err
+	}
+	return g.getObject(ctx, client, dst, bucket, object)
+}
+
+func (g *GCSGetter) getObject(ctx context.Context, client *storage.Client, dst, bucket, object string) error {
+	rc, err := client.Bucket(bucket).Object(object).NewReader(ctx)
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+
+	// Create all the parent directories
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	f, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = Copy(ctx, f, rc)
+	return err
+}
+
+func (g *GCSGetter) parseURL(u *url.URL) (bucket, path string, err error) {
+	if strings.Contains(u.Host, "googleapis.com") {
+		hostParts := strings.Split(u.Host, ".")
+		if len(hostParts) != 3 {
+			err = fmt.Errorf("URL is not a valid GCS URL")
+			return
+		}
+
+		pathParts := strings.SplitN(u.Path, "/", 5)
+		if len(pathParts) != 5 {
+			err = fmt.Errorf("URL is not a valid GCS URL")
+			return
+		}
+		bucket = pathParts[3]
+		path = pathParts[4]
+	}
+	return
+}