feat(test reports): storage addition

.gitignore

@@ -31,7 +31,7 @@ release/
 *.iml
 *.ipr
 *.iws
-*.xml
+#*.xml
 
 # Secrets environment file
 secrets.env

DOCS.md

@@ -56,6 +56,12 @@ echo "VELA_SCM_CLIENT=<Github OAuth Client ID>" >> .env
 echo "VELA_SCM_SECRET=<Github OAuth Client Secret>" >> .env
 ```
 
+* Add `minio` to `/etc/hosts` for nginx to resolve the local minio service when running Vela:
+
+```bash
+ sudo sh -c 'echo "127.0.0.1 minio" >> /etc/hosts'
+````
+
 ## Start
 
 **NOTE: Please review the [setup section](#setup) before moving forward.**

api/admin/storage.go

@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package admin
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/storage"
+	"github.com/go-vela/server/util"
+)
+
+// swagger:operation PUT /api/v1/admin/storage/bucket admin CreateBucket
+//
+// Create a new bucket
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+// - in: body
+//   name: body
+//   description: The bucket name to be created
+//   required: true
+//   schema:
+//     type: object
+//     properties:
+//       bucketName:
+//         type: string
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '201':
+//     description: Successfully created the bucket
+//   '400':
+//     description: Invalid request payload
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '500':
+//     description: Unexpected server error
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// CreateBucket represents the API handler to create a new bucket.
+func CreateBucket(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: creating bucket")
+
+	// capture body from API request
+	input := new(types.Bucket)
+
+	err := c.Bind(input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to decode JSON for bucket %s: %w", input.BucketName, err)
+
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	l.Debugf("bucket name: %s", input.BucketName)
+
+	err = storage.FromGinContext(c).CreateBucket(ctx, input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to create bucket: %w", err)
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+
+		return
+	}
+
+	c.Status(http.StatusCreated)
+}
+
+// swagger:operation GET /api/v1/admin/storage/presign admin GetPresignedURL
+//
+// # Generate a presigned URL for an object
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - in: query
+//     name: bucketName
+//     description: The name of the bucket
+//     required: true
+//     type: string
+//   - in: query
+//     name: objectName
+//     description: The name of the object
+//     required: true
+//     type: string
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully generated the presigned URL
+//   '400':
+//     description: Invalid request payload
+//     schema:
+//       "$ref": "#/definitions/Error"
+//   '500':
+//     description: Unexpected server error
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// GetPresignedURL represents the API handler to generate a presigned URL for an object.
+func GetPresignedURL(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: generating presigned URL")
+
+	// capture query parameters from API request
+	bucketName := c.Query("bucketName")
+	objectName := c.Query("objectName")
+
+	if bucketName == "" || objectName == "" {
+		retErr := fmt.Errorf("bucketName and objectName are required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	input := &types.Object{
+		Bucket:     types.Bucket{BucketName: bucketName},
+		ObjectName: objectName,
+	}
+
+	url, err := storage.FromGinContext(c).PresignedGetObject(ctx, input)
+	if err != nil || url == "" {
+		retErr := fmt.Errorf("unable to generate presigned URL: %w", err)
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+
+	c.JSON(http.StatusOK, url)
+}

api/storage/doc.go

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: Apache-2.0
+
+// Package storage provides the storage handlers for the Vela API.
+//
+// Usage:
+//
+//	import "github.com/go-vela/server/api/storage"
+package storage

api/storage/storage.go

@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package storage
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/api/types"
+	"github.com/go-vela/server/storage"
+	"github.com/go-vela/server/util"
+)
+
+// swagger:operation POST /api/v1/storage/info storage StorageInfo
+//
+// Get storage credentials
+//
+// ---
+// produces:
+// - application/json
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully retrieved storage credentials
+//     schema:
+//       "$ref": "#/definitions/StorageInfo"
+//   '401':
+//     description: Unauthorized
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// Info represents the API handler to
+// retrieve storage credentials as part of worker onboarding.
+func Info(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+
+	l.Info("requesting storage credentials with registration token")
+
+	// extract the public key that was packed into gin context
+	k := c.MustGet("access-key").(string)
+
+	// extract the storage-address that was packed into gin context
+	a := c.MustGet("storage-address").(string)
+
+	// extract the secret key that was packed into gin context
+	s := c.MustGet("secret-key").(string)
+
+	// extract bucket name that was packed into gin context
+	b := c.MustGet("storage-bucket").(string)
+
+	wr := types.StorageInfo{
+		StorageAccessKey: &k,
+		StorageAddress:   &a,
+		StorageSecretKey: &s,
+		StorageBucket:    &b,
+	}
+
+	c.JSON(http.StatusOK, wr)
+}
+
+// swagger:operation GET /api/v1/storage/{bucket}/objects storage ListObjects
+//
+// List objects in a bucket
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+// - in: path
+//   name: bucket
+//   description: Name of the bucket
+//   required: true
+//   type: string
+// security:
+//   - ApiKeyAuth: []
+// responses:
+//   '200':
+//     description: Successfully listed objects in the bucket
+//     schema:
+//       type: array
+//       items:
+//         type: string
+//   '500':
+//     description: Unexpected server error
+//     schema:
+//       "$ref": "#/definitions/Error"
+
+// ListObjects represents the API handler to list objects in a bucket.
+func ListObjects(c *gin.Context) {
+	enable := c.MustGet("storage-enable").(bool)
+	if !enable {
+		l := c.MustGet("logger").(*logrus.Entry)
+		l.Info("storage is not enabled, skipping credentials request")
+		c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"})
+
+		return
+	}
+
+	l := c.MustGet("logger").(*logrus.Entry)
+
+	l.Debug("listing objects in bucket")
+
+	// extract the bucket name from the request
+	bucketName := util.PathParameter(c, "bucket")
+
+	// create a new bucket object
+	b := &types.Bucket{
+		BucketName: bucketName,
+	}
+
+	// list objects in the bucket
+	objects, err := storage.FromGinContext(c).ListObjects(c.Request.Context(), b)
+	if err != nil {
+		l.Errorf("unable to list objects in bucket %s: %v", bucketName, err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"objects": objects})
+}

api/types/storage.go

@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: Apache-2.0
+
+package types
+
+import (
+	"github.com/minio/minio-go/v7"
+)
+
+// Bucket is the API types representation of an object storage.
+//
+// swagger:model CreateBucket
+type Bucket struct {
+	BucketName         string                   `json:"bucket_name,omitempty"`
+	MakeBucketOptions  minio.MakeBucketOptions  `json:"make_bucket_options,omitempty"`
+	ListObjectsOptions minio.ListObjectsOptions `json:"list_objects_options,omitempty"`
+}
+
+type Object struct {
+	ObjectName string `json:"object_name,omitempty"`
+	Bucket     Bucket `json:"bucket,omitempty"`
+	FilePath   string `json:"file_path,omitempty"`
+}