diff --git a/.gitignore b/.gitignore index a0b951aa0..cb2067738 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ release/ *.iml *.ipr *.iws -*.xml +#*.xml # Secrets environment file secrets.env diff --git a/DOCS.md b/DOCS.md index 940ee02ba..95ba914a5 100644 --- a/DOCS.md +++ b/DOCS.md @@ -56,6 +56,12 @@ echo "VELA_SCM_CLIENT=" >> .env echo "VELA_SCM_SECRET=" >> .env ``` +* Add `minio` to `/etc/hosts` for nginx to resolve the local minio service when running Vela: + +```bash + sudo sh -c 'echo "127.0.0.1 minio" >> /etc/hosts' +```` + ## Start **NOTE: Please review the [setup section](#setup) before moving forward.** diff --git a/api/admin/storage.go b/api/admin/storage.go new file mode 100644 index 000000000..405e1c0e3 --- /dev/null +++ b/api/admin/storage.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 + +package admin + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/storage" + "github.com/go-vela/server/util" +) + +// swagger:operation PUT /api/v1/admin/storage/bucket admin CreateBucket +// +// Create a new bucket +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: The bucket name to be created +// required: true +// schema: +// type: object +// properties: +// bucketName: +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '201': +// description: Successfully created the bucket +// '400': +// description: Invalid request payload +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// CreateBucket represents the API handler to create a new bucket. +func CreateBucket(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + + l.Debug("platform admin: creating bucket") + + // capture body from API request + input := new(types.Bucket) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for bucket %s: %w", input.BucketName, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + l.Debugf("bucket name: %s", input.BucketName) + + err = storage.FromGinContext(c).CreateBucket(ctx, input) + if err != nil { + retErr := fmt.Errorf("unable to create bucket: %w", err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.Status(http.StatusCreated) +} + +// swagger:operation GET /api/v1/admin/storage/presign admin GetPresignedURL +// +// # Generate a presigned URL for an object +// +// --- +// produces: +// - application/json +// parameters: +// - in: query +// name: bucketName +// description: The name of the bucket +// required: true +// type: string +// - in: query +// name: objectName +// description: The name of the object +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully generated the presigned URL +// '400': +// description: Invalid request payload +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// GetPresignedURL represents the API handler to generate a presigned URL for an object. +func GetPresignedURL(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + ctx := c.Request.Context() + + l.Debug("platform admin: generating presigned URL") + + // capture query parameters from API request + bucketName := c.Query("bucketName") + objectName := c.Query("objectName") + + if bucketName == "" || objectName == "" { + retErr := fmt.Errorf("bucketName and objectName are required") + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + input := &types.Object{ + Bucket: types.Bucket{BucketName: bucketName}, + ObjectName: objectName, + } + + url, err := storage.FromGinContext(c).PresignedGetObject(ctx, input) + if err != nil || url == "" { + retErr := fmt.Errorf("unable to generate presigned URL: %w", err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + c.JSON(http.StatusOK, url) +} diff --git a/api/storage/doc.go b/api/storage/doc.go new file mode 100644 index 000000000..aae3e99dd --- /dev/null +++ b/api/storage/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package storage provides the storage handlers for the Vela API. +// +// Usage: +// +// import "github.com/go-vela/server/api/storage" +package storage diff --git a/api/storage/storage.go b/api/storage/storage.go new file mode 100644 index 000000000..3ac9c9854 --- /dev/null +++ b/api/storage/storage.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/storage" + "github.com/go-vela/server/util" +) + +// swagger:operation POST /api/v1/storage/info storage StorageInfo +// +// Get storage credentials +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved storage credentials +// schema: +// "$ref": "#/definitions/StorageInfo" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" + +// Info represents the API handler to +// retrieve storage credentials as part of worker onboarding. +func Info(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + + l.Info("requesting storage credentials with registration token") + + // extract the public key that was packed into gin context + k := c.MustGet("access-key").(string) + + // extract the storage-address that was packed into gin context + a := c.MustGet("storage-address").(string) + + // extract the secret key that was packed into gin context + s := c.MustGet("secret-key").(string) + + // extract bucket name that was packed into gin context + b := c.MustGet("storage-bucket").(string) + + wr := types.StorageInfo{ + StorageAccessKey: &k, + StorageAddress: &a, + StorageSecretKey: &s, + StorageBucket: &b, + } + + c.JSON(http.StatusOK, wr) +} + +// swagger:operation GET /api/v1/storage/{bucket}/objects storage ListObjects +// +// List objects in a bucket +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: bucket +// description: Name of the bucket +// required: true +// type: string +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully listed objects in the bucket +// schema: +// type: array +// items: +// type: string +// '500': +// description: Unexpected server error +// schema: +// "$ref": "#/definitions/Error" + +// ListObjects represents the API handler to list objects in a bucket. +func ListObjects(c *gin.Context) { + enable := c.MustGet("storage-enable").(bool) + if !enable { + l := c.MustGet("logger").(*logrus.Entry) + l.Info("storage is not enabled, skipping credentials request") + c.JSON(http.StatusForbidden, gin.H{"error": "storage is not enabled"}) + + return + } + + l := c.MustGet("logger").(*logrus.Entry) + + l.Debug("listing objects in bucket") + + // extract the bucket name from the request + bucketName := util.PathParameter(c, "bucket") + + // create a new bucket object + b := &types.Bucket{ + BucketName: bucketName, + } + + // list objects in the bucket + objects, err := storage.FromGinContext(c).ListObjects(c.Request.Context(), b) + if err != nil { + l.Errorf("unable to list objects in bucket %s: %v", bucketName, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + + return + } + + c.JSON(http.StatusOK, gin.H{"objects": objects}) +} diff --git a/api/types/storage.go b/api/types/storage.go new file mode 100644 index 000000000..607b55782 --- /dev/null +++ b/api/types/storage.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "github.com/minio/minio-go/v7" +) + +// Bucket is the API types representation of an object storage. +// +// swagger:model CreateBucket +type Bucket struct { + BucketName string `json:"bucket_name,omitempty"` + MakeBucketOptions minio.MakeBucketOptions `json:"make_bucket_options,omitempty"` + ListObjectsOptions minio.ListObjectsOptions `json:"list_objects_options,omitempty"` +} + +type Object struct { + ObjectName string `json:"object_name,omitempty"` + Bucket Bucket `json:"bucket,omitempty"` + FilePath string `json:"file_path,omitempty"` +} diff --git a/api/types/storage_info.go b/api/types/storage_info.go new file mode 100644 index 000000000..247a64046 --- /dev/null +++ b/api/types/storage_info.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +// StorageInfo is the API representation of a StorageInfo. +// +// swagger:model StorageInfo +type StorageInfo struct { + StorageAccessKey *string `json:"storage_access_key,omitempty"` + StorageSecretKey *string `json:"storage_secret_key,omitempty"` + StorageAddress *string `json:"storage_address,omitempty"` + StorageBucket *string `json:"storage_bucket,omitempty"` +} + +// GetAccessKey returns the StorageAccessKey field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetAccessKey() string { + // return zero value if StorageInfo type or StorageAccessKey field is nil + if w == nil || w.StorageAccessKey == nil { + return "" + } + + return *w.StorageAccessKey +} + +// GetSecretKey returns the StorageSecretKey field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetSecretKey() string { + // return zero value if StorageInfo type or StorageSecretKey field is nil + if w == nil || w.StorageSecretKey == nil { + return "" + } + + return *w.StorageSecretKey +} + +// GetStorageAddress returns the StorageAddress field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetStorageAddress() string { + // return zero value if StorageInfo type or StorageAddress field is nil + if w == nil || w.StorageAddress == nil { + return "" + } + + return *w.StorageAddress +} + +// GetStorageBucket returns the StorageBucket field. +// +// When the provided StorageInfo type is nil, or the field within +// the type is nil, it returns an empty string for the field. +func (w *StorageInfo) GetStorageBucket() string { + // return zero value if StorageInfo type or StorageBucket field is nil + if w == nil || w.StorageBucket == nil { + return "" + } + + return *w.StorageBucket +} + +// SetAccessKey sets the StorageAccessKey field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetAccessKey(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageAccessKey = &v +} + +// SetSecretKey sets the StorageSecretKey field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetSecretKey(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageSecretKey = &v +} + +// SetStorageAddress sets the StorageAddress field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetStorageAddress(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageAddress = &v +} + +// SetStorageBucket sets the StorageBucket field. +// +// When the provided StorageInfo type is nil, it +// will set nothing and immediately return. +func (w *StorageInfo) SetStorageBucket(v string) { + // return if StorageInfo type is nil + if w == nil { + return + } + + w.StorageBucket = &v +} diff --git a/api/types/storage_info_test.go b/api/types/storage_info_test.go new file mode 100644 index 000000000..17a5b2595 --- /dev/null +++ b/api/types/storage_info_test.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "testing" +) + +func TestTypes_StorageInfo_Getters(t *testing.T) { + // setup tests + tests := []struct { + sI *StorageInfo + want *StorageInfo + }{ + { + sI: testStorageInfo(), + want: testStorageInfo(), + }, + { + sI: new(StorageInfo), + want: new(StorageInfo), + }, + } + + // run tests + for _, test := range tests { + if test.sI.GetAccessKey() != test.want.GetAccessKey() { + t.Errorf("GetAccessKey is %v, want %v", test.sI.GetAccessKey(), test.want.GetAccessKey()) + } + + if test.sI.GetSecretKey() != test.want.GetSecretKey() { + t.Errorf("GetSecretKey is %v, want %v", test.sI.GetSecretKey(), test.want.GetSecretKey()) + } + + if test.sI.GetStorageAddress() != test.want.GetStorageAddress() { + t.Errorf("GetStorageAddress is %v, want %v", test.sI.GetStorageAddress(), test.want.GetStorageAddress()) + } + if test.sI.GetStorageBucket() != test.want.GetStorageBucket() { + t.Errorf("GetStorageBucket is %v, want %v", test.sI.GetStorageBucket(), test.want.GetStorageBucket()) + } + } +} + +func TestTypes_StorageInfo_Setters(t *testing.T) { + // setup types + var sI *StorageInfo + + // setup tests + tests := []struct { + sI *StorageInfo + want *StorageInfo + }{ + { + sI: testStorageInfo(), + want: testStorageInfo(), + }, + { + sI: sI, + want: new(StorageInfo), + }, + } + + // run tests + for _, test := range tests { + test.sI.SetAccessKey(test.want.GetAccessKey()) + test.sI.SetSecretKey(test.want.GetSecretKey()) + test.sI.SetStorageAddress(test.want.GetStorageAddress()) + test.sI.SetStorageBucket(test.want.GetStorageBucket()) + + if test.sI.GetAccessKey() != test.want.GetAccessKey() { + t.Errorf("GetAccessKey is %v, want %v", test.sI.GetAccessKey(), test.want.GetAccessKey()) + } + + if test.sI.GetSecretKey() != test.want.GetSecretKey() { + t.Errorf("GetSecretKey is %v, want %v", test.sI.GetSecretKey(), test.want.GetSecretKey()) + } + + if test.sI.GetStorageAddress() != test.want.GetStorageAddress() { + t.Errorf("GetStorageAddress is %v, want %v", test.sI.GetStorageAddress(), test.want.GetStorageAddress()) + } + } +} + +// testStorageInfo is a test helper function to register a StorageInfo +// type with all fields set to a fake value. +func testStorageInfo() *StorageInfo { + sI := new(StorageInfo) + sI.SetAccessKey("fakeAccessKey") + sI.SetSecretKey("fakeSecretKey") + sI.SetStorageAddress("http://localhost:8080") + + return sI +} diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 01c41c999..2967434aa 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -17,6 +17,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/scm" "github.com/go-vela/server/secret" + "github.com/go-vela/server/storage" "github.com/go-vela/server/tracing" "github.com/go-vela/server/version" ) @@ -58,6 +59,9 @@ func main() { // Add Tracing Flags cmd.Flags = append(cmd.Flags, tracing.Flags...) + // Add S3 Flags + cmd.Flags = append(cmd.Flags, storage.Flags...) + if err = cmd.Run(context.Background(), os.Args); err != nil { logrus.Fatal(err) } diff --git a/cmd/vela-server/metadata.go b/cmd/vela-server/metadata.go index f23e309c1..347c972b4 100644 --- a/cmd/vela-server/metadata.go +++ b/cmd/vela-server/metadata.go @@ -45,6 +45,13 @@ func setupMetadata(c *cli.Command) (*internal.Metadata, error) { m.Vela = vela + storage, err := metadataStorage(c) + if err != nil { + return nil, err + } + + m.Storage = storage + return m, nil } @@ -93,6 +100,21 @@ func metadataSource(c *cli.Command) (*internal.Source, error) { }, nil } +// helper function to capture the queue metadata from the CLI arguments. +func metadataStorage(c *cli.Command) (*internal.Storage, error) { + logrus.Trace("creating storage metadata from CLI configuration") + + u, err := url.Parse(c.String("storage.addr")) + if err != nil { + return nil, err + } + + return &internal.Storage{ + Driver: c.String("storage.driver"), + Host: u.Host, + }, nil +} + // helper function to capture the Vela metadata from the CLI arguments. // //nolint:unparam // ignore unparam for now diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 2e8882db6..4501a9181 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -26,6 +26,7 @@ import ( "github.com/go-vela/server/queue" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" + "github.com/go-vela/server/storage" "github.com/go-vela/server/tracing" "github.com/go-vela/server/util" ) @@ -108,6 +109,11 @@ func server(ctx context.Context, cmd *cli.Command) error { return err } + st, err := storage.FromCLICommand(ctx, cmd) + if err != nil { + return err + } + metadata, err := setupMetadata(cmd) if err != nil { return err @@ -183,10 +189,12 @@ func server(ctx context.Context, cmd *cli.Command) error { middleware.Metadata(metadata), middleware.TokenManager(tm), middleware.Queue(queue), + middleware.Storage(st), middleware.RequestVersion, middleware.Secret(cmd.String("vela-secret")), middleware.Secrets(secrets), middleware.Scm(scm), + middleware.Storage(st), middleware.QueueSigningPrivateKey(cmd.String("queue.private-key")), middleware.QueueSigningPublicKey(cmd.String("queue.public-key")), middleware.QueueAddress(cmd.String("queue.addr")), @@ -203,6 +211,11 @@ func server(ctx context.Context, cmd *cli.Command) error { middleware.ScheduleFrequency(cmd.Duration("schedule-minimum-frequency")), middleware.TracingClient(tc), middleware.TracingInstrumentation(tc), + middleware.StorageAddress(cmd.String("storage.addr")), + middleware.StorageAccessKey(cmd.String("storage.access.key")), + middleware.StorageSecretKey(cmd.String("storage.secret.key")), + middleware.StorageBucket(cmd.String("storage.bucket.name")), + middleware.StorageEnable(cmd.Bool("storage.enable")), ) addr, err := url.Parse(cmd.String("server-addr")) diff --git a/compiler/registry/github/github.go b/compiler/registry/github/github.go index 6ef0a6c13..d45c980bb 100644 --- a/compiler/registry/github/github.go +++ b/compiler/registry/github/github.go @@ -59,6 +59,7 @@ func New(ctx context.Context, address, token string) (*Client, error) { // overwrite the github client c.githubClient = gitClient + //nolint:revive // ignore returning unexported engine return c, nil } diff --git a/compiler/types/yaml/yaml/secret.go b/compiler/types/yaml/yaml/secret.go index 97b836a09..efb0b86a9 100644 --- a/compiler/types/yaml/yaml/secret.go +++ b/compiler/types/yaml/yaml/secret.go @@ -157,7 +157,7 @@ func (o *Origin) Empty() bool { // MergeEnv takes a list of environment variables and attempts // to set them in the secret environment. If the environment -// variable already exists in the secret, than this will +// variable already exists in the secret, then this will // overwrite the existing environment variable. func (o *Origin) MergeEnv(environment map[string]string) error { // check if the secret container is empty diff --git a/constants/driver.go b/constants/driver.go index e42b924c5..fb716e435 100644 --- a/constants/driver.go +++ b/constants/driver.go @@ -62,3 +62,11 @@ const ( // DriverGitLab defines the driver type when integrating with a Gitlab source code system. DriverGitlab = "gitlab" ) + +// Server storage drivers. +const ( + // DriverMinio defines the driver type when integrating with a local storage system. + DriverMinio = "minio" + // DriverAws defines the driver type when integrating with an AWS S3 storage system. + DriverAws = "aws" +) diff --git a/database/build/build.go b/database/build/build.go index 9ee4df149..e29260035 100644 --- a/database/build/build.go +++ b/database/build/build.go @@ -77,5 +77,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableBuild, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/database/deployment/deployment.go b/database/deployment/deployment.go index b6f8af475..9860257d0 100644 --- a/database/deployment/deployment.go +++ b/database/deployment/deployment.go @@ -77,5 +77,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableDeployment, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/database/hook/hook.go b/database/hook/hook.go index 4f24fc5cd..fea492dfb 100644 --- a/database/hook/hook.go +++ b/database/hook/hook.go @@ -77,5 +77,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableHook, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/database/repo/repo.go b/database/repo/repo.go index 99549b07b..e9c35e210 100644 --- a/database/repo/repo.go +++ b/database/repo/repo.go @@ -77,5 +77,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableRepo, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/database/secret/secret.go b/database/secret/secret.go index 46406f30e..2b8607ccc 100644 --- a/database/secret/secret.go +++ b/database/secret/secret.go @@ -77,5 +77,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableSecret, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/database/service/service.go b/database/service/service.go index f92ec461a..f294406f7 100644 --- a/database/service/service.go +++ b/database/service/service.go @@ -69,5 +69,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create %s table: %w", constants.TableService, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/database/settings/settings.go b/database/settings/settings.go index a10ddb55c..51919e3fc 100644 --- a/database/settings/settings.go +++ b/database/settings/settings.go @@ -71,5 +71,6 @@ func New(opts ...EngineOpt) (*Engine, error) { return nil, fmt.Errorf("unable to create %s table: %w", TableSettings, err) } + //nolint:revive // ignore returning unexported engine return e, nil } diff --git a/docker-compose.yml b/docker-compose.yml index a3ab66ba1..939f1ce4f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,13 @@ services: VELA_OTEL_TRACING_ENABLE: true VELA_OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4318 VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100 + VELA_STORAGE_ENABLE: 'true' + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: "http://minio:9000" # Address of the MinIO server + VELA_STORAGE_ACCESS_KEY: minioadmin + VELA_STORAGE_SECRET_KEY: minioadmin + VELA_STORAGE_USE_SSL: 'false' + VELA_STORAGE_BUCKET: vela env_file: - .env restart: always @@ -83,6 +90,11 @@ services: VELA_SERVER_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' WORKER_ADDR: 'http://worker:8080' WORKER_CHECK_IN: 2m + VELA_EXECUTOR_OUTPUTS_IMAGE: 'alpine:latest' + VELA_STORAGE_ENABLE: 'true' + VELA_STORAGE_DRIVER: minio + VELA_STORAGE_ADDRESS: "http://minio:9000" + VELA_STORAGE_BUCKET: vela restart: always ports: - '8081:8080' @@ -177,5 +189,34 @@ services: - '16686:16686' - '4318:4318' + minio: + container_name: minio + image: minio/minio + restart: always + ports: + - "9000:9000" + - "9001:9001" + networks: + - vela + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + - MINIO_SERVER_URL=http://minio + command: minio server --address ":9000" --console-address ":9001" /data + + # The `nginx` compose service hosts the NGINX reverse proxy. + nginx: + image: nginx:latest + container_name: minio_nginx + restart: always + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "80:80" + depends_on: + - minio + networks: + - vela + networks: vela: diff --git a/go.mod b/go.mod index 51afebb41..f61f671ef 100644 --- a/go.mod +++ b/go.mod @@ -30,11 +30,13 @@ require ( github.com/lestrrat-go/jwx/v3 v3.0.0 github.com/lib/pq v1.10.9 github.com/microcosm-cc/bluemonday v1.0.27 + github.com/minio/minio-go/v7 v7.0.83 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.22.0 github.com/redis/go-redis/v9 v9.7.3 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.14.0 + github.com/stretchr/testify v1.10.0 github.com/uptrace/opentelemetry-go-extra/otelgorm v0.3.2 github.com/urfave/cli/v3 v3.1.1 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.60.0 @@ -72,12 +74,15 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.5 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/gin-contrib/sse v1.0.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -106,6 +111,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect @@ -115,6 +121,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -124,9 +131,11 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect diff --git a/go.sum b/go.sum index 73ad53da3..50eb58994 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= @@ -86,6 +88,8 @@ github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -189,6 +193,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -227,6 +232,10 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0+GA= +github.com/minio/minio-go/v7 v7.0.83/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -267,6 +276,8 @@ github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0 github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= diff --git a/internal/metadata.go b/internal/metadata.go index d5e30737d..de9dc9748 100644 --- a/internal/metadata.go +++ b/internal/metadata.go @@ -24,6 +24,12 @@ type ( Host string `json:"host"` } + // Storage is the extra set of Storage data passed to the compiler. + Storage struct { + Driver string `json:"driver"` + Host string `json:"host"` + } + // Vela is the extra set of Vela data passed to the compiler. Vela struct { Address string `json:"address"` @@ -42,5 +48,6 @@ type ( Queue *Queue `json:"queue"` Source *Source `json:"source"` Vela *Vela `json:"vela"` + Storage *Storage `json:"storage"` } ) diff --git a/mock/server/server.go b/mock/server/server.go index 62cdc47fc..493557c29 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -158,5 +158,8 @@ func FakeHandler() http.Handler { // mock endpoint for queue credentials e.GET("/api/v1/queue/info", getQueueCreds) + // mock endpoint for storage credentials + e.GET("/api/v1/storage/info", getStorageCreds) + return e } diff --git a/mock/server/worker.go b/mock/server/worker.go index c92b508ec..4dc4c53e9 100644 --- a/mock/server/worker.go +++ b/mock/server/worker.go @@ -188,6 +188,15 @@ const ( "queue_public_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", "queue_address": "redis://redis:6000" }` + + // StorageInfoResp represents a JSON return for an admin requesting a storage registration info. + // + //not actual credentials. + StorageInfoResp = `{ + "storage_access_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", + "storage_secret_key": "DXeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ98zmko=", + "storage_address": "http://storage:9000" + }` ) // getWorkers returns mock JSON for a http GET. @@ -334,3 +343,24 @@ func getQueueCreds(c *gin.Context) { c.JSON(http.StatusCreated, body) } + +// getStorageCreds returns mock JSON for a http GET. +// Pass "" to Authorization header to test receiving a http 401 response. +func getStorageCreds(c *gin.Context) { + token := c.Request.Header.Get("Authorization") + // verify token if empty + if token == "" { + msg := "unable get storage credentials; invalid registration token" + + c.AbortWithStatusJSON(http.StatusUnauthorized, api.Error{Message: &msg}) + + return + } + + data := []byte(StorageInfoResp) + + var body api.StorageInfo + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusCreated, body) +} diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 000000000..04eac4ae2 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,41 @@ +events {} + +http{ + upstream minio { + least_conn; + server minio:9000; + } + + server { + + listen 80; + listen [::]:80; + server_name minio; + + + # Allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + + proxy_request_buffering off; + + location / { + # Disable buffering + proxy_buffering off; + proxy_pass_request_headers on; + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + # proxy_set_header Connection ""; + chunked_transfer_encoding off; + proxy_set_header Host minio:9000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization $http_authorization; + proxy_pass "http://minio"; # This uses the upstream directive definition to load balance + } + } +} diff --git a/router/admin.go b/router/admin.go index d2eb667ac..5c71d94ae 100644 --- a/router/admin.go +++ b/router/admin.go @@ -25,7 +25,9 @@ import ( // POST /api/v1/admin/workers/:worker/register // GET /api/v1/admin/settings // PUT /api/v1/admin/settings -// DELETE /api/v1/admin/settings. +// DELETE /api/v1/admin/settings +// PUT /api/v1/admin/storage/bucket +// GET /api/v1/admin/storage/presign. func AdminHandlers(base *gin.RouterGroup) { // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) @@ -60,6 +62,12 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin step endpoint _admin.PUT("/step", admin.UpdateStep) + // Admin storage bucket endpoints + _admin.PUT("/storage/bucket", admin.CreateBucket) + + // Admin storage presign endpoints + _admin.GET("/storage/presign", admin.GetPresignedURL) + // Admin user endpoint _admin.PUT("/user", admin.UpdateUser) diff --git a/router/build.go b/router/build.go index 924576763..28808d4c0 100644 --- a/router/build.go +++ b/router/build.go @@ -13,38 +13,7 @@ import ( "github.com/go-vela/server/router/middleware/perm" ) -// BuildHandlers is a function that extends the provided base router group -// with the API handlers for build functionality. -// -// POST /api/v1/repos/:org/:repo/builds -// GET /api/v1/repos/:org/:repo/builds -// POST /api/v1/repos/:org/:repo/builds/:build -// GET /api/v1/repos/:org/:repo/builds/:build -// PUT /api/v1/repos/:org/:repo/builds/:build -// DELETE /api/v1/repos/:org/:repo/builds/:build -// POST /api/v1/repos/:org/:repo/builds/:build/approve -// DELETE /api/v1/repos/:org/:repo/builds/:build/cancel -// GET /api/v1/repos/:org/:repo/builds/:build/logs -// GET /api/v1/repos/:org/:repo/builds/:build/token -// GET /api/v1/repos/:org/:repo/builds/:build/executable -// POST /api/v1/repos/:org/:repo/builds/:build/services -// GET /api/v1/repos/:org/:repo/builds/:build/services -// GET /api/v1/repos/:org/:repo/builds/:build/services/:service -// PUT /api/v1/repos/:org/:repo/builds/:build/services/:service -// DELETE /api/v1/repos/:org/:repo/builds/:build/services/:service -// POST /api/v1/repos/:org/:repo/builds/:build/services/:service/logs -// GET /api/v1/repos/:org/:repo/builds/:build/services/:service/logs -// PUT /api/v1/repos/:org/:repo/builds/:build/services/:service/logs -// DELETE /api/v1/repos/:org/:repo/builds/:build/services/:service/logs -// POST /api/v1/repos/:org/:repo/builds/:build/steps -// GET /api/v1/repos/:org/:repo/builds/:build/steps -// GET /api/v1/repos/:org/:repo/builds/:build/steps/:step -// PUT /api/v1/repos/:org/:repo/builds/:build/steps/:step -// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step -// POST /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs -// GET /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs -// PUT /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs -// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs . +// DELETE /api/v1/repos/:org/:repo/builds/:build/steps/:step/logs. func BuildHandlers(base *gin.RouterGroup) { // Builds endpoints builds := base.Group("/builds") diff --git a/router/middleware/signing.go b/router/middleware/signing.go index 6b9dd4c1f..cfa9218b0 100644 --- a/router/middleware/signing.go +++ b/router/middleware/signing.go @@ -32,3 +32,48 @@ func QueueAddress(address string) gin.HandlerFunc { c.Next() } } + +// StorageAccessKey is a middleware function that attaches the access key used +// to open the connection to the storage. +func StorageAccessKey(key string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("access-key", key) + c.Next() + } +} + +// StorageSecretKey is a middleware function that attaches the secret key used +// to open the connection to the storage. +func StorageSecretKey(key string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("secret-key", key) + c.Next() + } +} + +// StorageAddress is a middleware function that attaches the storage address used +// to open the connection to the storage. +func StorageAddress(address string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-address", address) + c.Next() + } +} + +// StorageBucket is a middleware function that attaches the bucket name used +// to open the connection to the storage. +func StorageBucket(bucket string) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-bucket", bucket) + c.Next() + } +} + +// StorageEnable is a middleware function that sets a flag in the context +// to determined if storage is enabled. +func StorageEnable(enabled bool) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("storage-enable", enabled) + c.Next() + } +} diff --git a/router/middleware/signing_test.go b/router/middleware/signing_test.go index 3f152a3c9..0da89c6ae 100644 --- a/router/middleware/signing_test.go +++ b/router/middleware/signing_test.go @@ -106,3 +106,160 @@ func TestMiddleware_QueueAddress(t *testing.T) { t.Errorf("QueueAddress is %v, want %v", got, want) } } + +func TestMiddleware_StorageAddress(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageAddress(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-address").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageAddress returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageAddress is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageAccessKey(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageAccessKey(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("access-key").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageAccessKey returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageAccessKey is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageSecretKey(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageSecretKey(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("secret-key").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageSecretKey returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageSecretKey is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageBucket(t *testing.T) { + // setup types + got := "" + want := "foobar" + + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(StorageBucket(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-bucket").(string) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageBucket returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageBucket is %v, want %v", got, want) + } +} + +func TestMiddleware_StorageEnable(t *testing.T) { + // setup types + got := false + want := true + // setup context + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + // setup mock server + engine.Use(StorageEnable(want)) + engine.GET("/health", func(c *gin.Context) { + got = c.Value("storage-enable").(bool) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("StorageEnable returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("StorageEnable is %v, want %v", got, want) + } +} diff --git a/router/middleware/storage.go b/router/middleware/storage.go new file mode 100644 index 000000000..c27c1820b --- /dev/null +++ b/router/middleware/storage.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/storage" +) + +// Storage is a middleware function that initializes the object storage and +// attaches to the context of every http.Request. +func Storage(q storage.Storage) gin.HandlerFunc { + return func(c *gin.Context) { + // attach the object storage to the context + storage.ToContext(c, q) + + c.Next() + } +} diff --git a/router/middleware/storage_test.go b/router/middleware/storage_test.go new file mode 100644 index 000000000..017fbef27 --- /dev/null +++ b/router/middleware/storage_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/storage" + "github.com/go-vela/server/storage/minio" +) + +func TestMiddleware_Storage(t *testing.T) { + // setup types + var got storage.Storage + want, _ := minio.NewTest("", "", "", "", false) + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + context, engine := gin.CreateTestContext(resp) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) + + // setup mock server + engine.Use(Storage(want)) + engine.GET("/health", func(c *gin.Context) { + got = storage.FromGinContext(c) + + c.Status(http.StatusOK) + }) + + // run test + engine.ServeHTTP(context.Writer, context.Request) + + if resp.Code != http.StatusOK { + t.Errorf("Storage returned %v, want %v", resp.Code, http.StatusOK) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("Storage is %v, want %v", got, want) + } +} diff --git a/router/router.go b/router/router.go index ca131ae63..1b1fb57d4 100644 --- a/router/router.go +++ b/router/router.go @@ -155,6 +155,9 @@ func Load(options ...gin.HandlerFunc) *gin.Engine { // Queue endpoints QueueHandlers(baseAPI) + + // Storage endpoints + StorageHandlers(baseAPI) } // end of api return r diff --git a/router/storage.go b/router/storage.go new file mode 100644 index 000000000..0082f5055 --- /dev/null +++ b/router/storage.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 + +package router + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/storage" + "github.com/go-vela/server/router/middleware/perm" +) + +// StorageHandlers is a function that extends the provided base router group +// with the API handlers for storage functionality. +// +// GET /api/v1/storage/info . +func StorageHandlers(base *gin.RouterGroup) { + // Storage endpoints + _storage := base.Group("/storage") + { + _storage.GET("/info", perm.MustWorkerRegisterToken(), storage.Info) + _storage.GET("/:bucket/objects", storage.ListObjects) + } // end of storage endpoints +} diff --git a/secret/vault/vault.go b/secret/vault/vault.go index 2b9cb0ddc..cabd40332 100644 --- a/secret/vault/vault.go +++ b/secret/vault/vault.go @@ -126,6 +126,7 @@ func New(opts ...ClientOpt) (*Client, error) { go c.refreshToken() } + //nolint:revive // ignore returning unexported engine return c, nil } diff --git a/storage/context.go b/storage/context.go new file mode 100644 index 000000000..d131287ad --- /dev/null +++ b/storage/context.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + + "github.com/gin-gonic/gin" +) + +// key is the key used to store minio service in context. +const key = "minio" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext retrieves minio service from the context. +func FromContext(ctx context.Context) Storage { + // get minio value from context.Context + v := ctx.Value(key) + if v == nil { + return nil + } + + // cast minio value to expected Storage type + s, ok := v.(Storage) + if !ok { + return nil + } + + return s +} + +// FromGinContext retrieves the S3 Service from the gin.Context. +func FromGinContext(c *gin.Context) Storage { + // get minio value from gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Get + v, ok := c.Get(key) + if !ok { + return nil + } + + // cast minio value to expected Service type + s, ok := v.(Storage) + if !ok { + return nil + } + + return s +} + +// ToContext adds the secret Service to this +// context if it supports the Setter interface. +func ToContext(c Setter, s Storage) { + c.Set(key, s) +} + +// WithContext adds the minio Storage to the context. +func WithContext(ctx context.Context, s Storage) context.Context { + // set the storage Service in the context.Context + // + // https://pkg.go.dev/context?tab=doc#WithValue + // + //nolint:revive // ignore using string with context value + return context.WithValue(ctx, key, s) +} + +// WithGinContext inserts the minio Storage into the gin.Context. +func WithGinContext(c *gin.Context, s Storage) { + // set the minio Storage in the gin.Context + // + // https://pkg.go.dev/github.com/gin-gonic/gin?tab=doc#Context.Set + c.Set(key, s) +} diff --git a/storage/context_test.go b/storage/context_test.go new file mode 100644 index 000000000..a2c7148e5 --- /dev/null +++ b/storage/context_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "reflect" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestExecutor_FromContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + // setup tests + tests := []struct { + context context.Context + want Storage + }{ + { + //nolint:revive // ignore using string with context value + context: context.WithValue(context.Background(), key, _service), + want: _service, + }, + { + context: context.Background(), + want: nil, + }, + { + //nolint:revive // ignore using string with context value + context: context.WithValue(context.Background(), key, "foo"), + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := FromContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_FromGinContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + // setup tests + tests := []struct { + context *gin.Context + value interface{} + want Storage + }{ + { + context: new(gin.Context), + value: _service, + want: _service, + }, + { + context: new(gin.Context), + value: nil, + want: nil, + }, + { + context: new(gin.Context), + value: "foo", + want: nil, + }, + } + + // run tests + for _, test := range tests { + if test.value != nil { + test.context.Set(key, test.value) + } + + got := FromGinContext(test.context) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromGinContext is %v, want %v", got, test.want) + } + } +} + +func TestExecutor_WithContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + //nolint:revive // ignore using string with context value + want := context.WithValue(context.Background(), key, _service) + + // run test + got := WithContext(context.Background(), _service) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithContext is %v, want %v", got, want) + } +} + +func TestExecutor_WithGinContext(t *testing.T) { + // setup types + _service, _ := New(&Setup{}) + + want := new(gin.Context) + want.Set(key, _service) + + // run test + got := new(gin.Context) + WithGinContext(got, _service) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WithGinContext is %v, want %v", got, want) + } +} diff --git a/storage/flags.go b/storage/flags.go new file mode 100644 index 000000000..d14d2d71a --- /dev/null +++ b/storage/flags.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "fmt" + "github.com/urfave/cli/v3" + "strings" +) + +var Flags = []cli.Flag{ + // STORAGE Flags + + &cli.BoolFlag{ + Name: "storage.enable", + Usage: "enable object storage", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ENABLE"), + cli.File("vela/storage/enable"), + ), + }, + &cli.StringFlag{ + Name: "storage.driver", + Usage: "object storage driver", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_DRIVER"), + cli.EnvVar("STORAGE_DRIVER"), + cli.File("vela/storage/driver"), + ), + }, + &cli.StringFlag{ + Name: "storage.addr", + Usage: "set the storage endpoint (ex. scheme://host:port)", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ADDRESS"), + cli.EnvVar("STORAGE_ADDRESS"), + cli.File("vela/storage/addr"), + ), + Action: func(_ context.Context, _ *cli.Command, v string) error { + // check if the queue address has a scheme + if !strings.Contains(v, "://") { + return fmt.Errorf("queue address must be fully qualified (://)") + } + + // check if the queue address has a trailing slash + if strings.HasSuffix(v, "/") { + return fmt.Errorf("queue address must not have trailing slash") + } + + return nil + }, + }, + + &cli.StringFlag{ + Name: "storage.access.key", + Usage: "set storage access key", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_ACCESS_KEY"), + cli.EnvVar("STORAGE_ACCESS_KEY"), + cli.File("vela/storage/access_key"), + ), + }, + &cli.StringFlag{ + Name: "storage.secret.key", + Usage: "set storage secret key", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_SECRET_KEY"), + cli.EnvVar("STORAGE_SECRET_KEY"), + cli.File("vela/storage/secret_key"), + ), + }, + &cli.StringFlag{ + Name: "storage.bucket.name", + Usage: "set storage bucket name", + Sources: cli.NewValueSourceChain( + cli.EnvVar("VELA_STORAGE_BUCKET"), + cli.File("vela/storage/bucket"), + ), + }, + &cli.BoolFlag{ + Name: "storage.use.ssl", + Usage: "enable storage to use SSL", + Value: false, + Sources: cli.EnvVars("VELA_STORAGE_USE_SSL"), + }, +} diff --git a/storage/minio/bucket_exists.go b/storage/minio/bucket_exists.go new file mode 100644 index 000000000..ce1f5fa26 --- /dev/null +++ b/storage/minio/bucket_exists.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + + api "github.com/go-vela/server/api/types" +) + +// BucketExists checks if a bucket exists in MinIO. +func (c *Client) BucketExists(ctx context.Context, bucket *api.Bucket) (bool, error) { + c.Logger.Tracef("checking if bucket %s exists", bucket.BucketName) + + exists, err := c.client.BucketExists(ctx, bucket.BucketName) + if err != nil { + return false, err + } + + return exists, nil +} diff --git a/storage/minio/bucket_exists_test.go b/storage/minio/bucket_exists_test.go new file mode 100644 index 000000000..8d9849e7a --- /dev/null +++ b/storage/minio/bucket_exists_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_BucketExists(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + // mock bucket exists call + engine.HEAD("/foo/", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + ctx := context.TODO() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // create bucket + err := client.CreateBucket(ctx, &api.Bucket{BucketName: "foo"}) + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } + + // run test + exists, err := client.BucketExists(ctx, &api.Bucket{BucketName: "foo"}) + if resp.Code != http.StatusOK { + t.Errorf("BucketExists returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("BucketExists returned err: %v", err) + } + + if !exists { + t.Errorf("BucketExists returned %v, want %v", exists, true) + } +} + +func TestMinioClient_BucketExists_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.HEAD("/foo/", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + ctx := context.TODO() + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // run test + exists, err := client.BucketExists(ctx, &api.Bucket{BucketName: "bar"}) + if resp.Code != http.StatusOK { + t.Errorf("BucketExists returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("BucketExists returned err: %v", err) + } + + if exists { + t.Errorf("BucketExists returned %v, want %v", exists, false) + } +} diff --git a/storage/minio/create_bucket.go b/storage/minio/create_bucket.go new file mode 100644 index 000000000..d41784498 --- /dev/null +++ b/storage/minio/create_bucket.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + + api "github.com/go-vela/server/api/types" +) + +// CreateBucket creates a new bucket in MinIO. +func (c *Client) CreateBucket(ctx context.Context, bucket *api.Bucket) error { + c.Logger.Tracef("create new bucket: %s", bucket.BucketName) + + exists, errBucketExists := c.BucketExists(ctx, bucket) + if errBucketExists != nil && exists { + c.Logger.Tracef("Bucket %s already exists", bucket.BucketName) + + return fmt.Errorf("bucket %s already exists", bucket.BucketName) + } + + err := c.client.MakeBucket(ctx, bucket.BucketName, bucket.MakeBucketOptions) + if err != nil { + c.Logger.Errorf("unable to create bucket %s: %v", bucket.BucketName, err) + return err + } + + return nil +} diff --git a/storage/minio/create_bucket_test.go b/storage/minio/create_bucket_test.go new file mode 100644 index 000000000..83205c509 --- /dev/null +++ b/storage/minio/create_bucket_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_CreateBucket_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // run test + err := client.CreateBucket(ctx, b) + if resp.Code != http.StatusOK { + t.Errorf("CreateBucket returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } +} diff --git a/storage/minio/doc.go b/storage/minio/doc.go new file mode 100644 index 000000000..e1b1afd52 --- /dev/null +++ b/storage/minio/doc.go @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio diff --git a/storage/minio/get_bucket.go b/storage/minio/get_bucket.go new file mode 100644 index 000000000..9a09e055d --- /dev/null +++ b/storage/minio/get_bucket.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" +) + +func (c *Client) GetBucket(context.Context) string { + // GetBucket returns the bucket name for the MinIO client. + return c.config.Bucket +} diff --git a/storage/minio/list_bucket.go b/storage/minio/list_bucket.go new file mode 100644 index 000000000..8547a7af0 --- /dev/null +++ b/storage/minio/list_bucket.go @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" +) + +// ListBuckets lists all buckets in MinIO. +func (c *Client) ListBuckets(ctx context.Context) ([]string, error) { + c.Logger.Trace("listing all buckets") + + buckets, err := c.client.ListBuckets(ctx) + if err != nil { + return nil, err + } + + bucketNames := make([]string, len(buckets)) + for i, bucket := range buckets { + bucketNames[i] = bucket.Name + } + + return bucketNames, nil +} diff --git a/storage/minio/list_bucket_test.go b/storage/minio/list_bucket_test.go new file mode 100644 index 000000000..debf53a58 --- /dev/null +++ b/storage/minio/list_bucket_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_ListBuckets_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock list buckets call + engine.GET("/", func(c *gin.Context) { + c.Header("X-Meta-BucketName", "foo") + c.XML(200, gin.H{ + "Buckets": []minio.BucketInfo{ + { + Name: "foo", + CreationDate: time.Now(), + }, + }, + }) + }) + fake := httptest.NewServer(engine) + defer fake.Close() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + b := new(api.Bucket) + b.BucketName = "foo" + + _, err := client.ListBuckets(ctx) + if err != nil { + t.Errorf("ListBuckets returned err: %v", err) + } + + // Ignore for now as xmlDecoder from minio-go is does not parse correctly with sample data + // check if buckets are correct + //expectedBuckets := []string{"foo"} + //if len(buckets) != len(expectedBuckets) { + // t.Errorf("Expected %d buckets, got %d", len(expectedBuckets), len(buckets)) + //} + //for i, bucket := range buckets { + // if bucket != expectedBuckets[i] { + // t.Errorf("Expected bucket %v, got %v", expectedBuckets[i], bucket) + // } + //} +} + +func TestMinioClient_ListBuckets_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // mock list buckets call + engine.GET("/minio/buckets", func(c *gin.Context) { + c.Status(http.StatusInternalServerError) + }) + fake := httptest.NewServer(engine) + defer fake.Close() + ctx := context.TODO() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // run test + _, err := client.ListBuckets(ctx) + if err == nil { + t.Errorf("Expected error, got nil") + } +} diff --git a/storage/minio/list_objects.go b/storage/minio/list_objects.go new file mode 100644 index 000000000..1360ff15d --- /dev/null +++ b/storage/minio/list_objects.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// ListObjects lists the objects in a bucket. +func (c *Client) ListObjects(ctx context.Context, b *api.Bucket) ([]minio.ObjectInfo, error) { + c.Logger.Tracef("listing objects in bucket %s", b.BucketName) + + objectCh := c.client.ListObjects(ctx, b.BucketName, minio.ListObjectsOptions{}) + + var objects []minio.ObjectInfo + + for object := range objectCh { + if object.Err != nil { + return nil, object.Err + } + + objects = append(objects, object) + } + + return objects, nil +} diff --git a/storage/minio/list_objects_test.go b/storage/minio/list_objects_test.go new file mode 100644 index 000000000..d75ef0b99 --- /dev/null +++ b/storage/minio/list_objects_test.go @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_ListObjects_Success(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + engine.GET("/foo/", func(c *gin.Context) { + objects := []gin.H{ + {"etag": "982beba05db8083656a03f544c8c7927", + "name": "test.xml", + "lastModified": "2025-03-20T19:01:40.968Z", + "size": 558677, + "contentType": "", + "expires": "0001-01-01T00:00:00Z", + "metadata": "null", + "UserTagCount": 0, + "Owner": gin.H{ + "owner": gin.H{ + "Space": "http://s3.amazonaws.com/doc/2006-03-01/", + "Local": "Owner", + }, + "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "id": "minio", + }, + "Grant": "null", + "storageClass": "STANDARD", + "IsLatest": false, + "IsDeleteMarker": false, + "VersionID": "", + "ReplicationStatus": "", + "ReplicationReady": false, + "Expiration": "0001-01-01T00:00:00Z", + "ExpirationRuleID": "", + "Restore": "null", + "ChecksumCRC32": "", + "ChecksumCRC32C": "", + "ChecksumSHA1": "", + "ChecksumSHA256": "", + "ChecksumCRC64NVME": "", + "Internal": "null"}, + } + c.Stream(func(w io.Writer) bool { + _, err := w.Write([]byte(objects[0]["name"].(string))) + if err != nil { + return false + } + c.XML(http.StatusOK, objects) + c.Status(http.StatusOK) + return false + }) + }) + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Errorf("Failed to create MinIO client: %v", err) + } + + // For now, passing if listing objects returns no error + _, err = client.ListObjects(ctx, b) + if err != nil { + t.Errorf("ListObject returned err: %v", err) + } + + // + //expected := "test.xml" + //found := false + //for _, result := range results { + // if result.Key == expected { + // found = true + // } + //} + //if !found { + // t.Errorf("Object %v not found in list %v", expected, results) + //} +} + +func TestMinioClient_ListObjects_Failure(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.XML(http.StatusOK, gin.H{ + "bucketName": "foo", + "bucketLocation": "snowball", + "objectName": "test.xml", + }) + }) + + engine.GET("/foo/", func(c *gin.Context) { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Internal Server Error"}) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + + b := new(api.Bucket) + b.BucketName = "foo" + + client, err := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + if err != nil { + t.Errorf("Failed to create MinIO client: %v", err) + } + + // run test + _, err = client.ListObjects(ctx, b) + if err == nil { + t.Errorf("ListObject should have returned an error") + } +} diff --git a/storage/minio/minio.go b/storage/minio/minio.go new file mode 100644 index 000000000..461ab9b20 --- /dev/null +++ b/storage/minio/minio.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "fmt" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/sirupsen/logrus" +) + +// config holds the configuration for the MinIO client. +type config struct { + Endpoint string + AccessKey string + SecretKey string + Bucket string + Secure bool +} + +// Client implements the Storage interface using MinIO. +type Client struct { + config *config + client *minio.Client + Options *minio.Options + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + Logger *logrus.Entry +} + +// New creates a new MinIO client. +func New(endpoint string, opts ...ClientOpt) (*Client, error) { + // create new Minio client + c := new(Client) + + // default to secure connection + var urlEndpoint string + + // create new fields + c.config = new(config) + c.Options = new(minio.Options) + + // create new logger for the client + logger := logrus.StandardLogger() + c.Logger = logrus.NewEntry(logger).WithField("minio", "minio") + + // apply all provided configuration options + for _, opt := range opts { + err := opt(c) + if err != nil { + return nil, err + } + } + + c.Options.Creds = credentials.NewStaticV4(c.config.AccessKey, c.config.SecretKey, "") + c.Options.Secure = c.config.Secure + logrus.Debugf("secure: %v", c.config.Secure) + + if len(endpoint) > 0 { + useSSL := strings.HasPrefix(endpoint, "https://") + + if !useSSL { + if !strings.HasPrefix(endpoint, "http://") { + return nil, fmt.Errorf("invalid server %s: must to be a HTTP URI", endpoint) + } + + urlEndpoint = endpoint[7:] + } else { + urlEndpoint = endpoint[8:] + } + } + + // create the Minio client from the provided endpoint and options + minioClient, err := minio.New(urlEndpoint, c.Options) + if err != nil { + return nil, err + } + + c.client = minioClient + + return c, nil +} + +// NewTest returns a Storage implementation that +// integrates with a local MinIO instance. +// +// This function is intended for running tests only. +// + +func NewTest(endpoint, accessKey, secretKey, bucket string, secure bool) (*Client, error) { + return New(endpoint, WithAccessKey(accessKey), WithSecretKey(secretKey), WithSecure(secure), WithBucket(bucket)) +} diff --git a/storage/minio/minio_test.go b/storage/minio/minio_test.go new file mode 100644 index 000000000..bc4064796 --- /dev/null +++ b/storage/minio/minio_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "testing" +) + +var ( + endpoint = "http://localhost:9000" + _accessKey = "minio_access_user" + _secretKey = "minio_secret_key" + _bucket = "minio_bucket" + _useSSL = false +) + +func TestMinio_New(t *testing.T) { + // setup types + // create a local fake MinIO instance + // + // https://pkg.go.dev/github.com/minio/minio-go/v7#New + // setup tests + tests := []struct { + failure bool + endpoint string + }{ + { + failure: false, + endpoint: endpoint, + }, + { + failure: true, + endpoint: "", + }, + } + + // run tests + for _, test := range tests { + _, err := New( + test.endpoint, + WithAccessKey(_accessKey), + WithSecretKey(_secretKey), + WithSecure(_useSSL), + WithBucket(_bucket), + ) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + } +} diff --git a/storage/minio/opts.go b/storage/minio/opts.go new file mode 100644 index 000000000..d45f1227f --- /dev/null +++ b/storage/minio/opts.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "fmt" +) + +// ClientOpt represents a configuration option to initialize the MinIO client. +type ClientOpt func(client *Client) error + +// WithAccessKey sets the access key in the MinIO client. +func WithAccessKey(accessKey string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring access key in minio client") + + // check if the access key provided is empty + if len(accessKey) == 0 { + return fmt.Errorf("no MinIO access key provided") + } + + // set the access key in the minio client + c.config.AccessKey = accessKey + + return nil + } +} + +// WithSecretKey sets the secret key in the MinIO client. +func WithSecretKey(secretKey string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring secret key in minio client") + + // check if the secret key provided is empty + if len(secretKey) == 0 { + return fmt.Errorf("no MinIO secret key provided") + } + + // set the secret key in the minio client + c.config.SecretKey = secretKey + + return nil + } +} + +// WithSecure sets the secure connection mode in the MinIO client. +func WithSecure(secure bool) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring secure connection mode in minio client") + + // set the secure connection mode in the minio client + c.config.Secure = secure + + return nil + } +} + +// WithBucket sets the bucket name in the MinIO client. +func WithBucket(bucket string) ClientOpt { + return func(c *Client) error { + c.Logger.Trace("configuring bucket name in minio client") + + // check if the bucket name provided is empty + if len(bucket) == 0 { + return fmt.Errorf("no MinIO bucket name provided") + } + + // set the bucket name in the minio client + c.config.Bucket = bucket + + return nil + } +} diff --git a/storage/minio/opts_test.go b/storage/minio/opts_test.go new file mode 100644 index 000000000..c75fa9828 --- /dev/null +++ b/storage/minio/opts_test.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "reflect" + "testing" +) + +func TestWithAccessKey(t *testing.T) { + // setup tests + tests := []struct { + failure bool + accessKey string + want string + }{ + { + failure: false, + accessKey: "validAccessKey", + want: "validAccessKey", + }, + { + failure: true, + accessKey: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + test.accessKey, + "miniosecret", + "foo", + false) + + if test.failure { + if err == nil { + t.Errorf("WithAddress should have returned err") + } + + continue + } + if err != nil && test.accessKey != "" { + t.Errorf("WithAccessKey returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.AccessKey, test.want) { + t.Errorf("WithAccessKey is %v, want %v", client.config.AccessKey, test.want) + } + } +} + +func TestWithSecretKey(t *testing.T) { + // setup tests + tests := []struct { + failure bool + secretKey string + want string + }{ + { + failure: false, + secretKey: "validSecretKey", + want: "validSecretKey", + }, + { + failure: true, + secretKey: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + test.secretKey, + "foo", + false) + + if test.failure { + if err == nil { + t.Errorf("WithSecretKey should have returned err") + } + + continue + } + if err != nil && test.secretKey != "" { + t.Errorf("WithSecretKey returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.SecretKey, test.want) { + t.Errorf("WithSecretKey is %v, want %v", client.config.SecretKey, test.want) + } + } +} + +func TestWithSecure(t *testing.T) { + // setup tests + tests := []struct { + failure bool + secure bool + want bool + }{ + { + failure: false, + secure: true, + want: true, + }, + { + failure: false, + secure: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + "miniosecret", + "foo", + test.secure) + + if test.failure { + if err == nil { + t.Errorf("WithSecure should have returned err") + } + + continue + } + if err != nil { + t.Errorf("WithSecure returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.Secure, test.want) { + t.Errorf("WithSecure is %v, want %v", client.config.Secure, test.want) + } + } +} + +func TestWithBucket(t *testing.T) { + // setup tests + tests := []struct { + failure bool + bucket string + want string + }{ + { + failure: false, + bucket: "validBucket", + want: "validBucket", + }, + { + failure: true, + bucket: "", + want: "", + }, + } + + // run tests + for _, test := range tests { + client, err := NewTest("https://minio.example.com", + "minioaccess", + "miniosecret", + test.bucket, + false) + + if test.failure { + if err == nil { + t.Errorf("WithBucket should have returned err") + } + + continue + } + if err != nil && test.bucket != "" { + t.Errorf("WithBucket returned err: %v", err) + } + + if !reflect.DeepEqual(client.config.Bucket, test.want) { + t.Errorf("WithBucket is %v, want %v", client.config.Bucket, test.want) + } + } +} diff --git a/storage/minio/presigned_get_object.go b/storage/minio/presigned_get_object.go new file mode 100644 index 000000000..58879658c --- /dev/null +++ b/storage/minio/presigned_get_object.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "strings" + "time" + + "github.com/minio/minio-go/v7" + "github.com/sirupsen/logrus" + + api "github.com/go-vela/server/api/types" +) + +// TODO hide URL behind a different name +// PresignedGetObject generates a presigned URL for downloading an object. +func (c *Client) PresignedGetObject(ctx context.Context, object *api.Object) (string, error) { + c.Logger.Tracef("generating presigned URL for object %s in bucket %s", object.ObjectName, object.Bucket.BucketName) + + var url string + // collect metadata on the object + // make sure the object exists before generating the presigned URL + objInfo, err := c.client.StatObject(ctx, object.Bucket.BucketName, object.ObjectName, minio.StatObjectOptions{}) + if objInfo.Key == "" { + logrus.Errorf("unable to get object info %s from bucket %s: %v", object.ObjectName, object.Bucket.BucketName, err) + return "", err + } + + // Generate presigned URL for downloading the object. + // The URL is valid for 7 days. + presignedURL, err := c.client.PresignedGetObject(ctx, object.Bucket.BucketName, object.ObjectName, 7*24*time.Hour, nil) + if err != nil { + return "", err + } + + url = presignedURL.String() + + // replace minio:9000 with minio + // for local development + if strings.Contains(url, "minio:9000") { + // replace with minio:9002 + url = strings.Replace(url, "minio:9000", "minio", 1) + } + + return url, nil +} diff --git a/storage/minio/presigned_get_object_test.go b/storage/minio/presigned_get_object_test.go new file mode 100644 index 000000000..78008193b --- /dev/null +++ b/storage/minio/presigned_get_object_test.go @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func Test_PresignedGetObject_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "name": "test.xml", + }) + }) + // mock presigned get object call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(200, gin.H{ + "bucketName": "foo", + }) + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + ctx := context.TODO() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + url, err := client.PresignedGetObject(ctx, object) + if err != nil { + t.Errorf("PresignedGetObject returned err: %v", err) + } + + // check if URL is valid + if url == "" { + t.Errorf("PresignedGetObject returned empty URL") + } +} + +func Test_PresignedGetObject_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock presigned get object call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(500, gin.H{ + "error": "Internal Server Error", + }) + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + url, err := client.PresignedGetObject(ctx, object) + if err == nil { + t.Errorf("PresignedGetObject expected error but got none") + } + + if url != "" { + t.Errorf("PresignedGetObject returned URL when it should have failed") + } +} diff --git a/storage/minio/stat_object.go b/storage/minio/stat_object.go new file mode 100644 index 000000000..966a17699 --- /dev/null +++ b/storage/minio/stat_object.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "fmt" + + "github.com/minio/minio-go/v7" + + "github.com/go-vela/server/api/types" +) + +// StatObject retrieves the metadata of an object from the MinIO storage. +func (c *Client) StatObject(ctx context.Context, object *types.Object) (*types.Object, error) { + c.Logger.Tracef("retrieving metadata for object %s from bucket %s", object.ObjectName, object.Bucket.BucketName) + + // Get object info + info, err := c.client.StatObject(ctx, object.Bucket.BucketName, object.ObjectName, minio.StatObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("unable to get object info %s from bucket %s: %w", object.ObjectName, object.Bucket.BucketName, err) + } + + // Map MinIO object info to API object + return &types.Object{ + ObjectName: info.Key, + }, nil +} diff --git a/storage/minio/stat_object_test.go b/storage/minio/stat_object_test.go new file mode 100644 index 000000000..6c98734c1 --- /dev/null +++ b/storage/minio/stat_object_test.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + + api "github.com/go-vela/server/api/types" +) + +func Test_StatObject_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock create bucket call + engine.GET("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(200, gin.H{ + "Buckets": []minio.BucketInfo{ + { + Name: "foo", + }, + }, + }) + }) + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.Header("Last-Modified", "Mon, 2 Jan 2006 15:04:05 GMT") + c.XML(200, gin.H{ + "etag": "982beba05db8083656a03f544c8c7927", + "name": "test.xml", + "lastModified": "2025-03-20T19:01:40.968Z", + "size": 558677, + "contentType": "", + "expires": time.Now(), + "metadata": "null", + "UserTagCount": 0, + "Owner": gin.H{ + "owner": gin.H{ + "Space": "http://s3.amazonaws.com/doc/2006-03-01/", + "Local": "Owner", + }, + "name": "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4", + "id": "minio", + }, + "Grant": "null", + "storageClass": "STANDARD", + "IsLatest": false, + "IsDeleteMarker": false, + "VersionID": "", + "ReplicationStatus": "", + "ReplicationReady": false, + "Expiration": time.Now(), + "ExpirationRuleID": "", + "Restore": "null", + "ChecksumCRC32": "", + "ChecksumCRC32C": "", + "ChecksumSHA1": "", + "ChecksumSHA256": "", + "ChecksumCRC64NVME": "", + "Internal": "null", + }) + c.Status(http.StatusOK) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + result, err := client.StatObject(ctx, object) + assert.NoError(t, err) + assert.Equal(t, "test.xml", result.ObjectName) +} + +func Test_StatObject_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + resp := httptest.NewRecorder() + ctx, engine := gin.CreateTestContext(resp) + + // mock stat object call + engine.HEAD("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/xml") + c.XML(500, gin.H{ + "error": "Internal Server Error", + }) + c.Status(http.StatusInternalServerError) + }) + + fake := httptest.NewServer(engine) + defer fake.Close() + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + object := &api.Object{ + ObjectName: "test.xml", + Bucket: api.Bucket{ + BucketName: "foo", + }, + } + + // run test + result, err := client.StatObject(ctx, object) + assert.Error(t, err) + assert.Nil(t, result) +} diff --git a/storage/minio/test_data/create_bucket.json b/storage/minio/test_data/create_bucket.json new file mode 100644 index 000000000..b1e77da2f --- /dev/null +++ b/storage/minio/test_data/create_bucket.json @@ -0,0 +1,3 @@ +{ + "bucket_name": "foo" +} \ No newline at end of file diff --git a/storage/minio/test_data/test.xml b/storage/minio/test_data/test.xml new file mode 100644 index 000000000..66c653d49 --- /dev/null +++ b/storage/minio/test_data/test.xml @@ -0,0 +1,7 @@ + + + TEST + Upload + Reminder + Please upload me! + \ No newline at end of file diff --git a/storage/minio/upload.go b/storage/minio/upload.go new file mode 100644 index 000000000..9f986e6e7 --- /dev/null +++ b/storage/minio/upload.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "io" + "mime" + "path/filepath" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// Upload uploads an object to a bucket in MinIO.ts. +func (c *Client) Upload(ctx context.Context, object *api.Object) error { + c.Logger.Tracef("uploading data to bucket %s", object.Bucket.BucketName) + _, err := c.client.FPutObject(ctx, object.Bucket.BucketName, object.ObjectName, object.FilePath, minio.PutObjectOptions{}) + + return err +} + +// UploadObject uploads an object to a bucket in MinIO.ts. +func (c *Client) UploadObject(ctx context.Context, object *api.Object, reader io.Reader, size int64) error { + c.Logger.Infof("uploading data to bucket %s", object.Bucket.BucketName) + ext := filepath.Ext(object.FilePath) + contentType := mime.TypeByExtension(ext) + + c.Logger.Infof("uploading object %s with content type %s", object.ObjectName, contentType) + // TODO - better way to get bucket name + info, err := c.client.PutObject(ctx, object.Bucket.BucketName, object.ObjectName, reader, size, + minio.PutObjectOptions{ContentType: contentType}) + if err != nil { + c.Logger.Errorf("unable to upload object %s: %v", object.ObjectName, err) + return err + } + + c.Logger.Infof("uploaded object %v with size %d", info, info.Size) + + return nil +} diff --git a/storage/minio/upload_test.go b/storage/minio/upload_test.go new file mode 100644 index 000000000..2b1b2cc32 --- /dev/null +++ b/storage/minio/upload_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: Apache-2.0 + +package minio + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" +) + +func TestMinioClient_Upload_Success(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + // mock upload call + engine.PUT("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("test_data/test.xml") + }) + fake := httptest.NewServer(engine) + defer fake.Close() + ctx := context.TODO() + obj := new(api.Object) + obj.Bucket.BucketName = "foo" + obj.ObjectName = "test.xml" + obj.FilePath = "test_data/test.xml" + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // create bucket + err := client.CreateBucket(ctx, &api.Bucket{BucketName: "foo"}) + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } + + // run test + err = client.Upload(ctx, obj) + if resp.Code != http.StatusOK { + t.Errorf("Upload returned %v, want %v", resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("Upload returned err: %v", err) + } +} + +func TestMinioClient_Upload_Failure(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + // mock create bucket call + engine.PUT("/foo/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + // mock bucket exists call + engine.PUT("/foo/test.xml", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("test_data/test.xml") + }) + fake := httptest.NewServer(engine) + defer fake.Close() + ctx := context.TODO() + obj := new(api.Object) + obj.Bucket.BucketName = "foo" + obj.ObjectName = "test.xml" + obj.FilePath = "nonexist/test.xml" + client, _ := NewTest(fake.URL, "miniokey", "miniosecret", "foo", false) + + // create bucket + err := client.CreateBucket(ctx, &api.Bucket{BucketName: "foo"}) + if err != nil { + t.Errorf("CreateBucket returned err: %v", err) + } + + // run test + err = client.Upload(ctx, obj) + if resp.Code != http.StatusOK { + t.Errorf("Upload returned %v, want %v", resp.Code, http.StatusOK) + } + + if !os.IsNotExist(err) { + t.Errorf("Upload returned err: %v", err) + } +} diff --git a/storage/service.go b/storage/service.go new file mode 100644 index 000000000..a5da86140 --- /dev/null +++ b/storage/service.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "io" + + "github.com/minio/minio-go/v7" + + api "github.com/go-vela/server/api/types" +) + +// Storage defines the service interface for object storage operations. +type Storage interface { + // Bucket Management + CreateBucket(ctx context.Context, bucket *api.Bucket) error + BucketExists(ctx context.Context, bucket *api.Bucket) (bool, error) + ListBuckets(ctx context.Context) ([]string, error) + GetBucket(ctx context.Context) string + // Object Operations + StatObject(ctx context.Context, object *api.Object) (*api.Object, error) + Upload(ctx context.Context, object *api.Object) error + UploadObject(ctx context.Context, object *api.Object, reader io.Reader, size int64) error + //Download(ctx context.Context, object *api.Object) error + ListObjects(ctx context.Context, bucket *api.Bucket) ([]minio.ObjectInfo, error) + // Presigned URLs + PresignedGetObject(ctx context.Context, object *api.Object) (string, error) +} diff --git a/storage/setup.go b/storage/setup.go new file mode 100644 index 000000000..88e91df6d --- /dev/null +++ b/storage/setup.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "fmt" + + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/storage/minio" +) + +// Setup represents the configuration necessary for +// creating a Vela service capable of integrating +// with a configured S3 environment. +type Setup struct { + Enable bool + Driver string + Endpoint string + AccessKey string + SecretKey string + Bucket string + Region string + Secure bool +} + +// Minio creates and returns a Vela service capable +// of integrating with an S3 environment. +func (s *Setup) Minio() (Storage, error) { + //client, err := minio.New(s.Endpoint, &minio.MakeBucketOptions{ + // Creds: credentials.NewStaticV4(s.AccessKey, s.SecretKey, ""), + // Secure: s.Secure, + //}) + //if err != nil { + // return nil, err + //} + return minio.New( + s.Endpoint, + minio.WithAccessKey(s.AccessKey), + minio.WithSecretKey(s.SecretKey), + minio.WithSecure(s.Secure), + minio.WithBucket(s.Bucket), + ) +} + +// Validate verifies the necessary fields for the +// provided configuration are populated correctly. +func (s *Setup) Validate() error { + logrus.Trace("validating Storage setup for client") + + // verify storage is enabled + //if !s.Enable { + // return fmt.Errorf("Storage is not enabled") + //} + + // verify an endpoint was provided + if len(s.Endpoint) == 0 { + return fmt.Errorf("no storage endpoint provided") + } + + // verify an access key was provided + if len(s.AccessKey) == 0 { + return fmt.Errorf("no storage access key provided") + } + + // verify a secret key was provided + if len(s.SecretKey) == 0 { + return fmt.Errorf("no storage secret key provided") + } + + // verify a bucket was provided + //if len(s.Bucket) == 0 { + // return fmt.Errorf("no storage bucket provided") + //} + + // setup is valid + return nil +} diff --git a/storage/setup_test.go b/storage/setup_test.go new file mode 100644 index 000000000..ba5aff192 --- /dev/null +++ b/storage/setup_test.go @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSetup_Minio(t *testing.T) { + setup := &Setup{ + Enable: true, + Driver: "minio", + Endpoint: "http://minio.example.com", + AccessKey: "access-key", + SecretKey: "secret-key", + Bucket: "bucket-name", + Secure: true, + } + + storage, err := setup.Minio() + assert.NoError(t, err) + assert.NotNil(t, storage) +} + +func TestSetup_Validate(t *testing.T) { + tests := []struct { + failure bool + setup *Setup + }{ + { + failure: false, + setup: &Setup{ + Enable: true, + Endpoint: "example.com", + AccessKey: "access-key", + SecretKey: "secret-key", + Bucket: "bucket-name", + }, + }, + { + failure: true, + setup: &Setup{ + Enable: false, + }, + }, + { + failure: true, + setup: &Setup{ + AccessKey: "access-key", + SecretKey: "secret-key", + Bucket: "bucket-name", + }, + }, + { + failure: true, + setup: &Setup{ + Enable: true, + Endpoint: "example.com", + SecretKey: "secret-key", + Bucket: "bucket-name", + }, + }, + { + failure: true, + setup: &Setup{ + Enable: true, + Endpoint: "example.com", + AccessKey: "access-key", + Bucket: "bucket-name", + }, + }, + } + + for _, test := range tests { + err := test.setup.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + } +} diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 000000000..452fee50c --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + + "github.com/go-vela/server/constants" +) + +// FromCLICommand helper function to setup Minio Client from the CLI arguments. +func FromCLICommand(ctx context.Context, c *cli.Command) (Storage, error) { + // S3 configuration + _setup := &Setup{ + Enable: c.Bool("storage.enable"), + Driver: c.String("storage.driver"), + Endpoint: c.String("storage.addr"), + AccessKey: c.String("storage.access.key"), + SecretKey: c.String("storage.secret.key"), + Bucket: c.String("storage.bucket.name"), + Secure: c.Bool("storage.use.ssl"), + } + + return New(_setup) +} + +// New creates and returns a Vela service capable of +// integrating with the configured storage environment. +// Currently, the following storages are supported: +// +// * minio +// . +func New(s *Setup) (Storage, error) { + // validate the setup being provided + // + err := s.Validate() + if err != nil { + return nil, fmt.Errorf("unable to validate storage setup: %w", err) + } + + logrus.Debug("creating storage client from setup") + // process the storage driver being provided + switch s.Driver { + case constants.DriverMinio: + // handle the Kafka queue driver being provided + // + // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#Setup.Kafka + return s.Minio() + default: + // handle an invalid queue driver being provided + return nil, fmt.Errorf("invalid storage driver provided: %s", s.Driver) + } +} diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 000000000..970730d36 --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + "github.com/go-vela/server/constants" +) + +func TestStorage_New(t *testing.T) { + tests := []struct { + failure bool + setup *Setup + }{ + { + failure: false, + setup: &Setup{ + Driver: constants.DriverMinio, + Enable: true, + Endpoint: "http://minio.example.com", + AccessKey: "access-key", + SecretKey: "secret-key", + Bucket: "bucket-name", + Secure: true, + }, + }, + { + failure: true, + setup: &Setup{ + Driver: "invalid-driver", + Enable: false, + Endpoint: "http://invalid.example.com", + AccessKey: "access-key", + SecretKey: "secret-key", + Bucket: "bucket-name", + Secure: true, + }, + }, + } + + for _, test := range tests { + _, err := New(test.setup) + + if test.failure { + if err == nil { + t.Errorf("New should have returned err") + } + continue + } + + if err != nil { + t.Errorf("New returned err: %v", err) + } + } +}