Merge pull request #4514 from grafana/secretSource

Secret Source Implementation

Author: mstoykov

cmd/state/state.go

@@ -14,7 +14,9 @@ import (
 
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/ui/console"
+	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/lib/fsext"
+	"go.k6.io/k6/secretsource"
 )
 
 const defaultConfigFileName = "config.json"
@@ -55,6 +57,9 @@ type GlobalState struct {
 
 	Logger         *logrus.Logger //nolint:forbidigo //TODO:change to FieldLogger
 	FallbackLogger logrus.FieldLogger
+
+	SecretsManager *secretsource.Manager
+	Usage          *usage.Usage
 }
 
 // NewGlobalState returns a new GlobalState with the given ctx.
@@ -131,6 +136,7 @@ func NewGlobalState(ctx context.Context) *GlobalState {
 			Hooks:     make(logrus.LevelHooks),
 			Level:     logrus.InfoLevel,
 		},
+		Usage: usage.New(),
 	}
 }
 
@@ -142,6 +148,7 @@ type GlobalFlags struct {
 	Address          string
 	ProfilingEnabled bool
 	LogOutput        string
+	SecretSource     []string
 	LogFormat        string
 	Verbose          bool
 }

examples/secrets/file.secret

@@ -0,0 +1,2 @@
+cool=some
+else=source

examples/secrets/secrets.test.js

@@ -0,0 +1,9 @@
+// k6 run --secret-source=file=file.secret secrets.test.js
+import secrets from "k6/secrets";
+
+export default async () => {
+	const my_secret = await secrets.get("cool"); // get secret from a source with the provided identifier
+	console.log(my_secret);
+	await secrets.get("else"); // get secret from a source with the provided identifier
+	console.log(my_secret);
+}

ext/ext.go

@@ -25,6 +25,7 @@ type ExtensionType uint8
 const (
 	JSExtension ExtensionType = iota + 1
 	OutputExtension
+	SecretSourceExtension
 )
 
 func (e ExtensionType) String() string {
@@ -34,6 +35,8 @@ func (e ExtensionType) String() string {
 		s = "js"
 	case OutputExtension:
 		s = "output"
+	case SecretSourceExtension:
+		s = "secret-source"
 	}
 	return s
 }
@@ -157,4 +160,5 @@ func extractModuleInfo(mod interface{}) (path, version string) {
 func init() {
 	extensions[JSExtension] = make(map[string]*Extension)
 	extensions[OutputExtension] = make(map[string]*Extension)
+	extensions[SecretSourceExtension] = make(map[string]*Extension)
 }

internal/cmd/root.go

@@ -19,7 +19,11 @@ import (
 	"go.k6.io/k6/cmd/state"
 	"go.k6.io/k6/errext"
 	"go.k6.io/k6/errext/exitcodes"
+	"go.k6.io/k6/ext"
 	"go.k6.io/k6/internal/log"
+	"go.k6.io/k6/secretsource"
+
+	_ "go.k6.io/k6/internal/secretsource" // import it to register internal secret sources
 )
 
 const waitLoggerCloseTimeout = time.Second * 5
@@ -162,6 +166,10 @@ func rootCmdPersistentFlagSet(gs *state.GlobalState) *pflag.FlagSet {
 	// `gs.DefaultFlags.<value>`, so that the `k6 --help` message is
 	// not messed up...
 
+	// TODO(@mstoykov): likely needs work - no env variables and such. No config.json.
+	flags.StringArrayVar(&gs.Flags.SecretSource, "secret-source", gs.Flags.SecretSource,
+		"setting secret sources for k6 file[=./path.fileformat],")
+
 	flags.StringVar(&gs.Flags.LogOutput, "log-output", gs.Flags.LogOutput,
 		"change the output for k6 logs, possible values are stderr,stdout,none,loki[=host:port],file[=./path.fileformat]")
 	flags.Lookup("log-output").DefValue = gs.DefaultFlags.LogOutput
@@ -257,6 +265,22 @@ func (c *rootCommand) setupLoggers(stop <-chan struct{}) error {
 		c.globalState.Logger.Debug("Logger format: TEXT")
 	}
 
+	secretsources, err := createSecretSources(c.globalState)
+	if err != nil {
+		return err
+	}
+	// it is important that we add this hook first as hooks are executed in order of addition
+	// and this means no other hook will get secrets
+	var secretsHook logrus.Hook
+	c.globalState.SecretsManager, secretsHook, err = secretsource.NewManager(secretsources)
+	if err != nil {
+		return err
+	}
+	if len(secretsources) != 0 {
+		// don't actually filter anything if there will be no secrets
+		c.globalState.Logger.AddHook(secretsHook)
+	}
+
 	cancel := func() {} // noop as default
 	if hook != nil {
 		ctx := context.Background()
@@ -289,3 +313,71 @@ func (c *rootCommand) setLoggerHook(ctx context.Context, h log.AsyncHook) {
 	c.globalState.Logger.AddHook(h)
 	c.globalState.Logger.SetOutput(io.Discard) // don't output to anywhere else
 }
+
+func createSecretSources(gs *state.GlobalState) (map[string]secretsource.Source, error) {
+	baseParams := secretsource.Params{
+		Logger:      gs.Logger,
+		Environment: gs.Env,
+		FS:          gs.FS,
+		Usage:       gs.Usage,
+	}
+
+	result := make(map[string]secretsource.Source)
+	for _, line := range gs.Flags.SecretSource {
+		t, config, ok := strings.Cut(line, "=")
+		if !ok {
+			return nil, fmt.Errorf("couldn't parse secret source configuration %q", line)
+		}
+		secretSources := ext.Get(ext.SecretSourceExtension)
+		found, ok := secretSources[t]
+		if !ok {
+			return nil, fmt.Errorf("no secret source for type %q for configuration %q", t, line)
+		}
+		c := found.Module.(secretsource.Constructor) //nolint:forcetypeassert
+		params := baseParams
+		name, isDefault, config := extractNameAndDefault(config)
+		params.ConfigArgument = config
+
+		secretSource, err := c(params)
+		if err != nil {
+			return nil, err
+		}
+		_, alreadRegistered := result[name]
+		if alreadRegistered {
+			return nil, fmt.Errorf("secret source for name %q already registered before configuration %q", t, line)
+		}
+		result[name] = secretSource
+		if isDefault {
+			if _, ok := result["default"]; ok {
+				return nil, fmt.Errorf("can't have two secret sources that are default ones, second one was %q", config)
+			}
+			result["default"] = secretSource
+		}
+	}
+
+	if len(result) == 1 {
+		for _, l := range result {
+			result["default"] = l
+		}
+	}
+
+	return result, nil
+}
+
+func extractNameAndDefault(config string) (name string, isDefault bool, remaining string) {
+	list := strings.Split(config, ",")
+	remainingArray := make([]string, 0, len(list))
+	for _, kv := range list {
+		if kv == "default" {
+			isDefault = true
+			continue
+		}
+		k, v, _ := strings.Cut(kv, "=")
+		if k == "name" {
+			name = v
+			continue
+		}
+		remainingArray = append(remainingArray, kv)
+	}
+	return name, isDefault, strings.Join(remainingArray, ",")
+}

internal/cmd/test_load.go

@@ -18,7 +18,6 @@ import (
 	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/internal/js"
 	"go.k6.io/k6/internal/loader"
-	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/js/modules"
 	"go.k6.io/k6/lib"
 	"go.k6.io/k6/lib/fsext"
@@ -84,7 +83,8 @@ func loadLocalTest(gs *state.GlobalState, cmd *cobra.Command, args []string) (*l
 			val, ok := gs.Env[key]
 			return val, ok
 		},
-		Usage: usage.New(),
+		Usage:          gs.Usage,
+		SecretsManager: gs.SecretsManager,
 	}
 
 	test := &loadedTest{

internal/cmd/tests/cmd_run_test.go

@@ -2402,3 +2402,70 @@ func TestTypeScriptSupport(t *testing.T) {
 	t.Log(stderr)
 	assert.Contains(t, stderr, `something 42`)
 }
+
+func TestBasicSecrets(t *testing.T) {
+	// This is the example it will be nice if we can just run it ,but in this case it needs extra arguments so ... maybe not such a great idea
+	t.Parallel()
+	mainScript := `
+		import secrets from "k6/secrets";
+
+		export default async () => {
+			const my_secret = await secrets.get("cool"); // get secret from a source with the provided identifier
+			console.log(my_secret);
+			await secrets.get("else"); // get secret from a source with the provided identifier
+			console.log(my_secret);
+		}
+	`
+
+	ts := NewGlobalTestState(t)
+	require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "secrets.js"), []byte(mainScript), 0o644))
+
+	ts.CmdArgs = []string{"k6", "run", "--secret-source=mock=cool=something,else=source", "secrets.js"}
+
+	cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+	stderr := ts.Stderr.String()
+	t.Log(stderr)
+
+	assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" source=console`)
+	assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" ***SECRET_REDACTED***=console`)
+}
+
+func TestMultipleSecretSources(t *testing.T) {
+	// This is the example it will be nice if we can just run it ,but in this case it needs extra arguments so ... maybe not such a great idea
+	t.Parallel()
+	mainScript := `
+		import secrets from "k6/secrets";
+
+		export default async () => {
+			const my_secret = await secrets.source("first").get("cool");
+			console.log(my_secret);
+			await secrets.source("second").get("else");
+			console.log(my_secret);
+			try {
+				await secrets.source("second").get("unkwown");
+			} catch {
+				console.log("trigger exception on wrong key")
+			}
+			await secrets.get("else"); // testing default setting
+		}
+	`
+
+	ts := NewGlobalTestState(t)
+	require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "secrets.js"), []byte(mainScript), 0o644))
+
+	ts.CmdArgs = []string{
+		"k6", "run",
+		"--secret-source=mock=name=first,cool=something",
+		"--secret-source=mock=name=second,else=source,default", "secrets.js",
+	}
+
+	cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+	stderr := ts.Stderr.String()
+	t.Log(stderr)
+
+	assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" source=console`)
+	assert.Contains(t, stderr, `level=info msg="***SECRET_REDACTED***" ***SECRET_REDACTED***=console`)
+	assert.Contains(t, stderr, `level=info msg="trigger exception on wrong key" ***SECRET_REDACTED***=console`)
+}

internal/cmd/tests/test_state.go

@@ -18,6 +18,7 @@ import (
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/lib/testutils"
 	"go.k6.io/k6/internal/ui/console"
+	"go.k6.io/k6/internal/usage"
 	"go.k6.io/k6/lib/fsext"
 )
 
@@ -111,6 +112,7 @@ func NewGlobalTestState(tb testing.TB) *GlobalTestState {
 		SignalStop:     signal.Stop,
 		Logger:         logger,
 		FallbackLogger: testutils.NewLogger(tb).WithField("fallback", true),
+		Usage:          usage.New(),
 	}
 
 	return ts

internal/js/jsmodules.go

@@ -17,6 +17,7 @@ import (
 	expws "go.k6.io/k6/internal/js/modules/k6/experimental/websockets"
 	"go.k6.io/k6/internal/js/modules/k6/grpc"
 	"go.k6.io/k6/internal/js/modules/k6/metrics"
+	"go.k6.io/k6/internal/js/modules/k6/secrets"
 	"go.k6.io/k6/internal/js/modules/k6/timers"
 	"go.k6.io/k6/internal/js/modules/k6/webcrypto"
 	"go.k6.io/k6/internal/js/modules/k6/ws"
@@ -62,6 +63,7 @@ func getInternalJSModules() map[string]interface{} {
 		"k6/html":            html.New(),
 		"k6/http":            http.New(),
 		"k6/metrics":         metrics.New(),
+		"k6/secrets":         secrets.New(),
 		"k6/ws":              ws.New(),
 		"k6/experimental/grpc": newRemovedModule(
 			"k6/experimental/grpc has been graduated, please use k6/net/grpc instead." +