feat(config): support env substitution

examples/README.md

@@ -19,6 +19,7 @@ Examples of working configurations for various use cases are available [here](..
 # Configuration Parameters
 
 - [Configuration Parameters](#configuration-parameters)
+  - [Environment variables](#environment-variables)
   - [Network](#network)
   - [Storage](#storage)
   - [Authentication](#authentication)
@@ -34,6 +35,34 @@ Examples of working configurations for various use cases are available [here](..
   - [Sync](#sync)
   - [Search and CVE scanning (Trivy)](#search-and-cve-scanning-trivy)
 
+## Environment variables
+
+Configuration files support environment variable substitution for string values. Both `${VAR}` and `$VAR` forms are
+supported in the main zot config and in referenced secret files such as OpenID/OAuth2, LDAP, and session key files.
+Startup fails if a referenced environment variable is not set.
+
+For example:
+
+```
+{
+    "storage": {
+        "rootDirectory": "${ZOT_ROOT_DIRECTORY}"
+    },
+    "http": {
+        "auth": {
+            "openid": {
+                "providers": {
+                    "oidc": {
+                        "credentialsFile": "${ZOT_OPENID_CREDENTIALS_FILE}",
+                        "issuer": "${ZOT_OPENID_ISSUER}",
+                        "scopes": ["openid"]
+                    }
+                }
+            }
+        }
+    }
+}
+```
 
 ## Network
 
@@ -1189,4 +1218,3 @@ To set those options explicitly (for example to mirror standalone Trivy’s `--v
 - [config-cve-trivy.json](config-cve-trivy.json) — shows optional `dbRepository`, `javaDBRepository`, and `vulnSeveritySources`.
 
 `vulnSeveritySources` is a list of source names in priority order (for example `auto`, `nvd`, or vendor IDs such as `redhat`, `alpine`). If omitted, zot defaults it to `["auto"]`, consistent with the Trivy CLI. See [Trivy: severity selection](https://trivy.dev/docs/latest/scanner/vulnerability/#severity-selection).
-

pkg/cli/server/config_loader.go

@@ -0,0 +1,164 @@
+package server
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"slices"
+	"sort"
+	"strings"
+
+	"github.com/go-viper/mapstructure/v2"
+	"github.com/spf13/viper"
+
+	zerr "zotregistry.dev/zot/v2/errors"
+	zlog "zotregistry.dev/zot/v2/pkg/log"
+)
+
+func readConfigFile(viperInstance *viper.Viper, configPath string, logger zlog.Logger) error {
+	ext := filepath.Ext(configPath)
+	ext = strings.Replace(ext, ".", "", 1)
+
+	/* if file extension is not supported, try everything
+	it's also possible that the filename is starting with a dot eg: ".config". */
+	if !slices.Contains(viper.SupportedExts, ext) {
+		ext = ""
+	}
+
+	viperInstance.SetConfigFile(configPath)
+
+	switch ext {
+	case "":
+		logger.Info().Str("path", configPath).Msg("config file with no extension, trying all supported config types")
+
+		var readErr error
+
+		for _, configType := range viper.SupportedExts {
+			viperInstance.SetConfigType(configType)
+
+			readErr = viperInstance.ReadInConfig()
+			if readErr == nil {
+				return nil
+			}
+		}
+
+		return readErr
+	default:
+		return viperInstance.ReadInConfig()
+	}
+}
+
+func envSubstitutionDecodeHook() mapstructure.DecodeHookFuncType {
+	return func(from reflect.Type, _ reflect.Type, data any) (any, error) {
+		if from.Kind() != reflect.String {
+			return data, nil
+		}
+
+		return expandConfigEnv(data.(string))
+	}
+}
+
+func expandConfigEnv(configValue string) (string, error) {
+	missingEnv := map[string]struct{}{}
+	var expandedConfig strings.Builder
+
+	for index := 0; index < len(configValue); {
+		if configValue[index] != '$' {
+			expandedConfig.WriteByte(configValue[index])
+			index++
+
+			continue
+		}
+
+		if index+1 >= len(configValue) {
+			expandedConfig.WriteByte(configValue[index])
+			index++
+
+			continue
+		}
+
+		next := configValue[index+1]
+
+		switch {
+		case next == '{':
+			envNameEnd := strings.IndexByte(configValue[index+2:], '}')
+			if envNameEnd == -1 {
+				expandedConfig.WriteByte(configValue[index])
+				index++
+
+				continue
+			}
+
+			envName := configValue[index+2 : index+2+envNameEnd]
+			if !isConfigEnvName(envName) {
+				expandedConfig.WriteString(configValue[index : index+envNameEnd+3])
+				index += envNameEnd + 3
+
+				continue
+			}
+
+			expandedConfig.WriteString(lookupConfigEnv(envName, missingEnv))
+			index += envNameEnd + 3
+		case isConfigEnvNameStart(next):
+			envNameStart := index + 1
+			envNameEnd := envNameStart + 1
+
+			for envNameEnd < len(configValue) && isConfigEnvNameChar(configValue[envNameEnd]) {
+				envNameEnd++
+			}
+
+			envName := configValue[envNameStart:envNameEnd]
+
+			expandedConfig.WriteString(lookupConfigEnv(envName, missingEnv))
+			index = envNameEnd
+		default:
+			expandedConfig.WriteByte(configValue[index])
+			index++
+		}
+	}
+
+	if len(missingEnv) > 0 {
+		names := make([]string, 0, len(missingEnv))
+		for name := range missingEnv {
+			names = append(names, name)
+		}
+
+		sort.Strings(names)
+
+		return "", fmt.Errorf("%w: environment variable(s) not set: %s", zerr.ErrBadConfig, strings.Join(names, ", "))
+	}
+
+	return expandedConfig.String(), nil
+}
+
+func lookupConfigEnv(name string, missingEnv map[string]struct{}) string {
+	value, ok := os.LookupEnv(name)
+	if !ok {
+		missingEnv[name] = struct{}{}
+	}
+
+	return value
+}
+
+func isConfigEnvName(name string) bool {
+	if name == "" || !isConfigEnvNameStart(name[0]) {
+		return false
+	}
+
+	for index := 1; index < len(name); index++ {
+		if !isConfigEnvNameChar(name[index]) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func isConfigEnvNameStart(char byte) bool {
+	return char == '_' || (char >= 'A' && char <= 'Z') || (char >= 'a' && char <= 'z')
+}
+
+func isConfigEnvNameChar(char byte) bool {
+	return isConfigEnvNameStart(char) || (char >= '0' && char <= '9')
+}

pkg/cli/server/root.go

@@ -1088,44 +1088,10 @@ func LoadConfiguration(config *config.Config, configPath string) error {
 	// we need another key delimiter.
 	viperInstance := viper.NewWithOptions(viper.KeyDelimiter("::"))
 
-	ext := filepath.Ext(configPath)
-	ext = strings.Replace(ext, ".", "", 1)
+	if err := readConfigFile(viperInstance, configPath, logger); err != nil {
+		logger.Error().Err(err).Str("path", configPath).Msg("failed to read configuration")
 
-	/* if file extension is not supported, try everything
-	it's also possible that the filename is starting with a dot eg: ".config". */
-	if !slices.Contains(viper.SupportedExts, ext) {
-		ext = ""
-	}
-
-	switch ext {
-	case "":
-		logger.Info().Str("path", configPath).Msg("config file with no extension, trying all supported config types")
-
-		var err error
-
-		for _, configType := range viper.SupportedExts {
-			viperInstance.SetConfigType(configType)
-			viperInstance.SetConfigFile(configPath)
-
-			err = viperInstance.ReadInConfig()
-			if err == nil {
-				break
-			}
-		}
-
-		if err != nil {
-			logger.Error().Err(err).Str("path", configPath).Msg("failed to read configuration, tried all supported config types")
-
-			return err
-		}
-	default:
-		viperInstance.SetConfigFile(configPath)
-
-		if err := viperInstance.ReadInConfig(); err != nil {
-			logger.Error().Err(err).Str("path", configPath).Msg("failed to read configuration")
-
-			return err
-		}
+		return err
 	}
 
 	metaData := &mapstructure.Metadata{}
@@ -1134,6 +1100,7 @@ func LoadConfiguration(config *config.Config, configPath string) error {
 		metadataConfig(metaData),
 		viper.DecodeHook(
 			mapstructure.ComposeDecodeHookFunc(
+				envSubstitutionDecodeHook(),
 				mapstructure.StringToTimeDurationHookFunc(),
 				eventsconf.SinkConfigDecoderHook(),
 			),
@@ -1277,16 +1244,14 @@ func readSecretFile(path string, v any, checkUnsetFields bool) error { //nolint:
 
 	viperInstance := viper.NewWithOptions(viper.KeyDelimiter("::"))
 
-	viperInstance.SetConfigFile(path)
-
-	if err := viperInstance.ReadInConfig(); err != nil {
+	if err := readConfigFile(viperInstance, path, logger); err != nil {
 		logger.Error().Err(err).Str("path", path).Msg("failed to read secret file configuration")
 
 		return errors.Join(zerr.ErrBadConfig, err)
 	}
 
 	metaData := &mapstructure.Metadata{}
-	if err := viperInstance.Unmarshal(v, metadataConfig(metaData)); err != nil {
+	if err := viperInstance.Unmarshal(v, metadataConfig(metaData), viper.DecodeHook(envSubstitutionDecodeHook())); err != nil {
 		logger.Error().Err(err).Str("path", path).Msg("failed to unmarshal secret file config")
 
 		return errors.Join(zerr.ErrBadConfig, err)

pkg/cli/server/root_test.go

@@ -2450,6 +2450,91 @@ func TestLoadConfig(t *testing.T) {
 		err := cli.LoadConfiguration(config, "../../../examples/config-policy.json")
 		So(err, ShouldBeNil)
 	})
+
+	Convey("Test config environment variable substitution", t, func() {
+		rootDir := t.TempDir()
+		credentialsFile := filepath.Join(t.TempDir(), "openid-credentials.json")
+
+		t.Setenv("ZOT_TEST_ROOT_DIR", rootDir)
+		t.Setenv("ZOT_TEST_OIDC_CREDENTIALS", credentialsFile)
+		t.Setenv("ZOT_TEST_CLIENT_ID", "env-client-id")
+		t.Setenv("ZOT_TEST_CLIENT_SECRET", `env-client-secret-with-"quotes"`)
+		t.Setenv("ZOT_TEST_GC_DELAY", "2h")
+
+		err := os.WriteFile(credentialsFile, []byte(`{
+			"clientid": "${ZOT_TEST_CLIENT_ID}",
+			"clientsecret": "${ZOT_TEST_CLIENT_SECRET}"
+		}`), 0o600)
+		So(err, ShouldBeNil)
+
+		content := `{
+			"storage": {
+				"rootDirectory": "$ZOT_TEST_ROOT_DIR",
+				"gc": true,
+				"gcDelay": "${ZOT_TEST_GC_DELAY}"
+			},
+			"http": {
+				"address": "127.0.0.1",
+				"port": "8080",
+				"auth": {
+					"openid": {
+						"providers": {
+							"oidc": {
+								"name": "oidc",
+								"credentialsFile": "${ZOT_TEST_OIDC_CREDENTIALS}",
+								"issuer": "https://issuer.example.com",
+								"scopes": ["openid"]
+							}
+						}
+					}
+				}
+			},
+			"log": {
+				"level": "debug"
+			}
+		}`
+		tmpfile := MakeTempFileWithContent(t, "zot-test.json", content)
+
+		config := config.New()
+		err = cli.LoadConfiguration(config, tmpfile)
+		So(err, ShouldBeNil)
+		So(config.Storage.RootDirectory, ShouldEqual, rootDir)
+		So(config.Storage.GCDelay, ShouldEqual, 2*time.Hour)
+		So(config.HTTP.Auth.OpenID.Providers["oidc"].CredentialsFile, ShouldEqual, credentialsFile)
+		So(config.HTTP.Auth.OpenID.Providers["oidc"].ClientID, ShouldEqual, "env-client-id")
+		So(config.HTTP.Auth.OpenID.Providers["oidc"].ClientSecret, ShouldEqual, `env-client-secret-with-"quotes"`)
+	})
+
+	Convey("Test missing config environment variable", t, func() {
+		missingEnvName := "ZOT_TEST_MISSING_ROOT_DIR"
+		oldEnvValue, wasSet := os.LookupEnv(missingEnvName)
+
+		err := os.Unsetenv(missingEnvName)
+		So(err, ShouldBeNil)
+
+		defer func() {
+			if wasSet {
+				_ = os.Setenv(missingEnvName, oldEnvValue)
+			}
+		}()
+
+		content := fmt.Sprintf(`{
+			"storage": {
+				"rootDirectory": "${%s}"
+			},
+			"http": {
+				"address": "127.0.0.1",
+				"port": "8080"
+			}
+		}`, missingEnvName)
+		tmpfile := MakeTempFileWithContent(t, "zot-test.json", content)
+
+		config := config.New()
+		err = cli.LoadConfiguration(config, tmpfile)
+		So(err, ShouldNotBeNil)
+		So(err.Error(), ShouldContainSubstring, "environment variable(s) not set: "+missingEnvName)
+	})
+
 	Convey("Test subpath config combination", t, func(c C) {
 		config := config.New()
 		content := `{"storage":{"rootDirectory":"/tmp/zot",