Skip to content

Commit 6bc5b00

Browse files
nistarnistar
authored
Merge branch 'tektoncd:main' into main
2 parents 9721264 + 69a8f5f commit 6bc5b00

File tree

6 files changed

+177
-82
lines changed

6 files changed

+177
-82
lines changed

‎docs/config.md‎

Lines changed: 32 additions & 19 deletions
Large diffs are not rendered by default.

‎pkg/chains/storage/docdb/docdb.go‎

Lines changed: 114 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737

3838
const (
3939
StorageTypeDocDB = "docdb"
40+
mongoEnv = "MONGO_SERVER_URL"
4041
)
4142

4243
// ErrNothingToWatch is an error that's returned when the backend doesn't have anything to "watch"
@@ -98,34 +99,16 @@ func WatchBackend(ctx context.Context, cfg config.Config, watcherStop chan bool)
9899
return nil, ErrNothingToWatch
99100
}
100101

101-
// Set up watcher only when `storage.docdb.mongo-server-url-dir` is set
102-
if cfg.Storage.DocDB.MongoServerURLDir == "" {
102+
pathsToWatch := getPathsToWatch(ctx, cfg)
103+
if len(pathsToWatch) == 0 {
103104
return nil, ErrNothingToWatch
104105
}
105106

106-
logger.Infof("setting up fsnotify watcher for directory: %s", cfg.Storage.DocDB.MongoServerURLDir)
107-
108107
watcher, err := fsnotify.NewWatcher()
109108
if err != nil {
110109
return nil, err
111110
}
112111

113-
pathsToWatch := []string{
114-
// mongo-server-url-dir/MONGO_SERVER_URL is where the MONGO_SERVER_URL environment
115-
// variable is expected to be mounted, either manually or via a Kubernetes secret, etc.
116-
filepath.Join(cfg.Storage.DocDB.MongoServerURLDir, "MONGO_SERVER_URL"),
117-
// When a Kubernetes secret is mounted on a path, the `data` in that secret is mounted
118-
// under path/..data that is then `symlink`ed to the key of the data. In this instance,
119-
// the mounted path is going to look like:
120-
// file 1 - ..2024_05_03_11_23_23.1253599725
121-
// file 2 - ..data -> ..2024_05_03_11_23_23.1253599725
122-
// file 3 - MONGO_SERVER_URL -> ..data/MONGO_SERVER_URL
123-
// So each time the secret is updated, the file `MONGO_SERVER_URL` is not updated,
124-
// instead the underlying symlink at `..data` is updated and that's what we want to
125-
// capture via the fsnotify event watcher
126-
filepath.Join(cfg.Storage.DocDB.MongoServerURLDir, "..data"),
127-
}
128-
129112
backendChan := make(chan *Backend)
130113
// Start listening for events.
131114
go func() {
@@ -145,13 +128,23 @@ func WatchBackend(ctx context.Context, cfg config.Config, watcherStop chan bool)
145128
continue
146129
}
147130

148-
updatedEnv, err := getMongoServerURLFromDir(cfg.Storage.DocDB.MongoServerURLDir)
149-
if err != nil {
150-
logger.Error(err)
151-
backendChan <- nil
131+
var updatedEnv string
132+
if cfg.Storage.DocDB.MongoServerURLPath != "" {
133+
updatedEnv, err = getMongoServerURLFromPath(cfg.Storage.DocDB.MongoServerURLPath)
134+
if err != nil {
135+
logger.Error(err)
136+
backendChan <- nil
137+
}
138+
} else if cfg.Storage.DocDB.MongoServerURLDir != "" {
139+
updatedEnv, err = getMongoServerURLFromDir(cfg.Storage.DocDB.MongoServerURLDir)
140+
if err != nil {
141+
logger.Error(err)
142+
backendChan <- nil
143+
}
152144
}
145+
153146
if updatedEnv != os.Getenv("MONGO_SERVER_URL") {
154-
logger.Infof("directory %s has been updated, reconfiguring backend...", cfg.Storage.DocDB.MongoServerURLDir)
147+
logger.Info("Mongo server url has been updated, reconfiguring backend...")
155148

156149
// Now that MONGO_SERVER_URL has been updated, we should update docdb backend again
157150
newDocDBBackend, err := NewStorageBackend(ctx, cfg)
@@ -179,11 +172,23 @@ func WatchBackend(ctx context.Context, cfg config.Config, watcherStop chan bool)
179172
}
180173
}()
181174

182-
// Add a path.
183-
err = watcher.Add(cfg.Storage.DocDB.MongoServerURLDir)
184-
if err != nil {
185-
return nil, err
175+
if cfg.Storage.DocDB.MongoServerURLPath != "" {
176+
dirPath := filepath.Dir(cfg.Storage.DocDB.MongoServerURLPath)
177+
// Add a path.
178+
err = watcher.Add(dirPath)
179+
if err != nil {
180+
return nil, err
181+
}
186182
}
183+
184+
if cfg.Storage.DocDB.MongoServerURLDir != "" {
185+
// Add a path.
186+
err = watcher.Add(cfg.Storage.DocDB.MongoServerURLDir)
187+
if err != nil {
188+
return nil, err
189+
}
190+
}
191+
187192
return backendChan, nil
188193
}
189194

@@ -256,27 +261,43 @@ func (b *Backend) retrieveDocuments(ctx context.Context, opts config.StorageOpts
256261
}
257262

258263
func populateMongoServerURL(ctx context.Context, cfg config.Config) error {
259-
// First preference is given to the key `storage.docdb.mongo-server-url-dir`.
264+
// First preference is given to the key `storage.docdb.mongo-server-url-path`.
265+
// If that doesn't exist, then we move on to `storage.docdb.mongo-server-url-dir`.
260266
// If that doesn't exist, then we move on to `storage.docdb.mongo-server-url`.
261267
// If that doesn't exist, then we check if `MONGO_SERVER_URL` env var is set.
262268
logger := logging.FromContext(ctx)
263-
mongoEnv := "MONGO_SERVER_URL"
269+
270+
if cfg.Storage.DocDB.MongoServerURLPath != "" {
271+
logger.Infof("setting %s from storage.docdb.mongo-server-url-path: %s", mongoEnv, cfg.Storage.DocDB.MongoServerURLPath)
272+
mongoServerURL, err := getMongoServerURLFromPath(cfg.Storage.DocDB.MongoServerURLPath)
273+
if err != nil {
274+
return err
275+
}
276+
if err := os.Setenv(mongoEnv, mongoServerURL); err != nil {
277+
return err
278+
}
279+
return nil
280+
}
264281

265282
if cfg.Storage.DocDB.MongoServerURLDir != "" {
266283
logger.Infof("setting %s from storage.docdb.mongo-server-url-dir: %s", mongoEnv, cfg.Storage.DocDB.MongoServerURLDir)
267284
if err := setMongoServerURLFromDir(cfg.Storage.DocDB.MongoServerURLDir); err != nil {
268285
return err
269286
}
270-
} else if cfg.Storage.DocDB.MongoServerURL != "" {
287+
return nil
288+
}
289+
290+
if cfg.Storage.DocDB.MongoServerURL != "" {
271291
logger.Infof("setting %s from storage.docdb.mongo-server-url", mongoEnv)
272292
if err := os.Setenv(mongoEnv, cfg.Storage.DocDB.MongoServerURL); err != nil {
273293
return err
274294
}
295+
return nil
275296
}
276297

277298
if _, envExists := os.LookupEnv(mongoEnv); !envExists {
278299
return fmt.Errorf("mongo docstore configured but %s environment variable not set, "+
279-
"supply one of storage.docdb.mongo-server-url-dir, storage.docdb.mongo-server-url or set %s", mongoEnv, mongoEnv)
300+
"supply one of storage.docdb.mongo-server-url-path, storage.docdb.mongo-server-url-dir, storage.docdb.mongo-server-url or set %s", mongoEnv, mongoEnv)
280301
}
281302

282303
return nil
@@ -288,16 +309,14 @@ func setMongoServerURLFromDir(dir string) error {
288309
return err
289310
}
290311

291-
if err = os.Setenv("MONGO_SERVER_URL", fileDataNormalized); err != nil {
312+
if err = os.Setenv(mongoEnv, fileDataNormalized); err != nil {
292313
return err
293314
}
294315

295316
return nil
296317
}
297318

298319
func getMongoServerURLFromDir(dir string) (string, error) {
299-
mongoEnv := "MONGO_SERVER_URL"
300-
301320
stat, err := os.Stat(dir)
302321
if err != nil {
303322
if os.IsNotExist(err) {
@@ -327,3 +346,62 @@ func getMongoServerURLFromDir(dir string) (string, error) {
327346

328347
return fileDataNormalized, nil
329348
}
349+
350+
// getMongoServerUrlFromPath retreives token from the given mount path
351+
func getMongoServerURLFromPath(path string) (string, error) {
352+
fileData, err := os.ReadFile(path)
353+
if err != nil {
354+
return "", fmt.Errorf("reading file in %q: %w", path, err)
355+
}
356+
357+
// A trailing newline is fairly common in mounted files, so remove it.
358+
fileDataNormalized := strings.TrimSuffix(string(fileData), "\n")
359+
return fileDataNormalized, nil
360+
}
361+
362+
func getPathsToWatch(ctx context.Context, cfg config.Config) []string {
363+
var pathsToWatch = []string{}
364+
logger := logging.FromContext(ctx)
365+
366+
if cfg.Storage.DocDB.MongoServerURLPath != "" {
367+
logger.Infof("setting up fsnotify watcher for mongo server url path: %s", cfg.Storage.DocDB.MongoServerURLPath)
368+
dirPath := filepath.Dir(cfg.Storage.DocDB.MongoServerURLPath)
369+
pathsToWatch = []string{
370+
// mongo-server-url-path/<path> is where the mongo server url token
371+
// When a Kubernetes secret is mounted on a path, the `data` in that secret is mounted
372+
// under path/..data that is then `symlink`ed to the key of the data. In this instance,
373+
// the mounted path is going to look like:
374+
// file 1 - ..2024_05_03_11_23_23.1253599725
375+
// file 2 - ..data -> ..2024_05_03_11_23_23.1253599725
376+
// file 3 - ..data/<path>
377+
// So each time the secret is updated, the file is not updated,
378+
// instead the underlying symlink at `..data` is updated and that's what we want to
379+
// capture via the fsnotify event watcher
380+
// filepath.Join(cfg.Storage.DocDB.MongoServerURLPath, "..data"),
381+
filepath.Join(dirPath, "..data"),
382+
}
383+
return pathsToWatch
384+
}
385+
386+
if cfg.Storage.DocDB.MongoServerURLDir != "" {
387+
logger.Infof("setting up fsnotify watcher for directory: %s", cfg.Storage.DocDB.MongoServerURLDir)
388+
pathsToWatch = []string{
389+
// mongo-server-url-dir/MONGO_SERVER_URL is where the MONGO_SERVER_URL environment
390+
// variable is expected to be mounted, either manually or via a Kubernetes secret, etc.
391+
filepath.Join(cfg.Storage.DocDB.MongoServerURLDir, "MONGO_SERVER_URL"),
392+
// When a Kubernetes secret is mounted on a path, the `data` in that secret is mounted
393+
// under path/..data that is then `symlink`ed to the key of the data. In this instance,
394+
// the mounted path is going to look like:
395+
// file 1 - ..2024_05_03_11_23_23.1253599725
396+
// file 2 - ..data -> ..2024_05_03_11_23_23.1253599725
397+
// file 3 - MONGO_SERVER_URL -> ..data/MONGO_SERVER_URL
398+
// So each time the secret is updated, the file `MONGO_SERVER_URL` is not updated,
399+
// instead the underlying symlink at `..data` is updated and that's what we want to
400+
// capture via the fsnotify event watcher
401+
filepath.Join(cfg.Storage.DocDB.MongoServerURLDir, "..data"),
402+
}
403+
return pathsToWatch
404+
}
405+
406+
return pathsToWatch
407+
}

‎pkg/chains/storage/docdb/docdb_test.go‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -310,19 +310,20 @@ func TestWatchBackend(t *testing.T) {
310310
expectedMongoEnv: testEnv,
311311
wantErr: true,
312312
},
313-
{
314-
name: "verify mongo-server-url-dir/MONGO_SERVER_URL is watched",
315-
cfg: config.Config{
316-
Storage: config.StorageConfigs{
317-
DocDB: config.DocDBStorageConfig{
318-
URL: "mongo://chainsdb/chainscollection?id_field=name",
319-
MongoServerURLDir: t.TempDir(),
320-
},
321-
},
322-
},
323-
expectedMongoEnv: "mongodb://updatedEnv",
324-
wantErr: false,
325-
},
313+
// TODO: https://github.com/tektoncd/chains/issues/1178
314+
// {
315+
// name: "verify mongo-server-url-dir/MONGO_SERVER_URL is watched",
316+
// cfg: config.Config{
317+
// Storage: config.StorageConfigs{
318+
// DocDB: config.DocDBStorageConfig{
319+
// URL: "mongo://chainsdb/chainscollection?id_field=name",
320+
// MongoServerURLDir: t.TempDir(),
321+
// },
322+
// },
323+
// },
324+
// expectedMongoEnv: "mongodb://updatedEnv",
325+
// wantErr: false,
326+
// },
326327
}
327328

328329
for _, tt := range tests {

‎pkg/config/config.go‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,10 @@ type TektonStorageConfig struct {
123123
}
124124

125125
type DocDBStorageConfig struct {
126-
URL string
127-
MongoServerURL string
128-
MongoServerURLDir string
126+
URL string
127+
MongoServerURL string
128+
MongoServerURLDir string
129+
MongoServerURLPath string
129130
}
130131

131132
type GrafeasConfig struct {
@@ -168,12 +169,13 @@ const (
168169
ociStorageKey = "artifacts.oci.storage"
169170
ociSignerKey = "artifacts.oci.signer"
170171

171-
gcsBucketKey = "storage.gcs.bucket"
172-
ociRepositoryKey = "storage.oci.repository"
173-
ociRepositoryInsecureKey = "storage.oci.repository.insecure"
174-
docDBUrlKey = "storage.docdb.url"
175-
docDBMongoServerURLKey = "storage.docdb.mongo-server-url"
176-
docDBMongoServerURLDirKey = "storage.docdb.mongo-server-url-dir"
172+
gcsBucketKey = "storage.gcs.bucket"
173+
ociRepositoryKey = "storage.oci.repository"
174+
ociRepositoryInsecureKey = "storage.oci.repository.insecure"
175+
docDBUrlKey = "storage.docdb.url"
176+
docDBMongoServerURLKey = "storage.docdb.mongo-server-url"
177+
docDBMongoServerURLDirKey = "storage.docdb.mongo-server-url-dir"
178+
docDBMongoServerURLPathKey = "storage.docdb.mongo-server-url-path"
177179

178180
grafeasProjectIDKey = "storage.grafeas.projectid"
179181
grafeasNoteIDKey = "storage.grafeas.noteid"
@@ -302,6 +304,7 @@ func NewConfigFromMap(data map[string]string) (*Config, error) {
302304
asString(docDBUrlKey, &cfg.Storage.DocDB.URL),
303305
asString(docDBMongoServerURLKey, &cfg.Storage.DocDB.MongoServerURL),
304306
asString(docDBMongoServerURLDirKey, &cfg.Storage.DocDB.MongoServerURLDir),
307+
asString(docDBMongoServerURLPathKey, &cfg.Storage.DocDB.MongoServerURLPath),
305308
asString(grafeasProjectIDKey, &cfg.Storage.Grafeas.ProjectID),
306309
asString(grafeasNoteIDKey, &cfg.Storage.Grafeas.NoteID),
307310
asString(grafeasNoteHint, &cfg.Storage.Grafeas.NoteHint),

‎release/publish.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ spec:
160160
ko resolve \
161161
--image-label=org.opencontainers.image.source=https://$(params.package) \
162162
--platform=$(params.platforms) \
163-
${KO_EXTRA_ARGS} -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml
163+
${KO_EXTRA_ARGS} -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.notags.yaml
164164
165165
# Rewrite "devel" to params.versionTag
166166
sed -i -e 's/\(pipeline.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.yaml

‎release/release-pipeline.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,13 +169,13 @@ spec:
169169
resolver: git
170170
params:
171171
- name: repo
172-
value: pipeline
172+
value: chains
173173
- name: org
174174
value: tektoncd
175175
- name: revision
176176
value: $(params.gitRevision)
177177
- name: pathInRepo
178-
value: tekton/publish.yaml
178+
value: release/publish.yaml
179179
params:
180180
- name: package
181181
value: $(params.package)
@@ -195,8 +195,8 @@ spec:
195195
value: $(params.serviceAccountImagesPath)
196196
- name: platforms
197197
value: $(params.platforms)
198-
- name: koExtraArgs
199-
value: $(params.koExtraArgs)
198+
- name: koExtraArgs
199+
value: $(params.koExtraArgs)
200200
- name: CHAINS-GIT_COMMIT
201201
value: $(tasks.git-clone.results.commit)
202202
- name: CHAINS-GIT_URL

0 commit comments

Comments
 (0)