refactor(artifact): remove oci artifact support (#1359)

* refactor(artifact): remove oci artifact support
- add header to referrers call to indicated applied artifact type filters

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

* feat(gc): simplify gc logic to increase coverage

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

---------

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

Author: laurentiuNiculae

codecov.yml

@@ -6,3 +6,4 @@ ignore:
     - "./cmd/zb/*.go"
     - "./pkg/test/mocks/*.go"
     - "./swagger/*.go"
+    - "./pkg/test/test_http_server.go"

errors/errors.go

@@ -63,7 +63,6 @@ var (
 	ErrManifestConflict               = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound           = errors.New("repodb: image metadata not found for given manifest reference")
 	ErrManifestDataNotFound           = errors.New("repodb: image data not found for given manifest digest")
-	ErrArtifactDataNotFound           = errors.New("repodb: artifact data not found for given digest")
 	ErrIndexDataNotFount              = errors.New("repodb: index data not found for given digest")
 	ErrRepoMetaNotFound               = errors.New("repodb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound                = errors.New("repodb: tag metadata not found for given repo and tag names")

examples/config-dynamodb.json

@@ -20,7 +20,6 @@
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
             "manifestDataTablename": "ZotManifestDataTable",
-            "artifactDataTablename": "ZotArtifactDataTable",
             "userDataTablename": "ZotUserDataTable",
             "versionTablename": "ZotVersion"
         }

go.mod

@@ -462,7 +462,7 @@ require (
 )
 
 replace (
-	github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20230228184813-24efbd35e8ef
+	github.com/opencontainers/image-spec => github.com/project-zot/image-spec v0.0.0-20230428141304-a263befc67ea
 	// v0.3.1-0.20230104082527-d6f58551be3f is taken from github.com/moby/buildkit v0.11.0
 	// spdx logic write on v0.3.0 and incompatible with v0.3.1-0.20230104082527-d6f58551be3f
 	// see https://github.com/aquasecurity/trivy/commit/2033e05b6b563aa4bc46d6234b3657a54e71ee46

go.sum

@@ -52,8 +52,8 @@ linters-settings:
   gomoddirectives:
     replace-allow-list:
       - helm.sh/helm/v3
-      - github.com/opencontainers/umoci
       - github.com/spdx/tools-golang
+      - github.com/opencontainers/image-spec
 issues:
   exclude-rules:
     - path: pkg/extensions/search/schema.resolvers.go

golangcilint.yaml

@@ -51,7 +51,6 @@ import (
 	"zotregistry.io/zot/pkg/log"
 	"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
 	"zotregistry.io/zot/pkg/storage"
-	storageConstants "zotregistry.io/zot/pkg/storage/constants"
 	"zotregistry.io/zot/pkg/storage/local"
 	"zotregistry.io/zot/pkg/test"
 )
@@ -157,7 +156,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 			"cacheTablename":        "BlobTable",
 			"repoMetaTablename":     "RepoMetadataTable",
 			"manifestDataTablename": "ManifestDataTable",
-			"artifactDataTablename": "ArtifactDataTable",
 			"userDataTablename":     "ZotUserDataTable",
 			"versionTablename":      "Version",
 		}
@@ -173,7 +171,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 			"cacheTablename":        "BlobTable",
 			"repoMetaTablename":     "RepoMetadataTable",
 			"manifestDataTablename": "ManifestDataTable",
-			"artifactDataTablename": "ArtifactDataTable",
 			"userDataTablename":     "ZotUserDataTable",
 			"versionTablename":      "Version",
 		}
@@ -188,7 +185,6 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 			"cacheTablename":        "BlobTable",
 			"repoMetaTablename":     "RepoMetadataTable",
 			"manifestDataTablename": "ManifestDataTable",
-			"artifactDataTablename": "ArtifactDataTable",
 			"userDataTablename":     "ZotUserDataTable",
 			"versionTablename":      "Version",
 		}
@@ -222,7 +218,6 @@ func TestCreateRepoDBDriver(t *testing.T) {
 			"cachetablename":        "BlobTable",
 			"repometatablename":     "RepoMetadataTable",
 			"manifestdatatablename": "ManifestDataTable",
-			"artifactDataTablename": "ArtifactDataTable",
 			"userdatatablename":     "UserDatatable",
 		}
 
@@ -236,7 +231,6 @@ func TestCreateRepoDBDriver(t *testing.T) {
 			"cachetablename":        "",
 			"repometatablename":     "RepoMetadataTable",
 			"manifestdatatablename": "ManifestDataTable",
-			"artifactDataTablename": "ArtifactDataTable",
 			"userDataTablename":     "ZotUserDataTable",
 			"versiontablename":      1,
 		}
@@ -252,7 +246,6 @@ func TestCreateRepoDBDriver(t *testing.T) {
 			"repometatablename":     "RepoMetadataTable",
 			"manifestdatatablename": "ManifestDataTable",
 			"indexdatatablename":    "IndexDataTable",
-			"artifactdatatablename": "ArtifactDataTable",
 			"userdatatablename":     "ZotUserDataTable",
 			"versiontablename":      "1",
 		}
@@ -4395,138 +4388,15 @@ func TestArtifactReferences(t *testing.T) {
 				So(err, ShouldBeNil)
 				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
 				So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
-			})
-		})
-
-		Convey("Validate Artifact Manifest Reference", func() {
-			resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-			So(err, ShouldBeNil)
-			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-			var referrers ispec.Index
-			err = json.Unmarshal(resp.Body(), &referrers)
-			So(err, ShouldBeNil)
-			So(referrers.Manifests, ShouldBeEmpty)
-
-			// now upload a reference
-
-			// upload image config blob
-			resp, err = resty.R().Post(baseURL + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName))
-			So(err, ShouldBeNil)
-			So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
-			loc := test.Location(baseURL, resp)
-			cblob, cdigest := test.GetEmptyImageConfig()
-
-			resp, err = resty.R().
-				SetContentLength(true).
-				SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
-				SetHeader("Content-Type", "application/octet-stream").
-				SetQueryParam("digest", cdigest.String()).
-				SetBody(cblob).
-				Put(loc)
-			So(err, ShouldBeNil)
-			So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
-
-			// create a artifact
-			manifest := ispec.Artifact{
-				MediaType:    ispec.MediaTypeArtifactManifest,
-				ArtifactType: artifactType,
-				Blobs: []ispec.Descriptor{
-					{
-						MediaType: "application/vnd.oci.image.layer.v1.tar",
-						Digest:    digest,
-						Size:      int64(len(content)),
-					},
-				},
-				Subject: &ispec.Descriptor{
-					MediaType: ispec.MediaTypeImageManifest,
-					Digest:    digest,
-					Size:      int64(len(content)),
-				},
-				Annotations: map[string]string{
-					"key": "val",
-				},
-			}
-			Convey("Using invalid content", func() {
-				content := []byte("invalid data")
-				So(err, ShouldBeNil)
-				mdigest := godigest.FromBytes(content)
-				So(mdigest, ShouldNotBeNil)
-				resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
-					SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest)
-
-				// unknown repo will return status not found
-				resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", "unknown", digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
-
-				resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+				So(resp.Header().Get("OCI-Filters-Applied"), ShouldEqual, artifactType)
 
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-			})
-			Convey("Using valid content", func() {
-				content, err = json.Marshal(manifest)
-				So(err, ShouldBeNil)
-				mdigest := godigest.FromBytes(content)
-				So(mdigest, ShouldNotBeNil)
-				resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
-					SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, mdigest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
-
-				resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifact": "invalid"}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": "invalid"}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
-				So(err, ShouldBeNil)
-				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
-
-				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType}).
-					Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()))
+				resp, err = resty.R().SetQueryParams(map[string]string{"artifactType": artifactType +
+					",otherArtType"}).Get(baseURL + fmt.Sprintf("/v2/%s/referrers/%s", repoName,
+					digest.String()))
 				So(err, ShouldBeNil)
 				So(resp.StatusCode(), ShouldEqual, http.StatusOK)
 				So(resp.Header().Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
-
-				var index ispec.Index
-				err = json.Unmarshal(resp.Body(), &index)
-				So(err, ShouldBeNil)
-				So(index.Manifests, ShouldNotBeEmpty)
-				So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty)
-
-				// filter by multiple artifactTypes
-				req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet,
-					baseURL+fmt.Sprintf("/v2/%s/referrers/%s", repoName, digest.String()), nil)
-				So(err, ShouldBeNil)
-				values := url.Values{}
-				values.Add("artifactType", artifactType)
-				values.Add("artifactType", "foobar")
-				req.URL.RawQuery = values.Encode()
-				rsp, err := http.DefaultClient.Do(req)
-				So(err, ShouldBeNil)
-				defer rsp.Body.Close()
-				So(rsp.StatusCode, ShouldEqual, http.StatusOK)
-				So(rsp.Header.Get("Content-Type"), ShouldEqual, ispec.MediaTypeImageIndex)
-				body, err := io.ReadAll(rsp.Body)
-				So(err, ShouldBeNil)
-				err = json.Unmarshal(body, &index)
-				So(err, ShouldBeNil)
-				So(index.Manifests, ShouldNotBeEmpty)
-				So(index.Annotations[storageConstants.ReferrerFilterAnnotation], ShouldNotBeEmpty)
-				So(len(strings.Split(index.Annotations[storageConstants.ReferrerFilterAnnotation], ",")), ShouldEqual, 2)
+				So(resp.Header().Get("OCI-Filters-Applied"), ShouldEqual, artifactType+",otherArtType")
 			})
 		})
 	})

pkg/api/controller_test.go

@@ -433,7 +433,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http.
 	}
 
 	if rh.c.RepoDB != nil {
-		err := meta.OnGetManifest(name, reference, digest, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log)
+		err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.RepoDB, rh.c.Log)
 
 		if errors.Is(err, zerr.ErrOrphanSignature) {
 			rh.c.Log.Error().Err(err).Msg("image is an orphan signature")
@@ -546,6 +546,10 @@ func (rh *RouteHandler) GetReferrers(response http.ResponseWriter, request *http
 		return
 	}
 
+	if len(artifactTypes) > 0 {
+		response.Header().Set("OCI-Filters-Applied", strings.Join(artifactTypes, ","))
+	}
+
 	WriteData(response, http.StatusOK, ispec.MediaTypeImageIndex, out)
 }
 

pkg/api/routes.go

@@ -490,10 +490,8 @@ func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf se
 		return false
 	}
 
-	for _, reference := range referrers.Manifests {
-		if reference.ArtifactType == notreg.ArtifactTypeNotation {
-			return true
-		}
+	if len(referrers.Manifests) > 0 {
+		return true
 	}
 
 	return false