feat(cache): remove temporary cache after filesystem scanning (#1868)

Author: afdesk

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
 	github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986
-	github.com/aquasecurity/fanal v0.0.0-20220317181013-c4fac2e5fe9c
+	github.com/aquasecurity/fanal v0.0.0-20220324154234-b2df5b98f8cd
 	github.com/aquasecurity/go-dep-parser v0.0.0-20220302151315-ff6d77c26988
 	github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce
 	github.com/aquasecurity/go-npm-version v0.0.0-20201110091526-0b796d180798

go.sum

@@ -237,8 +237,8 @@ github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986 h1:2a30
 github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986/go.mod h1:NT+jyeCzXk6vXR5MTkdn4z64TgGfE5HMLC8qfj5unl8=
 github.com/aquasecurity/defsec v0.17.1 h1:gen/DInkQZ+BnV2X/UCI4Kb7SgJzPKiSb91duNhOWcg=
 github.com/aquasecurity/defsec v0.17.1/go.mod h1:fmymhKkorY0+cTGAML6LQI+BpCEP1zURaI8smST5rV0=
-github.com/aquasecurity/fanal v0.0.0-20220317181013-c4fac2e5fe9c h1:Php9oTqRg5CyEfLaxaLROfKHu6Wldc20+PkJwJczaOI=
-github.com/aquasecurity/fanal v0.0.0-20220317181013-c4fac2e5fe9c/go.mod h1:PL2i7JtbuPnLlJVG5HVPAVLMmAUdpA9J/iV7b7E5Gbg=
+github.com/aquasecurity/fanal v0.0.0-20220324154234-b2df5b98f8cd h1:hMHRTOuuWsPOGhYBV2MWkTVF8E6oRH0CMH4tKuNzm2M=
+github.com/aquasecurity/fanal v0.0.0-20220324154234-b2df5b98f8cd/go.mod h1:F83w17YIlAOD45TNtwgp1sUE9XqVzLARpynv+emTvGw=
 github.com/aquasecurity/go-dep-parser v0.0.0-20220302151315-ff6d77c26988 h1:Hd6q0/VF/bC/MT1K/63W2u5ChRIy6cPSQk0YbJ3Vcb8=
 github.com/aquasecurity/go-dep-parser v0.0.0-20220302151315-ff6d77c26988/go.mod h1:XxIz2s4UymZBcg9WwAc2km77lFt9rVE/LmKJe2YVOtY=
 github.com/aquasecurity/go-gem-version v0.0.0-20201115065557-8eed6fe000ce h1:QgBRgJvtEOBtUXilDb1MLi1p1MWoyFDXAu5DEUl5nwM=

pkg/cache/remote.go

@@ -61,3 +61,12 @@ func (c RemoteCache) MissingBlobs(imageID string, layerIDs []string) (bool, []st
 	}
 	return layers.MissingArtifact, layers.MissingBlobIds, nil
 }
+
+// DeleteBlobs removes blobs by IDs from RemoteCache
+func (c RemoteCache) DeleteBlobs(blobIDs []string) error {
+	_, err := c.client.DeleteBlobs(c.ctx, rpc.ConvertToDeleteBlobsRequest(blobIDs))
+	if err != nil {
+		return xerrors.Errorf("unable to delete blobs on the server: %w", err)
+	}
+	return nil
+}

pkg/cache/remote_test.go

@@ -51,6 +51,15 @@ func (s *mockCacheServer) MissingBlobs(_ context.Context, in *rpcCache.MissingBl
 	return &rpcCache.MissingBlobsResponse{MissingArtifact: true, MissingBlobIds: layerIDs}, nil
 }
 
+func (s *mockCacheServer) DeleteBlobs(_ context.Context, in *rpcCache.DeleteBlobsRequest) (*google_protobuf.Empty, error) {
+	for _, blobId := range in.GetBlobIds() {
+		if strings.Contains(blobId, "invalid") {
+			return &google_protobuf.Empty{}, xerrors.New("invalid layer ID")
+		}
+	}
+	return &google_protobuf.Empty{}, nil
+}
+
 func withToken(base http.Handler, token, tokenHeader string) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if token != "" && token != r.Header.Get(tokenHeader) {

pkg/rpc/convert.go

@@ -584,3 +584,14 @@ func ConvertToRPCScanResponse(results types.Results, fos *ftypes.OS) *scanner.Sc
 		Results: rpcResults,
 	}
 }
+
+func ConvertToDeleteBlobsRequest(blobIDs []string) *cache.DeleteBlobsRequest {
+	return &cache.DeleteBlobsRequest{BlobIds: blobIDs}
+}
+
+func ConvertFromDeleteBlobsRequest(deleteBlobsRequest *cache.DeleteBlobsRequest) []string {
+	if deleteBlobsRequest == nil {
+		return []string{}
+	}
+	return deleteBlobsRequest.GetBlobIds()
+}

pkg/rpc/server/server.go

@@ -96,3 +96,12 @@ func (s *CacheServer) MissingBlobs(_ context.Context, in *rpcCache.MissingBlobsR
 	}
 	return &rpcCache.MissingBlobsResponse{MissingArtifact: missingArtifact, MissingBlobIds: blobIDs}, nil
 }
+
+// DeleteBlobs removes blobs by IDs
+func (s *CacheServer) DeleteBlobs(_ context.Context, in *rpcCache.DeleteBlobsRequest) (*google_protobuf.Empty, error) {
+	blobIDs := rpc.ConvertFromDeleteBlobsRequest(in)
+	if err := s.cache.DeleteBlobs(blobIDs); err != nil {
+		return nil, xerrors.Errorf("failed to remove a blobs: %w", err)
+	}
+	return &google_protobuf.Empty{}, nil
+}

pkg/scanner/scan.go

@@ -110,6 +110,11 @@ func (s Scanner) ScanArtifact(ctx context.Context, options types.ScanOptions) (t
 	if err != nil {
 		return types.Report{}, xerrors.Errorf("failed analysis: %w", err)
 	}
+	defer func() {
+		if err := s.artifact.Clean(artifactInfo); err != nil {
+			log.Logger.Warnf("Failed to clean the artifact %q: %v", artifactInfo.Name, err)
+		}
+	}()
 
 	results, osFound, err := s.driver.Scan(artifactInfo.Name, artifactInfo.ID, artifactInfo.BlobIDs, options)
 	if err != nil {

rpc/cache/service.pb.go

@@ -11,6 +11,7 @@ service Cache {
   rpc PutArtifact(PutArtifactRequest) returns (google.protobuf.Empty);
   rpc PutBlob(PutBlobRequest) returns (google.protobuf.Empty);
   rpc MissingBlobs(MissingBlobsRequest) returns (MissingBlobsResponse);
+  rpc DeleteBlobs(DeleteBlobsRequest) returns (google.protobuf.Empty);
 }
 
 message ArtifactInfo {
@@ -59,3 +60,7 @@ message MissingBlobsResponse {
   bool            missing_artifact = 1;
   repeated string missing_blob_ids = 2;
 }
+
+message DeleteBlobsRequest {
+  repeated string blob_ids = 1;
+}