Skip to content
This repository was archived by the owner on Mar 27, 2024. It is now read-only.

Add filesystem caching #232

Merged
merged 1 commit into from
Jun 12, 2018
Merged

Add filesystem caching #232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions Gopkg.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions cmd/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ go_library(
"//vendor/github.com/google/go-containerregistry/pkg/v1/daemon:go_default_library",
"//vendor/github.com/google/go-containerregistry/pkg/v1/remote:go_default_library",
"//vendor/github.com/google/go-containerregistry/pkg/v1/tarball:go_default_library",
"//vendor/github.com/mitchellh/go-homedir:go_default_library",
"//vendor/github.com/sirupsen/logrus:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
Expand Down
8 changes: 4 additions & 4 deletions cmd/analyze.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func analyzeImage(imageName string, analyzerArgs []string) error {
return err
}

if !save {
if noCache && !save {
defer pkgutil.CleanupImage(image)
}
if err != nil {
Expand All @@ -79,11 +79,11 @@ func analyzeImage(imageName string, analyzerArgs []string) error {
return fmt.Errorf("Error performing image analysis: %s", err)
}

output.PrintToStdErr("Retrieving analyses\n")
logrus.Info("retrieving analyses")
outputResults(analyses)

if save {
logrus.Infof("Image was saved at %s", image.FSPath)
if noCache && save {
logrus.Infof("image was saved at %s", image.FSPath)
}

return nil
Expand Down
10 changes: 5 additions & 5 deletions cmd/diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
var wg sync.WaitGroup
wg.Add(2)

output.PrintToStdErr("Starting diff on images %s and %s, using differs: %s\n", image1Arg, image2Arg, diffArgs)
logrus.Infof("starting diff on images %s and %s, using differs: %s\n", image1Arg, image2Arg, diffArgs)

imageMap := map[string]*pkgutil.Image{
image1Arg: {},
Expand All @@ -97,12 +97,12 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
}
wg.Wait()

if !save {
if noCache && !save {
defer pkgutil.CleanupImage(*imageMap[image1Arg])
defer pkgutil.CleanupImage(*imageMap[image2Arg])
}

output.PrintToStdErr("Computing diffs\n")
logrus.Info("computing diffs")
req := differs.DiffRequest{
Image1: *imageMap[image1Arg],
Image2: *imageMap[image2Arg],
Expand All @@ -114,14 +114,14 @@ func diffImages(image1Arg, image2Arg string, diffArgs []string) error {
outputResults(diffs)

if filename != "" {
output.PrintToStdErr("Computing filename diffs\n")
logrus.Info("computing filename diffs")
err := diffFile(imageMap[image1Arg], imageMap[image2Arg])
if err != nil {
return err
}
}

if save {
if noCache && save {
logrus.Infof("Images were saved at %s and %s", imageMap[image1Arg].FSPath,
imageMap[image2Arg].FSPath)
}
Expand Down
87 changes: 79 additions & 8 deletions cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ import (
"io/ioutil"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"time"

"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
Expand All @@ -35,14 +37,17 @@ import (
pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
"github.com/GoogleContainerTools/container-diff/util"
"github.com/google/go-containerregistry/pkg/v1"
homedir "github.com/mitchellh/go-homedir"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)

var json bool

var save bool
var types diffTypes
var noCache bool

var LogLevel string
var format string
Expand Down Expand Up @@ -125,14 +130,17 @@ func checkIfValidAnalyzer(_ []string) error {
}

func getImageForName(imageName string) (pkgutil.Image, error) {
logrus.Infof("getting image for name %s", imageName)
logrus.Infof("retrieving image: %s", imageName)
var img v1.Image
var err error
if pkgutil.IsTar(imageName) {
start := time.Now()
img, err = tarball.ImageFromPath(imageName, nil)
if err != nil {
return pkgutil.Image{}, err
}
elapsed := time.Now().Sub(start)
logrus.Infof("retrieving image from tar took %f seconds", elapsed.Seconds())
}

if strings.HasPrefix(imageName, DaemonPrefix) {
Expand All @@ -144,10 +152,16 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
return pkgutil.Image{}, err
}

img, err = daemon.Image(ref, &daemon.ReadOptions{})
start := time.Now()
// TODO(nkubala): specify gzip.NoCompression here when functional options are supported
img, err = daemon.Image(ref, &daemon.ReadOptions{
Buffer: true,
})
if err != nil {
return pkgutil.Image{}, err
}
elapsed := time.Now().Sub(start)
logrus.Infof("retrieving image from daemon took %f seconds", elapsed.Seconds())
} else {
// either has remote prefix or has no prefix, in which case we force remote
imageName = strings.Replace(imageName, RemotePrefix, "", -1)
Expand All @@ -159,22 +173,27 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
if err != nil {
return pkgutil.Image{}, err
}
start := time.Now()
img, err = remote.Image(ref, auth, http.DefaultTransport)
if err != nil {
return pkgutil.Image{}, err
}
elapsed := time.Now().Sub(start)
logrus.Infof("retrieving remote image took %f seconds", elapsed.Seconds())
}
// TODO(nkubala): implement caching

// create tempdir and extract fs into it
var layers []pkgutil.Layer
if includeLayers() {
start := time.Now()
imgLayers, err := img.Layers()
if err != nil {
return pkgutil.Image{}, err
}
for _, layer := range imgLayers {
path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
layerStart := time.Now()
digest, err := layer.Digest()
path, err := getExtractPathForName(digest.String())
if err != nil {
return pkgutil.Image{
Layers: layers,
Expand All @@ -188,12 +207,15 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
layers = append(layers, pkgutil.Layer{
FSPath: path,
})
elapsed := time.Now().Sub(layerStart)
logrus.Infof("time elapsed retrieving layer: %fs", elapsed.Seconds())
}
elapsed := time.Now().Sub(start)
logrus.Infof("time elapsed retrieving image layers: %fs", elapsed.Seconds())
}
path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
if err != nil {
return pkgutil.Image{}, err
}

path, err := getExtractPathForImage(imageName, img)
// extract fs into provided dir
if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
return pkgutil.Image{
FSPath: path,
Expand All @@ -208,6 +230,44 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
}, nil
}

func getExtractPathForImage(imageName string, image v1.Image) (string, error) {
start := time.Now()
digest, err := image.Digest()
if err != nil {
return "", err
}
elapsed := time.Now().Sub(start)
logrus.Infof("time elapsed retrieving image digest: %fs", elapsed.Seconds())
return getExtractPathForName(pkgutil.RemoveTag(imageName) + "@" + digest.String())
}

func getExtractPathForName(name string) (string, error) {
var path string
var err error
if !noCache {
path, err = cacheDir(name)
if err != nil {
return "", err
}
// if cachedir doesn't exist, create it
if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
err = os.MkdirAll(path, 0700)
if err != nil {
return "", err
}
logrus.Infof("caching filesystem at %s", path)
}
} else {
// otherwise, create tempdir
logrus.Infof("skipping caching")
path, err = ioutil.TempDir("", strings.Replace(name, "/", "", -1))
if err != nil {
return "", err
}
}
return path, nil
}

func includeLayers() bool {
for _, t := range types {
if t == "layer" {
Expand All @@ -217,6 +277,16 @@ func includeLayers() bool {
return false
}

func cacheDir(imageName string) (string, error) {
dir, err := homedir.Dir()
if err != nil {
return "", err
}
rootDir := filepath.Join(dir, ".container-diff", "cache")
imageName = strings.Replace(imageName, string(os.PathSeparator), "", -1)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably only cache on digests, not tags. If there's a tag, let's resolve it to a digest then cache.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In general I agree, but what about images that were just built locally and don't have digests? I could just do a docker build -t "foo:bar" . and that image won't have a digest.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, we need some way to cache at a content-addressable level, otherwise you risk stale images for things like "latest". What happens if you call Digest() on an image from the daemon? It should work.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The latest changes from go-containerregistry fix this (it used to take on the order of 7 minutes to call Digest()), so we're good here.

return filepath.Join(rootDir, filepath.Clean(imageName)), nil
}

func init() {
RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.")
RootCmd.PersistentFlags().StringVarP(&format, "format", "", "", "Format to output diff in.")
Expand Down Expand Up @@ -254,4 +324,5 @@ func addSharedFlags(cmd *cobra.Command) {
cmd.Flags().VarP(&types, "type", "t", "This flag sets the list of analyzer types to use. Set it repeatedly to use multiple analyzers.")
cmd.Flags().BoolVarP(&save, "save", "s", false, "Set this flag to save rather than remove the final image filesystems on exit.")
cmd.Flags().BoolVarP(&util.SortSize, "order", "o", false, "Set this flag to sort any file/package results by descending size. Otherwise, they will be sorted by name.")
cmd.Flags().BoolVarP(&noCache, "no-cache", "n", false, "Set this to force retrieval of image filesystem on each run.")
}
16 changes: 16 additions & 0 deletions pkg/util/fs_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package util

import (
"bytes"
"io"
"io/ioutil"
"os"
"path/filepath"
Expand Down Expand Up @@ -191,3 +192,18 @@ func HasFilepathPrefix(path, prefix string) bool {
}
return true
}

// given a path to a directory, check if it has any contents
func DirIsEmpty(path string) (bool, error) {
f, err := os.Open(path)
if err != nil {
return false, err
}
defer f.Close()

_, err = f.Readdir(1)
if err == io.EOF {
return true, nil
}
return false, err
}
24 changes: 23 additions & 1 deletion pkg/util/image_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ import (
"github.com/sirupsen/logrus"
)

const tagRegexStr = ".*:([^/]+$)"

type Layer struct {
FSPath string
}
Expand Down Expand Up @@ -83,7 +85,16 @@ func GetFileSystemForLayer(layer v1.Layer, root string, whitelist []string) erro
}

// unpack image filesystem to local disk
// if provided directory is not empty, do nothing
func GetFileSystemForImage(image v1.Image, root string, whitelist []string) error {
empty, err := DirIsEmpty(root)
if err != nil {
return err
}
if !empty {
logrus.Infof("using cached filesystem in %s", root)
return nil
}
if err := unpackTar(tar.NewReader(mutate.Extract(image)), root, whitelist); err != nil {
return err
}
Expand Down Expand Up @@ -134,6 +145,17 @@ func copyToFile(outfile string, r io.Reader) error {

// checks to see if an image string contains a tag.
func HasTag(image string) bool {
tagRegex := regexp.MustCompile(".*:[^/]+$")
tagRegex := regexp.MustCompile(tagRegexStr)
return tagRegex.MatchString(image)
}

// returns a raw image name with the tag removed
func RemoveTag(image string) string {
if !HasTag(image) {
return image
}
tagRegex := regexp.MustCompile(tagRegexStr)
parts := tagRegex.FindStringSubmatch(image)
tag := parts[len(parts)-1]
return image[0 : len(image)-len(tag)-1]
}
Loading