implement layer caching

Author: nkubala

cmd/root.go

@@ -25,6 +25,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/google/go-containerregistry/authn"
 	"github.com/google/go-containerregistry/name"
@@ -129,14 +130,17 @@ func checkIfValidAnalyzer(_ []string) error {
 }
 
 func getImageForName(imageName string) (pkgutil.Image, error) {
-	logrus.Infof("getting image for name %s", imageName)
+	logrus.Infof("retrieving image: %s", imageName)
 	var img v1.Image
 	var err error
 	if pkgutil.IsTar(imageName) {
+		start := time.Now()
 		img, err = tarball.ImageFromPath(imageName, nil)
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("retrieving image from tar took %f seconds", elapsed.Seconds())
 	}
 
 	if strings.HasPrefix(imageName, DaemonPrefix) {
@@ -148,10 +152,15 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 			return pkgutil.Image{}, err
 		}
 
-		img, err = daemon.Image(ref, &daemon.ReadOptions{})
+		start := time.Now()
+		img, err = daemon.Image(ref, &daemon.ReadOptions{
+			Buffer: true,
+		})
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("retrieving image from daemon took %f seconds", elapsed.Seconds())
 	} else {
 		// either has remote prefix or has no prefix, in which case we force remote
 		imageName = strings.Replace(imageName, RemotePrefix, "", -1)
@@ -163,21 +172,28 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		start := time.Now()
 		img, err = remote.Image(ref, auth, http.DefaultTransport)
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("retrieving remote image took %f seconds", elapsed.Seconds())
 	}
 
 	// create tempdir and extract fs into it
 	var layers []pkgutil.Layer
 	if includeLayers() {
+		start := time.Now()
 		imgLayers, err := img.Layers()
 		if err != nil {
 			return pkgutil.Image{}, err
 		}
 		for _, layer := range imgLayers {
-			path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
+			layerStart := time.Now()
+			diffID, err := layer.DiffID()
+			logrus.Infof("layer digest: %s", diffID.String())
+			path, err := getExtractPathForName(diffID.String())
 			if err != nil {
 				return pkgutil.Image{
 					Layers: layers,
@@ -191,43 +207,53 @@ func getImageForName(imageName string) (pkgutil.Image, error) {
 			layers = append(layers, pkgutil.Layer{
 				FSPath: path,
 			})
+			elapsed := time.Now().Sub(layerStart)
+			logrus.Infof("time elapsed retrieving layer: %fs", elapsed.Seconds())
 		}
+		elapsed := time.Now().Sub(start)
+		logrus.Infof("time elapsed retrieving image layers: %fs", elapsed.Seconds())
 	}
 
+	path, err := getExtractPathForName(imageName)
+	// extract fs into provided dir
+	if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
+		return pkgutil.Image{
+			FSPath: path,
+			Layers: layers,
+		}, err
+	}
+	return pkgutil.Image{
+		Image:  img,
+		Source: imageName,
+		FSPath: path,
+		Layers: layers,
+	}, nil
+}
+
+func getExtractPathForName(name string) (string, error) {
 	var path string
+	var err error
 	if !noCache {
-		path, err = cacheDir(imageName)
+		path, err = cacheDir(name)
 		if err != nil {
-			return pkgutil.Image{}, err
+			return "", err
 		}
 		// if cachedir doesn't exist, create it
 		if _, err := os.Stat(path); err != nil && os.IsNotExist(err) {
 			err = os.MkdirAll(path, 0700)
 			if err != nil {
-				return pkgutil.Image{}, err
+				return "", err
 			}
 			logrus.Infof("Image fs cached at %s", path)
 		}
 	} else {
 		// otherwise, create tempdir
-		path, err = ioutil.TempDir("", strings.Replace(imageName, "/", "", -1))
+		path, err = ioutil.TempDir("", strings.Replace(name, "/", "", -1))
 		if err != nil {
-			return pkgutil.Image{}, err
+			return "", err
 		}
 	}
-	// extract fs into provided dir
-	if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil {
-		return pkgutil.Image{
-			FSPath: path,
-			Layers: layers,
-		}, err
-	}
-	return pkgutil.Image{
-		Image:  img,
-		Source: imageName,
-		FSPath: path,
-		Layers: layers,
-	}, nil
+	return path, nil
 }
 
 func includeLayers() bool {

pkg/util/tar_utils.go

@@ -83,16 +83,17 @@ func unpackTar(tr *tar.Reader, path string, whitelist []string) error {
 			// It's possible for a file to be included before the directory it's in is created.
 			baseDir := filepath.Dir(target)
 			if _, err := os.Stat(baseDir); os.IsNotExist(err) {
-				logrus.Debugf("baseDir %s for file %s does not exist. Creating.", baseDir, target)
+				logrus.Debugf("baseDir %s for file %s does not exist. Creating", baseDir, target)
 				if err := os.MkdirAll(baseDir, 0755); err != nil {
 					return err
 				}
 			}
 			// It's possible we end up creating files that can't be overwritten based on their permissions.
 			// Explicitly delete an existing file before continuing.
 			if _, err := os.Stat(target); !os.IsNotExist(err) {
-				logrus.Debugf("Removing %s for overwrite.", target)
+				logrus.Debugf("Removing %s for overwrite", target)
 				if err := os.Remove(target); err != nil {
+					logrus.Errorf("error removing file %s", target)
 					return err
 				}
 			}
@@ -117,7 +118,7 @@ func unpackTar(tr *tar.Reader, path string, whitelist []string) error {
 			// It's possible we end up creating files that can't be overwritten based on their permissions.
 			// Explicitly delete an existing file before continuing.
 			if _, err := os.Stat(target); !os.IsNotExist(err) {
-				logrus.Debugf("Removing %s to create symlink.", target)
+				logrus.Debugf("Removing %s to create symlink", target)
 				if err := os.RemoveAll(target); err != nil {
 					logrus.Debugf("Unable to remove %s: %s", target, err)
 				}
@@ -139,7 +140,7 @@ func unpackTar(tr *tar.Reader, path string, whitelist []string) error {
 	}
 
 	for target, linkname := range hardlinks {
-		logrus.Info("Resolving hard links.")
+		logrus.Info("Resolving hard links")
 		if _, err := os.Stat(linkname); !os.IsNotExist(err) {
 			// If it exists, create the hard link
 			if err := resolveHardlink(linkname, target); err != nil {