internal/gps: add PruneOptions and update WriteDepTree

This commit changes gps.WriteDepTree to be a method on gps.SourceManager.
It also updates the signature to accept a PruneOptions instead of a bool.

The manfiest is updated to accept a new table, prune. This table defines
pruning options accepted by dep. By default, dep will always prune nested
vendor directories. The user can define 3 additional rules:

 * non-go: Prune non-Go files (will keep LICENSE & COPYING files)
 * go-tests: Prune Go test files
 * unused-packages: Prune unused Go packages from dependencies.

The implementation of these flags is not part of this commit.

Signed-off-by: Ibrahim AshShohail <[email protected]>

Author: ibrasho

Gopkg.toml

@@ -17,3 +17,8 @@
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "0.8.0"
+
+[prune]
+  non-go = true
+  go-tests = true
+  unused-packages = true

cmd/dep/prune.go

@@ -97,7 +97,7 @@ func pruneProject(p *dep.Project, sm gps.SourceManager, logger *log.Logger) erro
 	}
 	defer os.RemoveAll(td)
 
-	if err := gps.WriteDepTree(td, p.Lock, sm, true); err != nil {
+	if err := sm.WriteDepTree(td, p.Lock, gps.PruneNestedVendorDirs); err != nil {
 		return err
 	}
 

internal/gps/bridge.go

@@ -23,8 +23,9 @@ type sourceBridge interface {
 	RevisionPresentIn(ProjectIdentifier, Revision) (bool, error)
 	ListPackages(ProjectIdentifier, Version) (pkgtree.PackageTree, error)
 	GetManifestAndLock(ProjectIdentifier, Version, ProjectAnalyzer) (Manifest, Lock, error)
-	ExportProject(ProjectIdentifier, Version, string) error
 	DeduceProjectRoot(ip string) (ProjectRoot, error)
+	ExportProject(ProjectIdentifier, Version, string) error
+	WriteDepTree(basedir string, l Lock, opts PruneOptions) error
 
 	//sourceExists(ProjectIdentifier) (bool, error)
 	//syncSourceFor(ProjectIdentifier) error
@@ -165,6 +166,10 @@ func (b *bridge) ExportProject(id ProjectIdentifier, v Version, path string) err
 	panic("bridge should never be used to ExportProject")
 }
 
+func (b *bridge) WriteDepTree(basedir string, l Lock, opts PruneOptions) error {
+	panic("bridge should never be used to WriteDepTree")
+}
+
 // verifyRoot ensures that the provided path to the project root is in good
 // working condition. This check is made only once, at the beginning of a solve
 // run.

internal/gps/example.go

@@ -54,7 +54,7 @@ func main() {
 		// If no failure, blow away the vendor dir and write a new one out,
 		// stripping nested vendor directories as we go.
 		os.RemoveAll(filepath.Join(root, "vendor"))
-		gps.WriteDepTree(filepath.Join(root, "vendor"), solution, sourcemgr, true)
+		sourcemgr.WriteDepTree(filepath.Join(root, "vendor"), solution, true)
 	}
 }
 

internal/gps/manager_test.go

@@ -883,3 +883,66 @@ func TestSupervisor(t *testing.T) {
 		return nil
 	})
 }
+
+func testWriteDepTree(t *testing.T) {
+	t.Parallel()
+
+	// This test is a bit slow, skip it on -short
+	if testing.Short() {
+		t.Skip("Skipping dep tree writing test in short mode")
+	}
+	requiresBins(t, "git", "hg", "bzr")
+
+	tmp, err := ioutil.TempDir("", "writetree")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %s", err)
+	}
+	defer os.RemoveAll(tmp)
+
+	r := solution{
+		att: 1,
+		p: []LockedProject{
+			pa2lp(atom{
+				id: pi("github.com/sdboyer/testrepo"),
+				v:  NewBranch("master").Pair(Revision("4d59fb584b15a94d7401e356d2875c472d76ef45")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("launchpad.net/govcstestbzrrepo"),
+				v:  NewVersion("1.0.0").Pair(Revision("[email protected]")),
+			}, nil),
+			pa2lp(atom{
+				id: pi("bitbucket.org/sdboyer/withbm"),
+				v:  NewVersion("v1.0.0").Pair(Revision("aa110802a0c64195d0a6c375c9f66668827c90b4")),
+			}, nil),
+		},
+	}
+
+	sm, clean := mkNaiveSM(t)
+	defer clean()
+
+	// Trigger simultaneous fetch of all three to speed up test execution time
+	for _, p := range r.p {
+		go func(pi ProjectIdentifier) {
+			sm.SyncSourceFor(pi)
+		}(p.pi)
+	}
+
+	// nil lock/result should err immediately
+	if err := sm.WriteDepTree(tmp, nil, PruneNestedVendorDirs); err == nil {
+		t.Errorf("Should error if nil lock is passed to WriteDepTree")
+	}
+
+	if err := sm.WriteDepTree(tmp, r, PruneNestedVendorDirs); err != nil {
+		t.Errorf("Unexpected error while creating vendor tree: %s", err)
+	}
+
+	if _, err = os.Stat(filepath.Join(tmp, "github.com", "sdboyer", "testrepo")); err != nil {
+		t.Errorf("Directory for github.com/sdboyer/testrepo does not exist")
+	}
+	if _, err = os.Stat(filepath.Join(tmp, "launchpad.net", "govcstestbzrrepo")); err != nil {
+		t.Errorf("Directory for launchpad.net/govcstestbzrrepo does not exist")
+	}
+	if _, err = os.Stat(filepath.Join(tmp, "bitbucket.org", "sdboyer", "withbm")); err != nil {
+		t.Errorf("Directory for bitbucket.org/sdboyer/withbm does not exist")
+	}
+}

internal/gps/prune.go

@@ -0,0 +1,20 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gps
+
+// PruneOptions represents the pruning options used to write the dependecy tree.
+type PruneOptions uint8
+
+const (
+	// PruneNestedVendorDirs indicates if nested vendor directories should be pruned.
+	PruneNestedVendorDirs = 1 << iota
+	// PruneNonGoFiles indicates if non-Go files should be pruned.
+	// LICENSE & COPYING files are kept for convience.
+	PruneNonGoFiles
+	// PruneGoTestFiles indicates if Go test files should be pruned.
+	PruneGoTestFiles
+	// PruneUnusedPackages indicates if unused Go packages should be pruned.
+	PruneUnusedPackages
+)

internal/gps/result.go

@@ -4,12 +4,6 @@
 
 package gps
 
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
 // A Solution is returned by a solver run. It is mostly just a Lock, with some
 // additional methods that report information about the solve run.
 type Solution interface {
@@ -42,42 +36,6 @@ type solution struct {
 	solv Solver
 }
 
-// WriteDepTree takes a basedir and a Lock, and exports all the projects
-// listed in the lock to the appropriate target location within the basedir.
-//
-// If the goal is to populate a vendor directory, basedir should be the absolute
-// path to that vendor directory, not its parent (a project root, typically).
-//
-// It requires a SourceManager to do the work, and takes a flag indicating
-// whether or not to strip vendor directories contained in the exported
-// dependencies.
-func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool) error {
-	if l == nil {
-		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
-	}
-
-	err := os.MkdirAll(basedir, 0777)
-	if err != nil {
-		return err
-	}
-
-	// TODO(sdboyer) parallelize
-	for _, p := range l.Projects() {
-		to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
-
-		err = sm.ExportProject(p.Ident(), p.Version(), to)
-		if err != nil {
-			removeAll(basedir)
-			return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
-		}
-		if sv {
-			filepath.Walk(to, stripVendor)
-		}
-	}
-
-	return nil
-}
-
 func (r solution) Projects() []LockedProject {
 	return r.p
 }

internal/gps/result_test.go

@@ -5,10 +5,8 @@
 package gps
 
 import (
-	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"testing"
 )
 
@@ -46,71 +44,6 @@ func init() {
 	*/
 }
 
-func testWriteDepTree(t *testing.T) {
-	t.Parallel()
-
-	// This test is a bit slow, skip it on -short
-	if testing.Short() {
-		t.Skip("Skipping dep tree writing test in short mode")
-	}
-	requiresBins(t, "git", "hg", "bzr")
-
-	tmp, err := ioutil.TempDir("", "writetree")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %s", err)
-	}
-	defer os.RemoveAll(tmp)
-
-	r := solution{
-		att: 1,
-		p: []LockedProject{
-			pa2lp(atom{
-				id: pi("github.com/sdboyer/testrepo"),
-				v:  NewBranch("master").Pair(Revision("4d59fb584b15a94d7401e356d2875c472d76ef45")),
-			}, nil),
-			pa2lp(atom{
-				id: pi("launchpad.net/govcstestbzrrepo"),
-				v:  NewVersion("1.0.0").Pair(Revision("[email protected]")),
-			}, nil),
-			pa2lp(atom{
-				id: pi("bitbucket.org/sdboyer/withbm"),
-				v:  NewVersion("v1.0.0").Pair(Revision("aa110802a0c64195d0a6c375c9f66668827c90b4")),
-			}, nil),
-		},
-	}
-
-	sm, clean := mkNaiveSM(t)
-	defer clean()
-
-	// Trigger simultaneous fetch of all three to speed up test execution time
-	for _, p := range r.p {
-		go func(pi ProjectIdentifier) {
-			sm.SyncSourceFor(pi)
-		}(p.pi)
-	}
-
-	// nil lock/result should err immediately
-	err = WriteDepTree(tmp, nil, sm, true)
-	if err == nil {
-		t.Errorf("Should error if nil lock is passed to WriteDepTree")
-	}
-
-	err = WriteDepTree(tmp, r, sm, true)
-	if err != nil {
-		t.Errorf("Unexpected error while creating vendor tree: %s", err)
-	}
-
-	if _, err = os.Stat(filepath.Join(tmp, "github.com", "sdboyer", "testrepo")); err != nil {
-		t.Errorf("Directory for github.com/sdboyer/testrepo does not exist")
-	}
-	if _, err = os.Stat(filepath.Join(tmp, "launchpad.net", "govcstestbzrrepo")); err != nil {
-		t.Errorf("Directory for launchpad.net/govcstestbzrrepo does not exist")
-	}
-	if _, err = os.Stat(filepath.Join(tmp, "bitbucket.org", "sdboyer", "withbm")); err != nil {
-		t.Errorf("Directory for bitbucket.org/sdboyer/withbm does not exist")
-	}
-}
-
 func BenchmarkCreateVendorTree(b *testing.B) {
 	// We're fs-bound here, so restrict to single parallelism
 	b.SetParallelism(1)
@@ -143,7 +76,7 @@ func BenchmarkCreateVendorTree(b *testing.B) {
 			// ease manual inspection
 			os.RemoveAll(exp)
 			b.StartTimer()
-			err = WriteDepTree(exp, r, sm, true)
+			err = sm.WriteDepTree(exp, r, PruneNestedVendorDirs)
 			b.StopTimer()
 			if err != nil {
 				b.Errorf("unexpected error after %v iterations: %s", i, err)

internal/gps/solve_basic_test.go

@@ -1472,6 +1472,10 @@ func (sm *depspecSourceManager) ExportProject(id ProjectIdentifier, v Version, t
 	return fmt.Errorf("dummy sm doesn't support exporting")
 }
 
+func (sm *depspecSourceManager) WriteDepTree(basedir string, l Lock, options PruneOptions) error {
+	return fmt.Errorf("dummy sm doesn't support WriteDepTree")
+}
+
 func (sm *depspecSourceManager) DeduceProjectRoot(ip string) (ProjectRoot, error) {
 	for _, ds := range sm.allSpecs() {
 		n := string(ds.n)

internal/gps/source_manager.go

@@ -64,10 +64,6 @@ type SourceManager interface {
 	// repository root.
 	GetManifestAndLock(ProjectIdentifier, Version, ProjectAnalyzer) (Manifest, Lock, error)
 
-	// ExportProject writes out the tree of the provided import path, at the
-	// provided version, to the provided directory.
-	ExportProject(ProjectIdentifier, Version, string) error
-
 	// DeduceRootProject takes an import path and deduces the corresponding
 	// project/source root.
 	DeduceProjectRoot(ip string) (ProjectRoot, error)
@@ -80,6 +76,14 @@ type SourceManager interface {
 	// InferConstraint tries to puzzle out what kind of version is given in a string -
 	// semver, a revision, or as a fallback, a plain tag
 	InferConstraint(s string, pi ProjectIdentifier) (Constraint, error)
+
+	// ExportProject writes out the tree of the provided import path, at the
+	// provided version, to the provided directory.
+	ExportProject(ProjectIdentifier, Version, string) error
+
+	// WriteDepTree takes a basedir, a Lock and a PruneOptions, and exports all the
+	// projects listed in the lock to the appropriate target location within basedir.
+	WriteDepTree(basedir string, l Lock, opts PruneOptions) error
 }
 
 // A ProjectAnalyzer is responsible for analyzing a given path for Manifest and
@@ -482,6 +486,46 @@ func (sm *SourceMgr) ExportProject(id ProjectIdentifier, v Version, to string) e
 	return srcg.exportVersionTo(context.TODO(), v, to)
 }
 
+// WriteDepTree takes a basedir, a Lock and a PruneOptions, and exports all the
+// projects listed in the lock to the appropriate target location within basedir.
+//
+// If the goal is to populate a vendor directory, basedir should be the absolute
+// path to that vendor directory, not its parent (a project root, typically).
+func (sm *SourceMgr) WriteDepTree(basedir string, l Lock, options PruneOptions) error {
+	if l == nil {
+		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
+	}
+
+	if err := os.MkdirAll(basedir, 0777); err != nil {
+		return err
+	}
+
+	// TODO(sdboyer) parallelize
+	for _, p := range l.Projects() {
+		to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
+
+		if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
+			removeAll(basedir)
+			return fmt.Errorf("error while exporting %s: %s", p.Ident().ProjectRoot, err)
+		}
+
+		if (options & PruneNestedVendorDirs) != 0 {
+			filepath.Walk(to, stripVendor)
+		}
+		if (options & PruneNonGoFiles) != 0 {
+			// TODO: prune non Go files
+		}
+		if (options & PruneGoTestFiles) != 0 {
+			// TODO: prune Go test files
+		}
+		if (options & PruneUnusedPackages) != 0 {
+			// TODO: prune unused packages
+		}
+	}
+
+	return nil
+}
+
 // DeduceProjectRoot takes an import path and deduces the corresponding
 // project/source root.
 //

internal/gps/version_unifier_test.go

@@ -125,6 +125,10 @@ func (lb lvFixBridge) ExportProject(ProjectIdentifier, Version, string) error {
 	panic("not implemented")
 }
 
+func (lb lvFixBridge) WriteDepTree(basedir string, l Lock, opts PruneOptions) error {
+	panic("not implemented")
+}
+
 func (lb lvFixBridge) DeduceProjectRoot(ip string) (ProjectRoot, error) {
 	panic("not implemented")
 }