[WIP] support slow OS commands.

internal/gps/cmd.go

@@ -8,12 +8,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/Masterminds/vcs"
+	"strconv"
 )
 
 // monitoredCmd wraps a cmd and will keep monitoring the process until it
@@ -196,6 +198,25 @@ func runFromRepoDir(ctx context.Context, repo vcs.Repo, timeout time.Duration, c
 	return c.combinedOutput(ctx)
 }
 
+func getIntEnv(envName string, defaultVal int) int {
+	val, found := os.LookupEnv(envName)
+
+	num, err := strconv.Atoi(val)
+	if found && (err == nil) {
+		return num
+	}
+
+	return defaultVal
+}
+
+func getExpensiveCmdTimeout() time.Duration {
+	return time.Duration(getIntEnv("DEP_EXP_CMD_TIMEOUT", 2)) * time.Minute
+}
+
+func getDefaultCmdTimeout() time.Duration {
+	return time.Duration(getIntEnv("DEP_CMD_TIMEOUT", 10)) * time.Second
+}
+
 const (
 	// expensiveCmdTimeout is meant to be used in a command that is expensive
 	// in terms of computation and we know it will take long or one that uses

internal/gps/strip_vendor.go

@@ -7,6 +7,7 @@
 package gps
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 )
@@ -33,7 +34,8 @@ func stripVendor(path string, info os.FileInfo, err error) error {
 
 		if info.IsDir() {
 			if err := os.RemoveAll(path); err != nil {
-				return err
+				fmt.Printf("could not remove %s completly: %s, skipping\n", path, err)
+				// return err
 			}
 			return filepath.SkipDir
 		}

internal/gps/vcs_repo.go

@@ -117,13 +117,13 @@ func (r *gitRepo) defendAgainstSubmodules(ctx context.Context) error {
 
 	// Now, do a special extra-aggressive clean in case changing versions caused
 	// one or more submodules to go away.
-	out, err = runFromRepoDir(ctx, r, defaultCmdTimeout, "git", "clean", "-x", "-d", "-f", "-f")
+	out, err = runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "git", "clean", "-x", "-d", "-f", "-f")
 	if err != nil {
 		return newVcsLocalErrorOr("unexpected error while defensively cleaning up after possible derelict submodule directories", err, string(out))
 	}
 
 	// Then, repeat just in case there are any nested submodules that went away.
-	out, err = runFromRepoDir(ctx, r, defaultCmdTimeout, "git", "submodule", "foreach", "--recursive", "git", "clean", "-x", "-d", "-f", "-f")
+	out, err = runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "git", "submodule", "foreach", "--recursive", "git", "clean", "-x", "-d", "-f", "-f")
 	if err != nil {
 		return newVcsLocalErrorOr("unexpected error while defensively cleaning up after possible derelict nested submodule directories", err, string(out))
 	}
@@ -250,7 +250,7 @@ func (r *svnRepo) CommitInfo(id string) (*vcs.CommitInfo, error) {
 			Commit commit `xml:"entry>commit"`
 		}
 
-		out, err := runFromRepoDir(ctx, r, defaultCmdTimeout, "svn", "info", "-r", id, "--xml")
+		out, err := runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "svn", "info", "-r", id, "--xml")
 		if err != nil {
 			return nil, newVcsLocalErrorOr("unable to retrieve commit information", err, string(out))
 		}
@@ -267,7 +267,7 @@ func (r *svnRepo) CommitInfo(id string) (*vcs.CommitInfo, error) {
 		}
 	}
 
-	out, err := runFromRepoDir(ctx, r, defaultCmdTimeout, "svn", "log", "-r", id, "--xml")
+	out, err := runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "svn", "log", "-r", id, "--xml")
 	if err != nil {
 		return nil, newVcsRemoteErrorOr("unable to retrieve commit information", err, string(out))
 	}

internal/gps/vcs_source.go

@@ -141,7 +141,7 @@ func (s *gitSource) exportRevisionTo(ctx context.Context, rev Revision, to strin
 	// could have an err here...but it's hard to imagine how?
 	defer fs.RenameWithFallback(bak, idx)
 
-	out, err := runFromRepoDir(ctx, r, defaultCmdTimeout, "git", "read-tree", rev.String())
+	out, err := runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "git", "read-tree", rev.String())
 	if err != nil {
 		return fmt.Errorf("%s: %s", out, err)
 	}
@@ -157,7 +157,7 @@ func (s *gitSource) exportRevisionTo(ctx context.Context, rev Revision, to strin
 	// though we have a bunch of housekeeping to do to set up, then tear
 	// down, the sparse checkout controls, as well as restore the original
 	// index and HEAD.
-	out, err = runFromRepoDir(ctx, r, defaultCmdTimeout, "git", "checkout-index", "-a", "--prefix="+to)
+	out, err = runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "git", "checkout-index", "-a", "--prefix="+to)
 	if err != nil {
 		return fmt.Errorf("%s: %s", out, err)
 	}
@@ -382,15 +382,15 @@ func (s *bzrSource) listVersions(ctx context.Context) ([]PairedVersion, error) {
 	}
 
 	// Now, list all the tags
-	out, err := runFromRepoDir(ctx, r, defaultCmdTimeout, "bzr", "tags", "--show-ids", "-v")
+	out, err := runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "bzr", "tags", "--show-ids", "-v")
 	if err != nil {
 		return nil, fmt.Errorf("%s: %s", err, string(out))
 	}
 
 	all := bytes.Split(bytes.TrimSpace(out), []byte("\n"))
 
 	var branchrev []byte
-	branchrev, err = runFromRepoDir(ctx, r, defaultCmdTimeout, "bzr", "version-info", "--custom", "--template={revision_id}", "--revision=branch:.")
+	branchrev, err = runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "bzr", "version-info", "--custom", "--template={revision_id}", "--revision=branch:.")
 	br := string(branchrev)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %s", err, br)
@@ -462,7 +462,7 @@ func (s *hgSource) listVersions(ctx context.Context) ([]PairedVersion, error) {
 	}
 
 	// Now, list all the tags
-	out, err := runFromRepoDir(ctx, r, defaultCmdTimeout, "hg", "tags", "--debug", "--verbose")
+	out, err := runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "hg", "tags", "--debug", "--verbose")
 	if err != nil {
 		return nil, fmt.Errorf("%s: %s", err, string(out))
 	}
@@ -496,7 +496,7 @@ func (s *hgSource) listVersions(ctx context.Context) ([]PairedVersion, error) {
 	// bookmarks next, because the presence of the magic @ bookmark has to
 	// determine how we handle the branches
 	var magicAt bool
-	out, err = runFromRepoDir(ctx, r, defaultCmdTimeout, "hg", "bookmarks", "--debug")
+	out, err = runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "hg", "bookmarks", "--debug")
 	if err != nil {
 		// better nothing than partial and misleading
 		return nil, fmt.Errorf("%s: %s", err, string(out))
@@ -529,7 +529,7 @@ func (s *hgSource) listVersions(ctx context.Context) ([]PairedVersion, error) {
 		}
 	}
 
-	out, err = runFromRepoDir(ctx, r, defaultCmdTimeout, "hg", "branches", "-c", "--debug")
+	out, err = runFromRepoDir(ctx, r, getDefaultCmdTimeout(), "hg", "branches", "-c", "--debug")
 	if err != nil {
 		// better nothing than partial and misleading
 		return nil, fmt.Errorf("%s: %s", err, string(out))