[content-service] Retry git clone

Author: corneliusludmann

components/content-service/go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	cloud.google.com/go/storage v1.14.0
 	github.com/Microsoft/hcsshim v0.8.15 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5 // indirect
 	github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
 	github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000

components/content-service/go.sum

@@ -118,6 +118,7 @@ github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8n
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=

components/content-service/pkg/initializer/git.go

@@ -10,7 +10,9 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
+	"github.com/cenkalti/backoff"
 	"github.com/opentracing/opentracing-go"
 	"golang.org/x/xerrors"
 
@@ -66,14 +68,36 @@ func (ws *GitInitializer) Run(ctx context.Context, mappings []archive.IDMapping)
 		return
 	}
 
-	if err := os.MkdirAll(ws.Location, 0770); err != nil {
-		return src, xerrors.Errorf("git initializer: %w", err)
+	gitClone := func() error {
+		if err := os.MkdirAll(ws.Location, 0770); err != nil {
+			return err
+		}
+
+		log.WithField("stage", "init").WithField("location", ws.Location).Debug("Running git clone on workspace")
+		return ws.Clone(ctx)
+	}
+	onGitCloneFailure := func(e error, d time.Duration) {
+		if err := os.RemoveAll(ws.Location); err != nil {
+			log.
+				WithField("stage", "init").
+				WithField("location", ws.Location).
+				WithError(err).
+				Error("Cleaning workspace location failed.")
+		}
+		log.
+			WithField("stage", "init").
+			WithField("location", ws.Location).
+			WithField("sleepTime", d).
+			WithError(e).
+			Debugf("Running git clone on workspace failed. Retrying in %s ...", d)
 	}
 
-	log.WithField("stage", "init").WithField("location", ws.Location).Debug("Running git clone on workspace")
-	if err := ws.Clone(ctx); err != nil {
+	b := backoff.NewExponentialBackOff()
+	b.MaxElapsedTime = 5 * time.Minute
+	if err = backoff.RetryNotify(gitClone, b, onGitCloneFailure); err != nil {
 		return src, xerrors.Errorf("git initializer: %w", err)
 	}
+
 	if ws.Chown {
 		// TODO (aledbf): refactor to remove the need of manual chown
 		args := []string{"-R", "-L", "gitpod", ws.Location}