PROTOTYPE: Add support for templated campaign specs

internal/campaigns/executor_test.go

@@ -98,6 +98,23 @@ func TestExecutor_Integration(t *testing.T) {
 			executorTimeout: 100 * time.Millisecond,
 			wantErrInclude:  "execution in github.com/sourcegraph/src-cli failed: Timeout reached. Execution took longer than 100ms.",
 		},
+		{
+			name:  "templated",
+			repos: []*graphql.Repository{srcCLIRepo},
+			archives: []mockRepoArchive{
+				{repo: srcCLIRepo, files: map[string]string{
+					"README.md": "# Welcome to the README\n",
+					"main.go":   "package main\n\nfunc main() {\n\tfmt.Println(     \"Hello World\")\n}\n",
+				}},
+			},
+			steps: []Step{
+				{Run: `go fmt main.go`, Container: "doesntmatter:13"},
+				{Run: `touch modified-${{ .PreviousStep.ModifiedFiles }}.md`, Container: "alpine:13"},
+			},
+			wantFilesChanged: map[string][]string{
+				srcCLIRepo.ID: []string{"main.go", "modified-main.go.md"},
+			},
+		},
 	}
 
 	for _, tc := range tests {
@@ -170,8 +187,13 @@ func TestExecutor_Integration(t *testing.T) {
 
 				diffsByName := map[string]*diff.FileDiff{}
 				for _, fd := range fileDiffs {
-					diffsByName[fd.OrigName] = fd
+					if fd.NewName == "/dev/null" {
+						diffsByName[fd.OrigName] = fd
+					} else {
+						diffsByName[fd.NewName] = fd
+					}
 				}
+
 				for _, file := range wantFiles {
 					if _, ok := diffsByName[file]; !ok {
 						t.Errorf("%s was not changed", file)

internal/campaigns/graphql/repository.go

@@ -1,6 +1,9 @@
 package graphql
 
-import "strings"
+import (
+	"sort"
+	"strings"
+)
 
 const RepositoryFieldsFragment = `
 fragment repositoryFields on Repository {
@@ -30,6 +33,8 @@ type Repository struct {
 	URL                string
 	ExternalRepository struct{ ServiceType string }
 	DefaultBranch      *Branch
+
+	FileMatches map[string]bool
 }
 
 func (r *Repository) BaseRef() string {
@@ -43,3 +48,16 @@ func (r *Repository) Rev() string {
 func (r *Repository) Slug() string {
 	return strings.ReplaceAll(r.Name, "/", "-")
 }
+
+func (r *Repository) SearchResultPaths() (list fileMatchPathList) {
+	var files []string
+	for f := range r.FileMatches {
+		files = append(files, f)
+	}
+	sort.Strings(files)
+	return fileMatchPathList(files)
+}
+
+type fileMatchPathList []string
+
+func (f fileMatchPathList) String() string { return strings.Join(f, " ") }

internal/campaigns/run_steps.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/hashicorp/go-multierror"
@@ -60,9 +61,10 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
 		return nil, errors.Wrap(err, "git commit failed")
 	}
 
+	results := make([]StepResult, len(steps))
+
 	for i, step := range steps {
 		logger.Logf("[Step %d] docker run %s %q", i+1, step.Container, step.Run)
-		reportProgress(step.Run)
 
 		cidFile, err := ioutil.TempFile(tempDir, repo.Slug()+"-container-id")
 		if err != nil {
@@ -93,11 +95,25 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
 		}
 		hostTemp := fp.Name()
 		defer os.Remove(hostTemp)
-		if _, err := fp.WriteString(step.Run); err != nil {
-			return nil, errors.Wrapf(err, "writing to temporary file %q", hostTemp)
+
+		stepContext := StepContext{Repository: repo}
+		if i > 0 {
+			stepContext.PreviousStep = results[i-1]
+		}
+
+		tmpl, err := parseStepRun(step.Run)
+		if err != nil {
+			return nil, errors.Wrap(err, "parsing step run")
+		}
+
+		var buf bytes.Buffer
+		if err := tmpl.Execute(io.MultiWriter(&buf, fp), stepContext); err != nil {
+			return nil, errors.Wrap(err, "executing template")
 		}
 		fp.Close()
 
+		reportProgress(buf.String())
+
 		const workDir = "/work"
 		cmd := exec.CommandContext(ctx, "docker", "run",
 			"--rm",
@@ -141,10 +157,22 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
 		}
 
 		logger.Logf("[Step %d] complete in %s", i+1, elapsed)
-	}
 
-	if _, err := runGitCmd("add", "--all"); err != nil {
-		return nil, errors.Wrap(err, "git add failed")
+		if _, err := runGitCmd("add", "--all"); err != nil {
+			return nil, errors.Wrap(err, "git add failed")
+		}
+
+		statusOut, err := runGitCmd("status", "--porcelain")
+		if err != nil {
+			return nil, errors.Wrap(err, "git status failed")
+		}
+
+		changes, err := parseGitStatus(statusOut)
+		if err != nil {
+			return nil, errors.Wrap(err, "parsing git status output")
+		}
+
+		results[i] = StepResult{Changes: changes}
 	}
 
 	reportProgress("Calculating diff")
@@ -162,6 +190,57 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor
 	return diffOut, err
 }
 
+func parseStepRun(run string) (*template.Template, error) {
+	return template.New("step-run").Delims("${{", "}}").Parse(run)
+}
+
+type StepContext struct {
+	PreviousStep StepResult
+	Repository   *graphql.Repository
+}
+
+type StepChanges struct {
+	Modified []string
+	Added    []string
+	Deleted  []string
+}
+
+type StepResult struct {
+	Changes StepChanges
+}
+
+func (r StepResult) ModifiedFiles() string { return strings.Join(r.Changes.Modified, " ") }
+func (r StepResult) AddedFiles() string    { return strings.Join(r.Changes.Added, " ") }
+func (r StepResult) DeletedFiles() string  { return strings.Join(r.Changes.Deleted, " ") }
+
+func parseGitStatus(out []byte) (StepChanges, error) {
+	result := StepChanges{}
+
+	stripped := strings.TrimSpace(string(out))
+	if len(stripped) == 0 {
+		return result, nil
+	}
+
+	for _, line := range strings.Split(stripped, "\n") {
+		if len(line) < 4 {
+			return result, fmt.Errorf("git status line has unrecognized format: %q", line)
+		}
+
+		file := line[3:len(line)]
+
+		switch line[0] {
+		case 'M':
+			result.Modified = append(result.Modified, file)
+		case 'A':
+			result.Added = append(result.Added, file)
+		case 'D':
+			result.Deleted = append(result.Deleted, file)
+		}
+	}
+
+	return result, nil
+}
+
 func probeImageForShell(ctx context.Context, image string) (shell, tempfile string, err error) {
 	// We need to know two things to be able to run a shell script:
 	//

internal/campaigns/run_steps_test.go

@@ -0,0 +1,94 @@
+package campaigns
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/sourcegraph/src-cli/internal/campaigns/graphql"
+)
+
+func TestParseGitStatus(t *testing.T) {
+	const input = `M  README.md
+M  another_file.go
+A  new_file.txt
+A  barfoo/new_file.txt
+D  to_be_deleted.txt
+`
+	parsed, err := parseGitStatus([]byte(input))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := StepChanges{
+		Modified: []string{"README.md", "another_file.go"},
+		Added:    []string{"new_file.txt", "barfoo/new_file.txt"},
+		Deleted:  []string{"to_be_deleted.txt"},
+	}
+
+	if !cmp.Equal(want, parsed) {
+		t.Fatalf("wrong output:\n%s", cmp.Diff(want, parsed))
+	}
+}
+
+func TestParseStepRun(t *testing.T) {
+	tests := []struct {
+		stepCtx StepContext
+		run     string
+		want    string
+	}{
+		{
+			stepCtx: StepContext{
+				Repository: &graphql.Repository{Name: "github.com/sourcegraph/src-cli"},
+				PreviousStep: StepResult{
+					Changes: StepChanges{
+						Modified: []string{"go.mod"},
+						Added:    []string{"main.go.swp"},
+						Deleted:  []string{".DS_Store"},
+					},
+				},
+			},
+
+			run:  `${{ .PreviousStep.ModifiedFiles }} ${{ .Repository.Name }}`,
+			want: `go.mod github.com/sourcegraph/src-cli`,
+		},
+		{
+			stepCtx: StepContext{
+				Repository: &graphql.Repository{Name: "github.com/sourcegraph/src-cli"},
+			},
+
+			run:  `${{ .PreviousStep.ModifiedFiles }} ${{ .Repository.Name }}`,
+			want: ` github.com/sourcegraph/src-cli`,
+		},
+		{
+			stepCtx: StepContext{
+				Repository: &graphql.Repository{
+					Name: "github.com/sourcegraph/src-cli",
+					FileMatches: map[string]bool{
+						"README.md": true,
+						"main.go":   true,
+					},
+				},
+			},
+
+			run:  `${{ .Repository.SearchResultPaths }}`,
+			want: `README.md main.go`,
+		},
+	}
+
+	for _, tc := range tests {
+		parsed, err := parseStepRun(tc.run)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var out bytes.Buffer
+		if err := parsed.Execute(&out, tc.stepCtx); err != nil {
+			t.Fatalf("executing template failed: %s", err)
+		}
+
+		if out.String() != tc.want {
+			t.Fatalf("wrong output:\n%s", cmp.Diff(tc.want, out.String()))
+		}
+	}
+}

internal/campaigns/service.go

@@ -430,6 +430,7 @@ query ChangesetRepos(
                     ...repositoryFields
                 }
                 ... on FileMatch {
+                    file { path }
                     repository {
                         ...repositoryFields
                     }
@@ -454,13 +455,18 @@ func (svc *Service) resolveRepositorySearch(ctx context.Context, query string) (
 		return nil, err
 	}
 
-	ids := map[string]struct{}{}
+	ids := map[string]*graphql.Repository{}
 	var repos []*graphql.Repository
 	for _, r := range result.Search.Results.Results {
-		if _, ok := ids[r.ID]; !ok {
+		existing, ok := ids[r.ID]
+		if !ok {
 			repo := r.Repository
 			repos = append(repos, &repo)
-			ids[r.ID] = struct{}{}
+			ids[r.ID] = &repo
+		} else {
+			for file := range r.FileMatches {
+				existing.FileMatches[file] = true
+			}
 		}
 	}
 	return repos, nil
@@ -492,12 +498,18 @@ func (sr *searchResult) UnmarshalJSON(data []byte) error {
 
 	switch tn.Typename {
 	case "FileMatch":
-		var result struct{ Repository graphql.Repository }
+		var result struct {
+			Repository graphql.Repository
+			File       struct {
+				Path string
+			}
+		}
 		if err := json.Unmarshal(data, &result); err != nil {
 			return err
 		}
 
 		sr.Repository = result.Repository
+		sr.Repository.FileMatches = map[string]bool{result.File.Path: true}
 		return nil
 
 	case "Repository":