diff --git a/internal/campaigns/executor_test.go b/internal/campaigns/executor_test.go index 4f54240824..088768f2ba 100644 --- a/internal/campaigns/executor_test.go +++ b/internal/campaigns/executor_test.go @@ -98,6 +98,23 @@ func TestExecutor_Integration(t *testing.T) { executorTimeout: 100 * time.Millisecond, wantErrInclude: "execution in github.com/sourcegraph/src-cli failed: Timeout reached. Execution took longer than 100ms.", }, + { + name: "templated", + repos: []*graphql.Repository{srcCLIRepo}, + archives: []mockRepoArchive{ + {repo: srcCLIRepo, files: map[string]string{ + "README.md": "# Welcome to the README\n", + "main.go": "package main\n\nfunc main() {\n\tfmt.Println( \"Hello World\")\n}\n", + }}, + }, + steps: []Step{ + {Run: `go fmt main.go`, Container: "doesntmatter:13"}, + {Run: `touch modified-${{ .PreviousStep.ModifiedFiles }}.md`, Container: "alpine:13"}, + }, + wantFilesChanged: map[string][]string{ + srcCLIRepo.ID: []string{"main.go", "modified-main.go.md"}, + }, + }, } for _, tc := range tests { @@ -170,8 +187,13 @@ func TestExecutor_Integration(t *testing.T) { diffsByName := map[string]*diff.FileDiff{} for _, fd := range fileDiffs { - diffsByName[fd.OrigName] = fd + if fd.NewName == "/dev/null" { + diffsByName[fd.OrigName] = fd + } else { + diffsByName[fd.NewName] = fd + } } + for _, file := range wantFiles { if _, ok := diffsByName[file]; !ok { t.Errorf("%s was not changed", file) diff --git a/internal/campaigns/graphql/repository.go b/internal/campaigns/graphql/repository.go index 584f3f0e66..6147bae78b 100644 --- a/internal/campaigns/graphql/repository.go +++ b/internal/campaigns/graphql/repository.go @@ -1,6 +1,9 @@ package graphql -import "strings" +import ( + "sort" + "strings" +) const RepositoryFieldsFragment = ` fragment repositoryFields on Repository { @@ -30,6 +33,8 @@ type Repository struct { URL string ExternalRepository struct{ ServiceType string } DefaultBranch *Branch + + FileMatches map[string]bool } func (r *Repository) BaseRef() string { @@ -43,3 +48,16 @@ func (r *Repository) Rev() string { func (r *Repository) Slug() string { return strings.ReplaceAll(r.Name, "/", "-") } + +func (r *Repository) SearchResultPaths() (list fileMatchPathList) { + var files []string + for f := range r.FileMatches { + files = append(files, f) + } + sort.Strings(files) + return fileMatchPathList(files) +} + +type fileMatchPathList []string + +func (f fileMatchPathList) String() string { return strings.Join(f, " ") } diff --git a/internal/campaigns/run_steps.go b/internal/campaigns/run_steps.go index 42b844d9b7..a08bc9d2ed 100644 --- a/internal/campaigns/run_steps.go +++ b/internal/campaigns/run_steps.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "strings" + "text/template" "time" "github.com/hashicorp/go-multierror" @@ -60,9 +61,10 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor return nil, errors.Wrap(err, "git commit failed") } + results := make([]StepResult, len(steps)) + for i, step := range steps { logger.Logf("[Step %d] docker run %s %q", i+1, step.Container, step.Run) - reportProgress(step.Run) cidFile, err := ioutil.TempFile(tempDir, repo.Slug()+"-container-id") if err != nil { @@ -93,11 +95,25 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor } hostTemp := fp.Name() defer os.Remove(hostTemp) - if _, err := fp.WriteString(step.Run); err != nil { - return nil, errors.Wrapf(err, "writing to temporary file %q", hostTemp) + + stepContext := StepContext{Repository: repo} + if i > 0 { + stepContext.PreviousStep = results[i-1] + } + + tmpl, err := parseStepRun(step.Run) + if err != nil { + return nil, errors.Wrap(err, "parsing step run") + } + + var buf bytes.Buffer + if err := tmpl.Execute(io.MultiWriter(&buf, fp), stepContext); err != nil { + return nil, errors.Wrap(err, "executing template") } fp.Close() + reportProgress(buf.String()) + const workDir = "/work" cmd := exec.CommandContext(ctx, "docker", "run", "--rm", @@ -141,10 +157,22 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor } logger.Logf("[Step %d] complete in %s", i+1, elapsed) - } - if _, err := runGitCmd("add", "--all"); err != nil { - return nil, errors.Wrap(err, "git add failed") + if _, err := runGitCmd("add", "--all"); err != nil { + return nil, errors.Wrap(err, "git add failed") + } + + statusOut, err := runGitCmd("status", "--porcelain") + if err != nil { + return nil, errors.Wrap(err, "git status failed") + } + + changes, err := parseGitStatus(statusOut) + if err != nil { + return nil, errors.Wrap(err, "parsing git status output") + } + + results[i] = StepResult{Changes: changes} } reportProgress("Calculating diff") @@ -162,6 +190,57 @@ func runSteps(ctx context.Context, wc *WorkspaceCreator, repo *graphql.Repositor return diffOut, err } +func parseStepRun(run string) (*template.Template, error) { + return template.New("step-run").Delims("${{", "}}").Parse(run) +} + +type StepContext struct { + PreviousStep StepResult + Repository *graphql.Repository +} + +type StepChanges struct { + Modified []string + Added []string + Deleted []string +} + +type StepResult struct { + Changes StepChanges +} + +func (r StepResult) ModifiedFiles() string { return strings.Join(r.Changes.Modified, " ") } +func (r StepResult) AddedFiles() string { return strings.Join(r.Changes.Added, " ") } +func (r StepResult) DeletedFiles() string { return strings.Join(r.Changes.Deleted, " ") } + +func parseGitStatus(out []byte) (StepChanges, error) { + result := StepChanges{} + + stripped := strings.TrimSpace(string(out)) + if len(stripped) == 0 { + return result, nil + } + + for _, line := range strings.Split(stripped, "\n") { + if len(line) < 4 { + return result, fmt.Errorf("git status line has unrecognized format: %q", line) + } + + file := line[3:len(line)] + + switch line[0] { + case 'M': + result.Modified = append(result.Modified, file) + case 'A': + result.Added = append(result.Added, file) + case 'D': + result.Deleted = append(result.Deleted, file) + } + } + + return result, nil +} + func probeImageForShell(ctx context.Context, image string) (shell, tempfile string, err error) { // We need to know two things to be able to run a shell script: // diff --git a/internal/campaigns/run_steps_test.go b/internal/campaigns/run_steps_test.go new file mode 100644 index 0000000000..2310b9f942 --- /dev/null +++ b/internal/campaigns/run_steps_test.go @@ -0,0 +1,94 @@ +package campaigns + +import ( + "bytes" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/sourcegraph/src-cli/internal/campaigns/graphql" +) + +func TestParseGitStatus(t *testing.T) { + const input = `M README.md +M another_file.go +A new_file.txt +A barfoo/new_file.txt +D to_be_deleted.txt +` + parsed, err := parseGitStatus([]byte(input)) + if err != nil { + t.Fatal(err) + } + + want := StepChanges{ + Modified: []string{"README.md", "another_file.go"}, + Added: []string{"new_file.txt", "barfoo/new_file.txt"}, + Deleted: []string{"to_be_deleted.txt"}, + } + + if !cmp.Equal(want, parsed) { + t.Fatalf("wrong output:\n%s", cmp.Diff(want, parsed)) + } +} + +func TestParseStepRun(t *testing.T) { + tests := []struct { + stepCtx StepContext + run string + want string + }{ + { + stepCtx: StepContext{ + Repository: &graphql.Repository{Name: "github.com/sourcegraph/src-cli"}, + PreviousStep: StepResult{ + Changes: StepChanges{ + Modified: []string{"go.mod"}, + Added: []string{"main.go.swp"}, + Deleted: []string{".DS_Store"}, + }, + }, + }, + + run: `${{ .PreviousStep.ModifiedFiles }} ${{ .Repository.Name }}`, + want: `go.mod github.com/sourcegraph/src-cli`, + }, + { + stepCtx: StepContext{ + Repository: &graphql.Repository{Name: "github.com/sourcegraph/src-cli"}, + }, + + run: `${{ .PreviousStep.ModifiedFiles }} ${{ .Repository.Name }}`, + want: ` github.com/sourcegraph/src-cli`, + }, + { + stepCtx: StepContext{ + Repository: &graphql.Repository{ + Name: "github.com/sourcegraph/src-cli", + FileMatches: map[string]bool{ + "README.md": true, + "main.go": true, + }, + }, + }, + + run: `${{ .Repository.SearchResultPaths }}`, + want: `README.md main.go`, + }, + } + + for _, tc := range tests { + parsed, err := parseStepRun(tc.run) + if err != nil { + t.Fatal(err) + } + + var out bytes.Buffer + if err := parsed.Execute(&out, tc.stepCtx); err != nil { + t.Fatalf("executing template failed: %s", err) + } + + if out.String() != tc.want { + t.Fatalf("wrong output:\n%s", cmp.Diff(tc.want, out.String())) + } + } +} diff --git a/internal/campaigns/service.go b/internal/campaigns/service.go index a55a1053d3..1a803f5f49 100644 --- a/internal/campaigns/service.go +++ b/internal/campaigns/service.go @@ -430,6 +430,7 @@ query ChangesetRepos( ...repositoryFields } ... on FileMatch { + file { path } repository { ...repositoryFields } @@ -454,13 +455,18 @@ func (svc *Service) resolveRepositorySearch(ctx context.Context, query string) ( return nil, err } - ids := map[string]struct{}{} + ids := map[string]*graphql.Repository{} var repos []*graphql.Repository for _, r := range result.Search.Results.Results { - if _, ok := ids[r.ID]; !ok { + existing, ok := ids[r.ID] + if !ok { repo := r.Repository repos = append(repos, &repo) - ids[r.ID] = struct{}{} + ids[r.ID] = &repo + } else { + for file := range r.FileMatches { + existing.FileMatches[file] = true + } } } return repos, nil @@ -492,12 +498,18 @@ func (sr *searchResult) UnmarshalJSON(data []byte) error { switch tn.Typename { case "FileMatch": - var result struct{ Repository graphql.Repository } + var result struct { + Repository graphql.Repository + File struct { + Path string + } + } if err := json.Unmarshal(data, &result); err != nil { return err } sr.Repository = result.Repository + sr.Repository.FileMatches = map[string]bool{result.File.Path: true} return nil case "Repository":