Add ability to download subpath archive

.golangci.yml

@@ -48,6 +48,8 @@ linters:
               desc: do not use the ini package, use gitea's config system instead
             - pkg: gitea.com/go-chi/cache
               desc: do not use the go-chi cache package, use gitea's cache system
+            - pkg: github.com/pkg/errors
+              desc: use builtin errors package instead
     nolintlint:
       allow-unused: false
       require-explanation: true

go.mod

@@ -95,7 +95,6 @@ require (
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
-	github.com/pkg/errors v0.9.1
 	github.com/pquerna/otp v1.5.0
 	github.com/prometheus/client_golang v1.23.0
 	github.com/quasoft/websspi v1.1.2
@@ -251,6 +250,7 @@ require (
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pjbgf/sha1cd v0.4.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect

modules/git/gitcmd/command.go

@@ -426,9 +426,9 @@ type RunStdError interface {
 }
 
 type runStdError struct {
-	err    error
-	stderr string
-	errMsg string
+	err    error  // usually the low-level error like exec.ExitError
+	stderr string // git command's stderr output
+	errMsg string // the cached error message for Error() method
 }
 
 func (r *runStdError) Error() string {
@@ -448,6 +448,22 @@ func (r *runStdError) Stderr() string {
 	return r.stderr
 }
 
+func ErrorAsStderr(err error) (string, bool) {
+	var runErr RunStdError
+	if errors.As(err, &runErr) {
+		return runErr.Stderr(), true
+	}
+	return "", false
+}
+
+func StderrHasPrefix(err error, prefix string) bool {
+	stderr, ok := ErrorAsStderr(err)
+	if !ok {
+		return false
+	}
+	return strings.HasPrefix(stderr, prefix)
+}
+
 func IsErrorExitCode(err error, code int) bool {
 	var exitError *exec.ExitError
 	if errors.As(err, &exitError) {

modules/git/gitcmd/command_test.go

@@ -12,6 +12,7 @@ import (
 	"code.gitea.io/gitea/modules/tempdir"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestMain(m *testing.M) {
@@ -99,3 +100,14 @@ func TestCommandString(t *testing.T) {
 	cmd = NewCommand("url: https://a:b@c/", "/root/dir-a/dir-b")
 	assert.Equal(t, cmd.prog+` "url: https://sanitized-credential@c/" .../dir-a/dir-b`, cmd.LogString())
 }
+
+func TestRunStdError(t *testing.T) {
+	e := &runStdError{stderr: "some error"}
+	var err RunStdError = e
+
+	var asErr RunStdError
+	require.ErrorAs(t, err, &asErr)
+	require.Equal(t, "some error", asErr.Stderr())
+
+	require.ErrorAs(t, fmt.Errorf("wrapped %w", err), &asErr)
+}

modules/git/gitcmd/utils.go

@@ -3,12 +3,18 @@
 
 package gitcmd
 
-import "fmt"
+import (
+	"fmt"
 
-// ConcatenateError concatenats an error with stderr string
+	"code.gitea.io/gitea/modules/util"
+)
+
+// ConcatenateError concatenates an error with stderr string
+// FIXME: use RunStdError instead
 func ConcatenateError(err error, stderr string) error {
 	if len(stderr) == 0 {
 		return err
 	}
-	return fmt.Errorf("%w - %s", err, stderr)
+	errMsg := fmt.Sprintf("%s - %s", err.Error(), stderr)
+	return util.ErrorWrap(&runStdError{err: err, stderr: stderr, errMsg: errMsg}, "%s", errMsg)
 }

modules/gitrepo/archive.go

@@ -8,15 +8,17 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git/gitcmd"
 	"code.gitea.io/gitea/modules/setting"
 )
 
 // CreateArchive create archive content to the target path
-func CreateArchive(ctx context.Context, repo Repository, format string, target io.Writer, usePrefix bool, commitID string) error {
+func CreateArchive(ctx context.Context, repo Repository, format string, target io.Writer, usePrefix bool, commitID string, paths []string) error {
 	if format == "unknown" {
 		return fmt.Errorf("unknown format: %v", format)
 	}
@@ -28,6 +30,13 @@ func CreateArchive(ctx context.Context, repo Repository, format string, target i
 	cmd.AddOptionFormat("--format=%s", format)
 	cmd.AddDynamicArguments(commitID)
 
+	paths = slices.Clone(paths)
+	for i := range paths {
+		// although "git archive" already ensures the paths won't go outside the repo, we still clean them here for safety
+		paths[i] = path.Clean(paths[i])
+	}
+	cmd.AddDynamicArguments(paths...)
+
 	var stderr strings.Builder
 	if err := RunCmd(ctx, repo, cmd.WithStdout(target).WithStderr(&stderr)); err != nil {
 		return gitcmd.ConcatenateError(err, stderr.String())

modules/test/utils.go

@@ -4,6 +4,9 @@
 package test
 
 import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -71,3 +74,31 @@ func SetupGiteaRoot() string {
 	_ = os.Setenv("GITEA_ROOT", giteaRoot)
 	return giteaRoot
 }
+
+func ReadAllTarGzContent(r io.Reader) (map[string]string, error) {
+	gzr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+
+	content := make(map[string]string)
+
+	tr := tar.NewReader(gzr)
+	for {
+		hd, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		buf, err := io.ReadAll(tr)
+		if err != nil {
+			return nil, err
+		}
+
+		content[hd.Name] = string(buf)
+	}
+	return content, nil
+}

options/locale/locale_en-US.json

@@ -977,6 +977,7 @@
   "repo.fork.blocked_user": "Cannot fork the repository because you are blocked by the repository owner.",
   "repo.use_template": "Use this template",
   "repo.open_with_editor": "Open with %s",
+  "repo.download_directory_as": "Download directory as %s",
   "repo.download_zip": "Download ZIP",
   "repo.download_tar": "Download TAR.GZ",
   "repo.download_bundle": "Download BUNDLE",

routers/api/v1/repo/download.go

@@ -8,25 +8,35 @@ import (
 	"net/http"
 
 	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/services/context"
 	archiver_service "code.gitea.io/gitea/services/repository/archiver"
 )
 
-func serveRepoArchive(ctx *context.APIContext, reqFileName string) {
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, reqFileName)
+func serveRepoArchive(ctx *context.APIContext, reqFileName string, paths []string) {
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, reqFileName, paths)
 	if err != nil {
-		if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) {
+		if errors.Is(err, util.ErrInvalidArgument) {
 			ctx.APIError(http.StatusBadRequest, err)
-		} else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) {
+		} else if errors.Is(err, util.ErrNotExist) {
 			ctx.APIError(http.StatusNotFound, err)
 		} else {
 			ctx.APIErrorInternal(err)
 		}
 		return
 	}
-	archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	err = archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) {
+			ctx.APIError(http.StatusBadRequest, err)
+		} else {
+			ctx.APIErrorInternal(err)
+		}
+	}
 }
 
+// DownloadArchive is the GitHub-compatible endpoint to download repository archives
+// TODO: The API document is missing: Add github compatible tarball download API endpoints (#32572)
 func DownloadArchive(ctx *context.APIContext) {
 	var tp repo_model.ArchiveType
 	switch ballType := ctx.PathParam("ball_type"); ballType {
@@ -40,5 +50,5 @@ func DownloadArchive(ctx *context.APIContext) {
 		ctx.APIError(http.StatusBadRequest, "Unknown archive type: "+ballType)
 		return
 	}
-	serveRepoArchive(ctx, ctx.PathParam("*")+"."+tp.String())
+	serveRepoArchive(ctx, ctx.PathParam("*")+"."+tp.String(), ctx.FormStrings("path"))
 }

routers/api/v1/repo/file.go

@@ -273,13 +273,19 @@ func GetArchive(ctx *context.APIContext) {
 	//   description: the git reference for download with attached archive format (e.g. master.zip)
 	//   type: string
 	//   required: true
+	// - name: path
+	//   in: query
+	//   type: array
+	//   items:
+	//     type: string
+	//   description: subpath of the repository to download
+	//   collectionFormat: multi
 	// responses:
 	//   200:
 	//     description: success
 	//   "404":
 	//     "$ref": "#/responses/notFound"
-
-	serveRepoArchive(ctx, ctx.PathParam("*"))
+	serveRepoArchive(ctx, ctx.PathParam("*"), ctx.FormStrings("path"))
 }
 
 // GetEditorconfig get editor config of a repository

routers/web/repo/repo.go

@@ -364,31 +364,39 @@ func RedirectDownload(ctx *context.Context) {
 
 // Download an archive of a repository
 func Download(ctx *context.Context) {
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"))
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"), ctx.FormStrings("path"))
 	if err != nil {
-		if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) {
+		if errors.Is(err, util.ErrInvalidArgument) {
 			ctx.HTTPError(http.StatusBadRequest, err.Error())
-		} else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) {
+		} else if errors.Is(err, util.ErrNotExist) {
 			ctx.HTTPError(http.StatusNotFound, err.Error())
 		} else {
 			ctx.ServerError("archiver_service.NewRequest", err)
 		}
 		return
 	}
-	archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	err = archiver_service.ServeRepoArchive(ctx.Base, aReq)
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) {
+			ctx.HTTPError(http.StatusBadRequest, err.Error())
+		} else {
+			ctx.ServerError("archiver_service.ServeRepoArchive", err)
+		}
+	}
 }
 
 // InitiateDownload will enqueue an archival request, as needed.  It may submit
 // a request that's already in-progress, but the archiver service will just
 // kind of drop it on the floor if this is the case.
 func InitiateDownload(ctx *context.Context) {
-	if setting.Repository.StreamArchives {
+	paths := ctx.FormStrings("path")
+	if setting.Repository.StreamArchives || len(paths) > 0 {
 		ctx.JSON(http.StatusOK, map[string]any{
 			"complete": true,
 		})
 		return
 	}
-	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"))
+	aReq, err := archiver_service.NewRequest(ctx.Repo.Repository, ctx.Repo.GitRepo, ctx.PathParam("*"), paths)
 	if err != nil {
 		ctx.HTTPError(http.StatusBadRequest, "invalid archive request")
 		return

services/pull/commit_status.go

@@ -6,6 +6,8 @@ package pull
 
 import (
 	"context"
+	"errors"
+	"fmt"
 
 	"code.gitea.io/gitea/models/db"
 	git_model "code.gitea.io/gitea/models/git"
@@ -14,8 +16,6 @@ import (
 	"code.gitea.io/gitea/modules/gitrepo"
 	"code.gitea.io/gitea/modules/glob"
 	"code.gitea.io/gitea/modules/log"
-
-	"github.com/pkg/errors"
 )
 
 // MergeRequiredContextsCommitStatus returns a commit status state for given required contexts
@@ -69,7 +69,7 @@ func MergeRequiredContextsCommitStatus(commitStatuses []*git_model.CommitStatus,
 func IsPullCommitStatusPass(ctx context.Context, pr *issues_model.PullRequest) (bool, error) {
 	pb, err := git_model.GetFirstMatchProtectedBranchRule(ctx, pr.BaseRepoID, pr.BaseBranch)
 	if err != nil {
-		return false, errors.Wrap(err, "GetLatestCommitStatus")
+		return false, fmt.Errorf("GetLatestCommitStatus: %w", err)
 	}
 	if pb == nil || !pb.EnableStatusCheck {
 		return true, nil
@@ -86,19 +86,19 @@ func IsPullCommitStatusPass(ctx context.Context, pr *issues_model.PullRequest) (
 func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullRequest) (commitstatus.CommitStatusState, error) {
 	// Ensure HeadRepo is loaded
 	if err := pr.LoadHeadRepo(ctx); err != nil {
-		return "", errors.Wrap(err, "LoadHeadRepo")
+		return "", fmt.Errorf("LoadHeadRepo: %w", err)
 	}
 
 	// check if all required status checks are successful
 	headGitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, pr.HeadRepo)
 	if err != nil {
-		return "", errors.Wrap(err, "OpenRepository")
+		return "", fmt.Errorf("OpenRepository: %w", err)
 	}
 	defer closer.Close()
 
 	if pr.Flow == issues_model.PullRequestFlowGithub {
 		if exist, err := git_model.IsBranchExist(ctx, pr.HeadRepo.ID, pr.HeadBranch); err != nil {
-			return "", errors.Wrap(err, "IsBranchExist")
+			return "", fmt.Errorf("IsBranchExist: %w", err)
 		} else if !exist {
 			return "", errors.New("Head branch does not exist, can not merge")
 		}
@@ -118,17 +118,17 @@ func GetPullRequestCommitStatusState(ctx context.Context, pr *issues_model.PullR
 	}
 
 	if err := pr.LoadBaseRepo(ctx); err != nil {
-		return "", errors.Wrap(err, "LoadBaseRepo")
+		return "", fmt.Errorf("LoadBaseRepo: %w", err)
 	}
 
 	commitStatuses, err := git_model.GetLatestCommitStatus(ctx, pr.BaseRepo.ID, sha, db.ListOptionsAll)
 	if err != nil {
-		return "", errors.Wrap(err, "GetLatestCommitStatus")
+		return "", fmt.Errorf("GetLatestCommitStatus: %w", err)
 	}
 
 	pb, err := git_model.GetFirstMatchProtectedBranchRule(ctx, pr.BaseRepoID, pr.BaseBranch)
 	if err != nil {
-		return "", errors.Wrap(err, "LoadProtectedBranch")
+		return "", fmt.Errorf("LoadProtectedBranch: %w", err)
 	}
 	var requiredContexts []string
 	if pb != nil {