From 7b0f3e647374c175c91ff9fa097da7873ecec98e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 15:49:01 -0700 Subject: [PATCH 01/30] uniform all temporary directories --- build/generate-gitignores.go | 3 +- build/generate-licenses.go | 3 +- cmd/dump.go | 2 +- models/migrations/base/tests.go | 2 +- models/unittest/testdb.go | 4 +-- modules/git/blame.go | 3 +- modules/git/repo_index.go | 2 +- modules/markup/external/external.go | 2 +- modules/repository/temp.go | 36 +++++++++++-------- modules/setting/global.go | 7 ++++ modules/setting/ssh.go | 3 +- modules/storage/local.go | 2 +- modules/storage/local_test.go | 3 +- modules/util/filebuffer/file_backed_buffer.go | 4 ++- modules/util/legacy_test.go | 4 ++- routers/web/repo/githttp.go | 2 +- routers/web/repo/setting/lfs.go | 8 ++--- services/pull/patch.go | 2 +- services/pull/temp_repo.go | 9 ++--- services/repository/create.go | 8 +++-- services/repository/files/temp_repo.go | 13 ++++--- services/repository/generate.go | 3 +- services/repository/repository.go | 5 ++- services/wiki/wiki.go | 16 +++------ tests/integration/dump_restore_test.go | 2 +- 25 files changed, 82 insertions(+), 66 deletions(-) diff --git a/build/generate-gitignores.go b/build/generate-gitignores.go index 1e09c83a6a716..164af8f71efd7 100644 --- a/build/generate-gitignores.go +++ b/build/generate-gitignores.go @@ -15,6 +15,7 @@ import ( "path/filepath" "strings" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -32,7 +33,7 @@ func main() { flag.StringVar(&githubApiToken, "token", "", "github api token") flag.Parse() - file, err := os.CreateTemp(os.TempDir(), prefix) + file, err := os.CreateTemp(setting.TempDir(), prefix) if err != nil { log.Fatalf("Failed to create temp file. %s", err) } diff --git a/build/generate-licenses.go b/build/generate-licenses.go index 66e1d3775515e..23236e32e174b 100644 --- a/build/generate-licenses.go +++ b/build/generate-licenses.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/build/license" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -39,7 +40,7 @@ func main() { flag.StringVar(&githubApiToken, "token", "", "github api token") flag.Parse() - file, err := os.CreateTemp(os.TempDir(), prefix) + file, err := os.CreateTemp(setting.TempDir(), prefix) if err != nil { log.Fatalf("Failed to create temp file. %s", err) } diff --git a/cmd/dump.go b/cmd/dump.go index ececc80f72507..4acc0558d2f78 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -49,7 +49,7 @@ var CmdDump = &cli.Command{ &cli.StringFlag{ Name: "tempdir", Aliases: []string{"t"}, - Value: os.TempDir(), + Value: setting.TempDir(), Usage: "Temporary dir path", }, &cli.StringFlag{ diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 85cafc1ab915e..676504e53016f 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -141,7 +141,7 @@ func MainTest(m *testing.M) { setting.CustomConf = giteaConf } - tmpDataPath, err := os.MkdirTemp("", "data") + tmpDataPath, err := os.MkdirTemp(setting.TempDir(), "data") if err != nil { fmt.Printf("Unable to create temporary data path %v\n", err) os.Exit(1) diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 53c9dbdd77254..889a878d1acb0 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -124,12 +124,12 @@ func MainTest(m *testing.M, testOpts ...*TestOptions) { setting.SSH.Domain = "try.gitea.io" setting.Database.Type = "sqlite3" setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - repoRootPath, err := os.MkdirTemp(os.TempDir(), "repos") + repoRootPath, err := os.MkdirTemp(setting.TempDir(), "repos") if err != nil { fatalTestError("TempDir: %v\n", err) } setting.RepoRootPath = repoRootPath - appDataPath, err := os.MkdirTemp(os.TempDir(), "appdata") + appDataPath, err := os.MkdirTemp(setting.TempDir(), "appdata") if err != nil { fatalTestError("TempDir: %v\n", err) } diff --git a/modules/git/blame.go b/modules/git/blame.go index a9b2706f21739..76c07c78f24ba 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -12,6 +12,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -195,7 +196,7 @@ func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { } defer r.Close() - f, err := os.CreateTemp("", "gitea_git-blame-ignore-revs") + f, err := os.CreateTemp(setting.TempDir(), "gitea_git-blame-ignore-revs") if err != nil { return nil } diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index 839057009872e..0cff4f6db5eca 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -51,7 +51,7 @@ func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) er // ReadTreeToTemporaryIndex reads a treeish to a temporary index file func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename, tmpDir string, cancel context.CancelFunc, err error) { - tmpDir, err = os.MkdirTemp("", "index") + tmpDir, err = os.MkdirTemp(os.TempDir(), "index") if err != nil { return filename, tmpDir, cancel, err } diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 122517ed11c0a..2e44bf3439594 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -89,7 +89,7 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. if p.IsInputFile { // write to temp file - f, err := os.CreateTemp("", "gitea_input") + f, err := os.CreateTemp(setting.TempDir(), "gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 04faa9db3de8e..9398b4c0ccefd 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -6,7 +6,6 @@ package repository import ( "fmt" "os" - "path" "path/filepath" "code.gitea.io/gitea/modules/log" @@ -14,30 +13,37 @@ import ( "code.gitea.io/gitea/modules/util" ) -// LocalCopyPath returns the local repository temporary copy path. -func LocalCopyPath() string { - if filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return setting.Repository.Local.LocalCopyPath +// localCopyPath returns the local repository temporary copy path. +func localCopyPath() string { + return filepath.Join(setting.TempDir(), "local-repo") +} + +func CleanUpTemporaryPaths() { + if err := util.RemoveAll(localCopyPath()); err != nil { + log.Error("Unable to remove local repository temporary copy path: %s (%v)", localCopyPath(), err) } - return path.Join(setting.AppDataPath, setting.Repository.Local.LocalCopyPath) } // CreateTemporaryPath creates a temporary path -func CreateTemporaryPath(prefix string) (string, error) { - if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) - return "", fmt.Errorf("Failed to create localcopypath directory %s: %w", LocalCopyPath(), err) +func CreateTemporaryPath(prefix string) (string, func(), error) { + if err := os.MkdirAll(localCopyPath(), os.ModePerm); err != nil { + log.Error("Unable to create localcopypath directory: %s (%v)", localCopyPath(), err) + return "", func() {}, fmt.Errorf("failed to create localcopypath directory %s: %w", localCopyPath(), err) } - basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") + basePath, err := os.MkdirTemp(localCopyPath(), prefix+".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) - return "", fmt.Errorf("Failed to create dir %s-*.git: %w", prefix, err) + return "", func() {}, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } - return basePath, nil + return basePath, func() { + if err := removeTemporaryPath(basePath); err != nil { + log.Error("Unable to remove temporary directory: %s (%v)", basePath, err) + } + }, nil } -// RemoveTemporaryPath removes the temporary path -func RemoveTemporaryPath(basePath string) error { +// removeTemporaryPath removes the temporary path +func removeTemporaryPath(basePath string) error { if _, err := os.Stat(basePath); !os.IsNotExist(err) { return util.RemoveAll(basePath) } diff --git a/modules/setting/global.go b/modules/setting/global.go index 55dfe485b21d1..1e119ffee125c 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -3,6 +3,8 @@ package setting +import "os" + // Global settings var ( // RunUser is the OS user that Gitea is running as. ini:"RUN_USER" @@ -16,3 +18,8 @@ var ( // AppName is the Application name, used in the page title. ini: "APP_NAME" AppName string ) + +// TempDir returns the OS temp directory +func TempDir() string { + return os.TempDir() +} diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index ea387e521fad5..deac56c285c10 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,7 +4,6 @@ package setting import ( - "os" "path" "path/filepath" "strings" @@ -124,7 +123,7 @@ func loadSSHFrom(rootCfg ConfigProvider) { if len(serverMACs) > 0 { SSH.ServerMACs = serverMACs } - SSH.KeyTestPath = os.TempDir() + SSH.KeyTestPath = TempDir() if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } diff --git a/modules/storage/local.go b/modules/storage/local.go index 9bb532f1df812..ab94a490bb842 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -36,7 +36,7 @@ func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorag } if config.TemporaryPath == "" { - config.TemporaryPath = filepath.Join(config.Path, "tmp") + config.TemporaryPath = filepath.Join(setting.TempDir(), filepath.Base(config.Path)) } if !filepath.IsAbs(config.TemporaryPath) { return nil, fmt.Errorf("LocalStorageConfig.TemporaryPath should be an absolute path, but not: %q", config.TemporaryPath) diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index e230323f67953..747ec6f58af73 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -4,7 +4,6 @@ package storage import ( - "os" "path/filepath" "testing" @@ -56,6 +55,6 @@ func TestBuildLocalPath(t *testing.T) { } func TestLocalStorageIterator(t *testing.T) { - dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") + dir := filepath.Join(t.TempDir(), "TestLocalStorageIteratorTestDir") testStorageIterator(t, setting.LocalStorageType, &setting.Storage{Path: dir}) } diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 739543e2977ef..fdc8352f17d74 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -9,6 +9,8 @@ import ( "io" "math" "os" + + "code.gitea.io/gitea/modules/setting" ) var ( @@ -73,7 +75,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp("", "gitea-buffer-") + b.file, err = os.CreateTemp(setting.TempDir(), "gitea-buffer-") if err != nil { return 0, err } diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index e732094c298a1..91d60f9f1c9ad 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -11,13 +11,15 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestCopyFile(t *testing.T) { testContent := []byte("hello") - tmpDir := os.TempDir() + tmpDir := setting.TempDir() now := time.Now() srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index ee1ec1fd0c763..e040c025ab978 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -303,7 +303,7 @@ var ( func dummyInfoRefs(ctx *context.Context) { infoRefsOnce.Do(func() { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-info-refs-cache") + tmpDir, err := os.MkdirTemp(setting.TempDir(), "gitea-info-refs-cache") if err != nil { log.Error("Failed to create temp dir for git-receive-pack cache: %v", err) return diff --git a/routers/web/repo/setting/lfs.go b/routers/web/repo/setting/lfs.go index fad635966864c..e70503ed0c895 100644 --- a/routers/web/repo/setting/lfs.go +++ b/routers/web/repo/setting/lfs.go @@ -109,17 +109,13 @@ func LFSLocks(ctx *context.Context) { } // Clone base repo. - tmpBasePath, err := repo_module.CreateTemporaryPath("locks") + tmpBasePath, cancel, err := repo_module.CreateTemporaryPath("locks") if err != nil { log.Error("Failed to create temporary path: %v", err) ctx.ServerError("LFSLocks", err) return } - defer func() { - if err := repo_module.RemoveTemporaryPath(tmpBasePath); err != nil { - log.Error("LFSLocks: RemoveTemporaryPath: %v", err) - } - }() + defer cancel() if err := git.Clone(ctx, ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{ Bare: true, diff --git a/services/pull/patch.go b/services/pull/patch.go index 0934a86c89a65..37eb6b70e288f 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -346,7 +346,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * } // 3b. Create a plain patch from head to base - tmpPatchFile, err := os.CreateTemp("", "patch") + tmpPatchFile, err := os.CreateTemp(setting.TempDir(), "patch") if err != nil { log.Error("Unable to create temporary patch file! Error: %v", err) return false, fmt.Errorf("unable to create temporary patch file! Error: %w", err) diff --git a/services/pull/temp_repo.go b/services/pull/temp_repo.go index e5753178b8981..99f9d87ad0d16 100644 --- a/services/pull/temp_repo.go +++ b/services/pull/temp_repo.go @@ -73,11 +73,13 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) } // Clone base repo. - tmpBasePath, err := repo_module.CreateTemporaryPath("pull") + tmpBasePath, cancel, err := repo_module.CreateTemporaryPath("pull") if err != nil { log.Error("CreateTemporaryPath[%-v]: %v", pr, err) return nil, nil, err } + defer cancel() + prCtx = &prContext{ Context: ctx, tmpBasePath: tmpBasePath, @@ -85,11 +87,6 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) outbuf: &strings.Builder{}, errbuf: &strings.Builder{}, } - cancel = func() { - if err := repo_module.RemoveTemporaryPath(tmpBasePath); err != nil { - log.Error("Error whilst removing removing temporary repo for %-v: %v", pr, err) - } - } baseRepoPath := pr.BaseRepo.RepoPath() headRepoPath := pr.HeadRepo.RepoPath() diff --git a/services/repository/create.go b/services/repository/create.go index 282b2d3e58d46..1e7c2b2c8260e 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -134,6 +134,10 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, return nil } +func createRepoTempDir(repoName string) (string, error) { + return os.MkdirTemp(filepath.Join(setting.TempDir(), "repos"), "gitea-"+repoName) +} + // InitRepository initializes README and .gitignore if needed. func initRepository(ctx context.Context, repoPath string, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { if err = repo_module.CheckInitRepository(ctx, repo.OwnerName, repo.Name, opts.ObjectFormatName); err != nil { @@ -142,9 +146,9 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re // Initialize repository according to user's choice. if opts.AutoInit { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + tmpDir, err := createRepoTempDir(repo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.RepoPath(), err) } defer func() { if err := util.RemoveAll(tmpDir); err != nil { diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index d70b1e8d54e77..94f8da350cee6 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -30,23 +30,28 @@ type TemporaryUploadRepository struct { repo *repo_model.Repository gitRepo *git.Repository basePath string + cancel func() } // NewTemporaryUploadRepository creates a new temporary upload repository func NewTemporaryUploadRepository(ctx context.Context, repo *repo_model.Repository) (*TemporaryUploadRepository, error) { - basePath, err := repo_module.CreateTemporaryPath("upload") + basePath, cancel, err := repo_module.CreateTemporaryPath("upload") if err != nil { return nil, err } - t := &TemporaryUploadRepository{ctx: ctx, repo: repo, basePath: basePath} + t := &TemporaryUploadRepository{ + ctx: ctx, repo: repo, + basePath: basePath, + cancel: cancel, + } return t, nil } // Close the repository cleaning up all files func (t *TemporaryUploadRepository) Close() { defer t.gitRepo.Close() - if err := repo_module.RemoveTemporaryPath(t.basePath); err != nil { - log.Error("Failed to remove temporary path %s: %v", t.basePath, err) + if t.cancel != nil { + t.cancel() } } diff --git a/services/repository/generate.go b/services/repository/generate.go index 2b95bbcd4d315..336db4e5a69e9 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -253,7 +254,7 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "gitea-"+repo.Name) + tmpDir, err := os.MkdirTemp(setting.TempDir(), "gitea-"+repo.Name) if err != nil { return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.RepoPath(), err) } diff --git a/services/repository/repository.go b/services/repository/repository.go index 59b4491132da9..a6982ca4fd64f 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -107,7 +107,10 @@ func Init(ctx context.Context) error { return err } system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) + repo_module.CleanUpTemporaryPaths() + if err := system_model.CreateNotice(db.DefaultContext, system_model.NoticeRepository, "Clean up temporary repositories"); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } if err := initPushQueue(); err != nil { return err } diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index 7a0419aea7c7e..439003999f717 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -102,15 +102,11 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model hasDefaultBranch := git.IsBranchExist(ctx, repo.WikiPath(), repo.DefaultWikiBranch) - basePath, err := repo_module.CreateTemporaryPath("update-wiki") + basePath, cancel, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { return err } - defer func() { - if err := repo_module.RemoveTemporaryPath(basePath); err != nil { - log.Error("Merge: RemoveTemporaryPath: %s", err) - } - }() + defer cancel() cloneOpts := git.CloneRepoOptions{ Bare: true, @@ -264,15 +260,11 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("InitWiki: %w", err) } - basePath, err := repo_module.CreateTemporaryPath("update-wiki") + basePath, cancel, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { return err } - defer func() { - if err := repo_module.RemoveTemporaryPath(basePath); err != nil { - log.Error("Merge: RemoveTemporaryPath: %s", err) - } - }() + defer cancel() if err := git.Clone(ctx, repo.WikiPath(), basePath, git.CloneRepoOptions{ Bare: true, diff --git a/tests/integration/dump_restore_test.go b/tests/integration/dump_restore_test.go index 47bb6f76e9791..f5a8d0730f788 100644 --- a/tests/integration/dump_restore_test.go +++ b/tests/integration/dump_restore_test.go @@ -44,7 +44,7 @@ func TestDumpRestore(t *testing.T) { reponame := "repo1" - basePath, err := os.MkdirTemp("", reponame) + basePath, err := os.MkdirTemp(setting.TempDir(), reponame) assert.NoError(t, err) defer util.RemoveAll(basePath) From fc25971f571b91393491407b4756f7b4ca9b48aa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 16:01:40 -0700 Subject: [PATCH 02/30] Fix unnecessary changes --- build/generate-gitignores.go | 3 +-- build/generate-licenses.go | 3 +-- modules/repository/temp.go | 7 ++++++- tests/test_utils.go | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/build/generate-gitignores.go b/build/generate-gitignores.go index 164af8f71efd7..1e09c83a6a716 100644 --- a/build/generate-gitignores.go +++ b/build/generate-gitignores.go @@ -15,7 +15,6 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -33,7 +32,7 @@ func main() { flag.StringVar(&githubApiToken, "token", "", "github api token") flag.Parse() - file, err := os.CreateTemp(setting.TempDir(), prefix) + file, err := os.CreateTemp(os.TempDir(), prefix) if err != nil { log.Fatalf("Failed to create temp file. %s", err) } diff --git a/build/generate-licenses.go b/build/generate-licenses.go index 23236e32e174b..66e1d3775515e 100644 --- a/build/generate-licenses.go +++ b/build/generate-licenses.go @@ -22,7 +22,6 @@ import ( "code.gitea.io/gitea/build/license" "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -40,7 +39,7 @@ func main() { flag.StringVar(&githubApiToken, "token", "", "github api token") flag.Parse() - file, err := os.CreateTemp(setting.TempDir(), prefix) + file, err := os.CreateTemp(os.TempDir(), prefix) if err != nil { log.Fatalf("Failed to create temp file. %s", err) } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 9398b4c0ccefd..4a0ad2487697c 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -15,7 +15,12 @@ import ( // localCopyPath returns the local repository temporary copy path. func localCopyPath() string { - return filepath.Join(setting.TempDir(), "local-repo") + if setting.Repository.Local.LocalCopyPath == "" { + return filepath.Join(os.TempDir(), "local-repo") + } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { + return filepath.Join(os.TempDir(), setting.Repository.Local.LocalCopyPath) + } + return setting.Repository.Local.LocalCopyPath } func CleanUpTemporaryPaths() { diff --git a/tests/test_utils.go b/tests/test_utils.go index 6f9592b204112..e5caf6cb485e7 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -78,7 +78,7 @@ func InitTest(requireGitea bool) { unittest.InitSettings() setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - _ = util.RemoveAll(repo_module.LocalCopyPath()) + repo_module.CleanUpTemporaryPaths() if err := git.InitFull(context.Background()); err != nil { log.Fatal("git.InitOnceWithSync: %v", err) From b362ffaa0c3aca33e14ea1e23d0315158273a78a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 16:45:00 -0700 Subject: [PATCH 03/30] use global configuration temp directories --- modules/git/repo.go | 5 +++-- modules/git/repo_index.go | 3 ++- modules/repository/temp.go | 7 ++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/git/repo.go b/modules/git/repo.go index 1c223018adddc..027bd56244691 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,6 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -279,11 +280,11 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, err := os.MkdirTemp(os.TempDir(), "gitea-bundle") + tmp, err := os.MkdirTemp(setting.TempDir(), "gitea-bundle") if err != nil { return err } - defer os.RemoveAll(tmp) + defer util.RemoveAll(tmp) env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) _, _, err = NewCommand(ctx, "init", "--bare").RunStdString(&RunOpts{Dir: tmp, Env: env}) diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index 0cff4f6db5eca..da5e786fab5d4 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) @@ -51,7 +52,7 @@ func (repo *Repository) readTreeToIndex(id ObjectID, indexFilename ...string) er // ReadTreeToTemporaryIndex reads a treeish to a temporary index file func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename, tmpDir string, cancel context.CancelFunc, err error) { - tmpDir, err = os.MkdirTemp(os.TempDir(), "index") + tmpDir, err = os.MkdirTemp(setting.TempDir(), "index") if err != nil { return filename, tmpDir, cancel, err } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 4a0ad2487697c..4299901292707 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -4,6 +4,7 @@ package repository import ( + "context" "fmt" "os" "path/filepath" @@ -16,9 +17,9 @@ import ( // localCopyPath returns the local repository temporary copy path. func localCopyPath() string { if setting.Repository.Local.LocalCopyPath == "" { - return filepath.Join(os.TempDir(), "local-repo") + return filepath.Join(setting.TempDir(), "local-repo") } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return filepath.Join(os.TempDir(), setting.Repository.Local.LocalCopyPath) + return filepath.Join(setting.TempDir(), setting.Repository.Local.LocalCopyPath) } return setting.Repository.Local.LocalCopyPath } @@ -30,7 +31,7 @@ func CleanUpTemporaryPaths() { } // CreateTemporaryPath creates a temporary path -func CreateTemporaryPath(prefix string) (string, func(), error) { +func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { if err := os.MkdirAll(localCopyPath(), os.ModePerm); err != nil { log.Error("Unable to create localcopypath directory: %s (%v)", localCopyPath(), err) return "", func() {}, fmt.Errorf("failed to create localcopypath directory %s: %w", localCopyPath(), err) From fafd7b73ba55c058698f1cd3380251ebe0fc41f2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 17:15:03 -0700 Subject: [PATCH 04/30] Fix recycle dependency --- modules/git/repo.go | 4 +++- modules/util/legacy_test.go | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/git/repo.go b/modules/git/repo.go index 027bd56244691..9240d0877876c 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -284,7 +284,9 @@ func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io. if err != nil { return err } - defer util.RemoveAll(tmp) + defer func() { + _ = util.RemoveAll(tmp) + }() env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) _, _, err = NewCommand(ctx, "init", "--bare").RunStdString(&RunOpts{Dir: tmp, Env: env}) diff --git a/modules/util/legacy_test.go b/modules/util/legacy_test.go index 91d60f9f1c9ad..e732094c298a1 100644 --- a/modules/util/legacy_test.go +++ b/modules/util/legacy_test.go @@ -11,15 +11,13 @@ import ( "testing" "time" - "code.gitea.io/gitea/modules/setting" - "github.com/stretchr/testify/assert" ) func TestCopyFile(t *testing.T) { testContent := []byte("hello") - tmpDir := setting.TempDir() + tmpDir := os.TempDir() now := time.Now() srcFile := fmt.Sprintf("%s/copy-test-%d-src.txt", tmpDir, now.UnixMicro()) dstFile := fmt.Sprintf("%s/copy-test-%d-dst.txt", tmpDir, now.UnixMicro()) From 30621ae20789a29d01c774647a0d5f620395c7f0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 17:30:20 -0700 Subject: [PATCH 05/30] Move all temporary directories under gitea and cleanup it when Gitea start --- cmd/web.go | 1 + models/unittest/testdb.go | 1 + modules/base/tool_test.go | 1 + modules/setting/global.go | 11 +++++++++-- modules/setting/path.go | 1 + routers/install/install.go | 1 + 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index ef8a7426c14d9..2e70f90f635e3 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -153,6 +153,7 @@ func serveInstall(ctx *cli.Context) error { func serveInstalled(ctx *cli.Context) error { setting.InitCfgProvider(setting.CustomConf) setting.LoadCommonSettings() + setting.CleanUpTempDirs() setting.MustInstalled() showWebStartupMessage("Prepare to run web server") diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 889a878d1acb0..5137b942a8861 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -60,6 +60,7 @@ func InitSettings() { } setting.InitCfgProvider(setting.CustomConf) setting.LoadCommonSettings() + setting.CleanUpTempDirs() if err := setting.PrepareAppDataPath(); err != nil { log.Fatal("Can not prepare APP_DATA_PATH: %v", err) diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index 4af8b9bc4d528..a4daf775107d0 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -55,6 +55,7 @@ func TestVerifyTimeLimitCode(t *testing.T) { JWT_SECRET = %s `, secret)) setting.LoadCommonSettings() + setting.CleanUpTempDirs() } initGeneralSecret("KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko") diff --git a/modules/setting/global.go b/modules/setting/global.go index 1e119ffee125c..db9d291276236 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -3,7 +3,10 @@ package setting -import "os" +import ( + "os" + "path/filepath" +) // Global settings var ( @@ -21,5 +24,9 @@ var ( // TempDir returns the OS temp directory func TempDir() string { - return os.TempDir() + return filepath.Join(os.TempDir(), "gitea") +} + +func CleanUpTempDirs() { + _ = os.RemoveAll(TempDir()) } diff --git a/modules/setting/path.go b/modules/setting/path.go index 0fdc305aa160c..1acf1ffb7727e 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -91,6 +91,7 @@ func (s *stringWithDefault) Set(v string) { func InitWorkPathAndCommonConfig(getEnvFn func(name string) string, args ArgWorkPathAndCustomConf) { InitWorkPathAndCfgProvider(getEnvFn, args) LoadCommonSettings() + CleanUpTempDirs() } // InitWorkPathAndCfgProvider will set AppWorkPath, CustomPath and CustomConf, init default config provider by CustomConf diff --git a/routers/install/install.go b/routers/install/install.go index e420d36da5a1b..ac5476a9bc17f 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -535,6 +535,7 @@ func SubmitInstall(ctx *context.Context) { // Reload settings (and re-initialize database connection) setting.InitCfgProvider(setting.CustomConf) setting.LoadCommonSettings() + setting.CleanUpTempDirs() setting.MustInstalled() setting.LoadDBSetting() if err := common.InitDBEngine(ctx); err != nil { From cb812144b53b84adc8814fa8220594933d7b6643 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 17:58:07 -0700 Subject: [PATCH 06/30] create temp root directory when startup --- modules/setting/global.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/setting/global.go b/modules/setting/global.go index db9d291276236..bfdcfffb93cb8 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -4,8 +4,10 @@ package setting import ( + "log" "os" "path/filepath" + "sync" ) // Global settings @@ -20,11 +22,19 @@ var ( // AppName is the Application name, used in the page title. ini: "APP_NAME" AppName string + + createTempOnce sync.Once ) // TempDir returns the OS temp directory func TempDir() string { - return filepath.Join(os.TempDir(), "gitea") + tempDir := filepath.Join(os.TempDir(), "gitea") + createTempOnce.Do(func() { + if err := os.MkdirAll(tempDir, os.ModePerm); err != nil { + log.Fatalf("Failed to create temp directory %s: %v", tempDir, err) + } + }) + return tempDir } func CleanUpTempDirs() { From f079b601c89c94a20fe95f57f7088bc462e6f154 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 27 Oct 2024 18:06:21 -0700 Subject: [PATCH 07/30] Fix temp dir name --- services/repository/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/repository/create.go b/services/repository/create.go index 1e7c2b2c8260e..180e0ae1b4b1f 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -135,7 +135,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, } func createRepoTempDir(repoName string) (string, error) { - return os.MkdirTemp(filepath.Join(setting.TempDir(), "repos"), "gitea-"+repoName) + return os.MkdirTemp(setting.TempDir(), "repos-"+repoName) } // InitRepository initializes README and .gitignore if needed. From fc448b5125436704023719852bc197c127eeaccc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 11 Mar 2025 17:15:13 -0700 Subject: [PATCH 08/30] Fix --- services/repository/files/temp_repo.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 6767e7c9c230a..494927ff3e469 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -34,11 +34,11 @@ type TemporaryUploadRepository struct { // NewTemporaryUploadRepository creates a new temporary upload repository func NewTemporaryUploadRepository(repo *repo_model.Repository) (*TemporaryUploadRepository, error) { - basePath, err := repo_module.CreateTemporaryPath("upload") + basePath, cancel, err := repo_module.CreateTemporaryPath("upload") if err != nil { return nil, err } - t := &TemporaryUploadRepository{repo: repo, basePath: basePath} + t := &TemporaryUploadRepository{repo: repo, basePath: basePath, cancel: cancel} return t, nil } From 1ad7ee807698d6fd1fbaaf280ba0aedae6b55444 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 11 Mar 2025 21:24:53 -0700 Subject: [PATCH 09/30] Fix --- modules/git/git_test.go | 2 +- modules/setting/global.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/git/git_test.go b/modules/git/git_test.go index 5472842b76e58..d2ab4ff622f0a 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -17,7 +17,7 @@ import ( ) func testRun(m *testing.M) error { - gitHomePath, err := os.MkdirTemp(os.TempDir(), "git-home") + gitHomePath, err := os.MkdirTemp(setting.TempDir(), "git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } diff --git a/modules/setting/global.go b/modules/setting/global.go index bfdcfffb93cb8..2f22d8cee447e 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -23,12 +23,13 @@ var ( // AppName is the Application name, used in the page title. ini: "APP_NAME" AppName string + tempDir string createTempOnce sync.Once ) // TempDir returns the OS temp directory func TempDir() string { - tempDir := filepath.Join(os.TempDir(), "gitea") + tempDir = filepath.Join(os.TempDir(), "gitea") createTempOnce.Do(func() { if err := os.MkdirAll(tempDir, os.ModePerm); err != nil { log.Fatalf("Failed to create temp directory %s: %v", tempDir, err) @@ -38,5 +39,5 @@ func TempDir() string { } func CleanUpTempDirs() { - _ = os.RemoveAll(TempDir()) + //_ = os.RemoveAll(TempDir()) } From d4d39dcd821e429bc6afb546ff9a417d72e14435 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 12 Mar 2025 18:48:15 -0700 Subject: [PATCH 10/30] Fix test --- cmd/migrate_storage_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index f8fa95a927363..2533e8a24cba8 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -68,7 +68,6 @@ func TestMigratePackages(t *testing.T) { entries, err := os.ReadDir(p) assert.NoError(t, err) - assert.Len(t, entries, 2) + assert.Len(t, entries, 1) // tmp directory should not be under storage any more assert.EqualValues(t, "01", entries[0].Name()) - assert.EqualValues(t, "tmp", entries[1].Name()) } From e03e58f4ad994b79fdce309a25cc3624b843b24c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 14 Mar 2025 16:31:14 -0700 Subject: [PATCH 11/30] Remove unnecessary change --- modules/repository/temp.go | 6 ------ services/repository/repository.go | 3 +-- tests/test_utils.go | 2 -- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 4299901292707..4b4d10ffc1f8e 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -24,12 +24,6 @@ func localCopyPath() string { return setting.Repository.Local.LocalCopyPath } -func CleanUpTemporaryPaths() { - if err := util.RemoveAll(localCopyPath()); err != nil { - log.Error("Unable to remove local repository temporary copy path: %s (%v)", localCopyPath(), err) - } -} - // CreateTemporaryPath creates a temporary path func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { if err := os.MkdirAll(localCopyPath(), os.ModePerm); err != nil { diff --git a/services/repository/repository.go b/services/repository/repository.go index 9d5295349fd24..fbf00f0072557 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -101,8 +101,7 @@ func Init(ctx context.Context) error { if err := repo_module.LoadRepoConfig(); err != nil { return err } - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) - repo_module.CleanUpTemporaryPaths() + if err := system_model.CreateNotice(db.DefaultContext, system_model.NoticeRepository, "Clean up temporary repositories"); err != nil { log.Error("CreateRepositoryNotice: %v", err) } diff --git a/tests/test_utils.go b/tests/test_utils.go index 48f4ce28c848d..14ed759103fcb 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/test" @@ -69,7 +68,6 @@ func InitTest(requireGitea bool) { unittest.InitSettings() setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - repo_module.CleanUpTemporaryPaths() if err := git.InitFull(context.Background()); err != nil { log.Fatal("git.InitOnceWithSync: %v", err) From d41d8b857fe9b95ea9b34705ed7a4958a8280a1b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 15:20:38 -0700 Subject: [PATCH 12/30] Remove cleanup temp dirs --- cmd/web.go | 1 - models/unittest/testdb.go | 1 - modules/base/tool_test.go | 1 - modules/repository/temp.go | 2 +- modules/setting/global.go | 4 ---- modules/setting/path.go | 1 - routers/install/install.go | 1 - 7 files changed, 1 insertion(+), 10 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 28fe051ffbbfe..dc5c6de48a309 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -166,7 +166,6 @@ func serveInstall(ctx *cli.Context) error { func serveInstalled(ctx *cli.Context) error { setting.InitCfgProvider(setting.CustomConf) setting.LoadCommonSettings() - setting.CleanUpTempDirs() setting.MustInstalled() showWebStartupMessage("Prepare to run web server") diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index adafd6cd5e53a..f35065115fef0 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -51,7 +51,6 @@ func InitSettings() { } setting.InitCfgProvider(setting.CustomConf) setting.LoadCommonSettings() - setting.CleanUpTempDirs() if err := setting.PrepareAppDataPath(); err != nil { log.Fatal("Can not prepare APP_DATA_PATH: %v", err) diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index fc02f377d850a..7cebedb073ceb 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -54,7 +54,6 @@ func TestVerifyTimeLimitCode(t *testing.T) { JWT_SECRET = %s `, secret)) setting.LoadCommonSettings() - setting.CleanUpTempDirs() } initGeneralSecret("KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko") diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 123e1c92d039a..4b4d10ffc1f8e 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -21,7 +21,7 @@ func localCopyPath() string { } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { return filepath.Join(setting.TempDir(), setting.Repository.Local.LocalCopyPath) } - return filepath.Join(setting.AppDataPath, setting.Repository.Local.LocalCopyPath) + return setting.Repository.Local.LocalCopyPath } // CreateTemporaryPath creates a temporary path diff --git a/modules/setting/global.go b/modules/setting/global.go index 2f22d8cee447e..e0c07e23e8dba 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -37,7 +37,3 @@ func TempDir() string { }) return tempDir } - -func CleanUpTempDirs() { - //_ = os.RemoveAll(TempDir()) -} diff --git a/modules/setting/path.go b/modules/setting/path.go index 1acf1ffb7727e..0fdc305aa160c 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -91,7 +91,6 @@ func (s *stringWithDefault) Set(v string) { func InitWorkPathAndCommonConfig(getEnvFn func(name string) string, args ArgWorkPathAndCustomConf) { InitWorkPathAndCfgProvider(getEnvFn, args) LoadCommonSettings() - CleanUpTempDirs() } // InitWorkPathAndCfgProvider will set AppWorkPath, CustomPath and CustomConf, init default config provider by CustomConf diff --git a/routers/install/install.go b/routers/install/install.go index a18993172ae2f..b81a5680d39fe 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -530,7 +530,6 @@ func SubmitInstall(ctx *context.Context) { // Reload settings (and re-initialize database connection) setting.InitCfgProvider(setting.CustomConf) setting.LoadCommonSettings() - setting.CleanUpTempDirs() setting.MustInstalled() setting.LoadDBSetting() if err := common.InitDBEngine(ctx); err != nil { From 178da3118bc7a06b4c21b80b61e37a67e5c5edd9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 16:20:19 -0700 Subject: [PATCH 13/30] Add configuration to allow customize global temp path --- cmd/dump.go | 2 +- custom/conf/app.example.ini | 3 ++ models/migrations/base/tests.go | 3 +- models/unittest/testdb.go | 5 ++-- modules/git/blame.go | 4 +-- modules/git/git_test.go | 7 +++-- modules/git/repo.go | 9 ++---- modules/git/repo_index.go | 18 +++--------- modules/markup/external/external.go | 3 +- modules/repository/temp.go | 4 +-- modules/setting/global.go | 22 ++------------- modules/setting/setting.go | 12 ++++++++ modules/setting/ssh.go | 2 +- modules/storage/local.go | 2 +- modules/temp/temp.go | 28 +++++++++++++++++++ modules/util/filebuffer/file_backed_buffer.go | 2 +- routers/web/repo/githttp.go | 11 ++------ services/pull/patch.go | 6 ++-- services/repository/create.go | 14 ++-------- services/repository/generate.go | 13 +++------ 20 files changed, 84 insertions(+), 86 deletions(-) create mode 100644 modules/temp/temp.go diff --git a/cmd/dump.go b/cmd/dump.go index 4acc0558d2f78..99911cbbd9d17 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -49,7 +49,7 @@ var CmdDump = &cli.Command{ &cli.StringFlag{ Name: "tempdir", Aliases: []string{"t"}, - Value: setting.TempDir(), + Value: setting.TempPath, Usage: "Temporary dir path", }, &cli.StringFlag{ diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 0fc49accef84e..63e703308d96d 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -53,6 +53,9 @@ RUN_USER = ; git ;; The working directory, see the comment of AppWorkPath above ;WORK_PATH = +;; The temporary directory, defaults to a directory named gitea under the system temporary directory +;TEMP_PATH = + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; [server] diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 754024b86df2f..013a98f020a61 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/testlogger" @@ -114,7 +115,7 @@ func MainTest(m *testing.M) { setting.CustomConf = giteaConf } - tmpDataPath, err := os.MkdirTemp(setting.TempDir(), "data") + tmpDataPath, _, err := temp.MkdirTemp("data") if err != nil { testlogger.Fatalf("Unable to create temporary data path %v\n", err) } diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index f35065115fef0..60f6b0052b9ec 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting/config" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" @@ -92,12 +93,12 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { setting.SSH.Domain = "try.gitea.io" setting.Database.Type = "sqlite3" setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - repoRootPath, err := os.MkdirTemp(setting.TempDir(), "repos") + repoRootPath, _, err := temp.MkdirTemp("repos") if err != nil { fatalTestError("TempDir: %v\n", err) } setting.RepoRootPath = repoRootPath - appDataPath, err := os.MkdirTemp(setting.TempDir(), "appdata") + appDataPath, _, err := temp.MkdirTemp("appdata") if err != nil { fatalTestError("TempDir: %v\n", err) } diff --git a/modules/git/blame.go b/modules/git/blame.go index 16c793cfd1226..a36f8a01c2768 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -11,7 +11,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/util" ) @@ -193,7 +193,7 @@ func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { } defer r.Close() - f, err := os.CreateTemp(setting.TempDir(), "gitea_git-blame-ignore-revs") + f, err := temp.CreateTemp("gitea_git-blame-ignore-revs") if err != nil { return nil } diff --git a/modules/git/git_test.go b/modules/git/git_test.go index d2ab4ff622f0a..cf6fa87c0c4dd 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,18 +10,19 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/temp" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" ) func testRun(m *testing.M) error { - gitHomePath, err := os.MkdirTemp(setting.TempDir(), "git-home") + gitHomePath, cleanup, err := temp.MkdirTemp("git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } - defer util.RemoveAll(gitHomePath) + defer cleanup() + setting.Git.HomePath = gitHomePath if err = InitFull(context.Background()); err != nil { diff --git a/modules/git/repo.go b/modules/git/repo.go index e351de90d0aba..be01c4104947a 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,8 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/temp" ) // GPGSettings represents the default GPG settings for this repository @@ -268,13 +267,11 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, err := os.MkdirTemp(setting.TempDir(), "gitea-bundle") + tmp, cleanup, err := temp.MkdirTemp("gitea-bundle") if err != nil { return err } - defer func() { - _ = util.RemoveAll(tmp) - }() + defer cleanup() env := append(os.Environ(), "GIT_OBJECT_DIRECTORY="+filepath.Join(repo.Path, "objects")) _, _, err = NewCommand("init", "--bare").RunStdString(ctx, &RunOpts{Dir: tmp, Env: env}) diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index bcdf58235329b..5bfd6b2d9dcc4 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -10,9 +10,7 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/temp" ) // ReadTreeToIndex reads a treeish to the index @@ -60,26 +58,18 @@ func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilena } }() - removeDirFn := func(dir string) func() { // it can't use the return value "tmpDir" directly because it is empty when error occurs - return func() { - if err := util.RemoveAll(dir); err != nil { - log.Error("failed to remove tmp index dir: %v", err) - } - } - } - - tmpDir, err = os.MkdirTemp(setting.TempDir(), "index") + tmpDir, cancel, err = temp.MkdirTemp("index") if err != nil { return "", "", nil, err } tmpIndexFilename = filepath.Join(tmpDir, ".tmp-index") - cancel = removeDirFn(tmpDir) + err = repo.ReadTreeToIndex(treeish, tmpIndexFilename) if err != nil { return "", "", cancel, err } - return tmpIndexFilename, tmpDir, cancel, err + return tmpIndexFilename, tmpDir, cancel, nil } // EmptyIndex empties the index diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 0a61580624729..f367f8b3bafde 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/util" ) @@ -88,7 +89,7 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. if p.IsInputFile { // write to temp file - f, err := os.CreateTemp(setting.TempDir(), "gitea_input") + f, err := temp.CreateTemp("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 4b4d10ffc1f8e..d51c038867afa 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -17,9 +17,9 @@ import ( // localCopyPath returns the local repository temporary copy path. func localCopyPath() string { if setting.Repository.Local.LocalCopyPath == "" { - return filepath.Join(setting.TempDir(), "local-repo") + return filepath.Join(setting.TempPath, "local-repo") } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return filepath.Join(setting.TempDir(), setting.Repository.Local.LocalCopyPath) + return filepath.Join(setting.TempPath, setting.Repository.Local.LocalCopyPath) } return setting.Repository.Local.LocalCopyPath } diff --git a/modules/setting/global.go b/modules/setting/global.go index e0c07e23e8dba..f57e04ef352a2 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -3,13 +3,6 @@ package setting -import ( - "log" - "os" - "path/filepath" - "sync" -) - // Global settings var ( // RunUser is the OS user that Gitea is running as. ini:"RUN_USER" @@ -23,17 +16,6 @@ var ( // AppName is the Application name, used in the page title. ini: "APP_NAME" AppName string - tempDir string - createTempOnce sync.Once + // TempPath is the directory used for temporary files. ini: "TEMP_PATH" + TempPath string ) - -// TempDir returns the OS temp directory -func TempDir() string { - tempDir = filepath.Join(os.TempDir(), "gitea") - createTempOnce.Do(func() { - if err := os.MkdirAll(tempDir, os.ModePerm); err != nil { - log.Fatalf("Failed to create temp directory %s: %v", tempDir, err) - } - }) - return tempDir -} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e14997801fed4..36cbf56f3d739 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -7,6 +7,7 @@ package setting import ( "fmt" "os" + "path/filepath" "runtime" "strings" "time" @@ -110,6 +111,7 @@ func LoadCommonSettings() { func loadCommonSettingsFrom(cfg ConfigProvider) error { // WARNING: don't change the sequence except you know what you are doing. loadRunModeFrom(cfg) + loadTempDir(cfg) loadLogGlobalFrom(cfg) loadServerFrom(cfg) loadSSHFrom(cfg) @@ -182,6 +184,16 @@ func loadRunModeFrom(rootCfg ConfigProvider) { } } +func loadTempDir(rootCfg ConfigProvider) { + rootSec := rootCfg.Section("") + tempDir := rootSec.Key("TEMP_DIR").MustString(filepath.Join(os.TempDir(), "gitea")) + if tempDir != "" { + if err := os.MkdirAll(tempDir, os.ModePerm); err != nil { + log.Fatal("Failed to create temp directory %s: %v", tempDir, err) + } + } +} + // HasInstallLock checks the install-lock in ConfigProvider directly, because sometimes the config file is not loaded into setting variables yet. func HasInstallLock(rootCfg ConfigProvider) bool { return rootCfg.Section("security").Key("INSTALL_LOCK").MustBool(false) diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index 6494208c9b9a1..e1acfc76ae2f0 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -122,7 +122,7 @@ func loadSSHFrom(rootCfg ConfigProvider) { if len(serverMACs) > 0 { SSH.ServerMACs = serverMACs } - SSH.KeyTestPath = TempDir() + SSH.KeyTestPath = TempPath if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } diff --git a/modules/storage/local.go b/modules/storage/local.go index 5bfaf7791e271..27d83f78bf742 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -36,7 +36,7 @@ func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorag } if config.TemporaryPath == "" { - config.TemporaryPath = filepath.Join(setting.TempDir(), filepath.Base(config.Path)) + config.TemporaryPath = filepath.Join(setting.TempPath, filepath.Base(config.Path)) } if !filepath.IsAbs(config.TemporaryPath) { return nil, fmt.Errorf("LocalStorageConfig.TemporaryPath should be an absolute path, but not: %q", config.TemporaryPath) diff --git a/modules/temp/temp.go b/modules/temp/temp.go new file mode 100644 index 0000000000000..d053dfe7e9e2d --- /dev/null +++ b/modules/temp/temp.go @@ -0,0 +1,28 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package temp + +import ( + "os" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +func MkdirTemp(pattern string) (string, func(), error) { + dir, err := os.MkdirTemp(setting.TempPath, pattern) + if err != nil { + return "", nil, err + } + return dir, func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("Failed to remove temp directory %s: %v", dir, err) + } + }, nil +} + +func CreateTemp(pattern string) (*os.File, error) { + return os.CreateTemp(setting.TempPath, pattern) +} diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index fdc8352f17d74..46af89165e693 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -75,7 +75,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp(setting.TempDir(), "gitea-buffer-") + b.file, err = os.CreateTemp(setting.TempPath, "gitea-buffer-") if err != nil { return 0, err } diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index f215a4db8383b..9d70194dddee3 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -29,7 +29,7 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/services/context" repo_service "code.gitea.io/gitea/services/repository" @@ -303,17 +303,12 @@ var ( func dummyInfoRefs(ctx *context.Context) { infoRefsOnce.Do(func() { - tmpDir, err := os.MkdirTemp(setting.TempDir(), "gitea-info-refs-cache") + tmpDir, cleanup, err := temp.MkdirTemp("gitea-info-refs-cache") if err != nil { log.Error("Failed to create temp dir for git-receive-pack cache: %v", err) return } - - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Error("RemoveAll: %v", err) - } - }() + defer cleanup() if err := git.InitRepository(ctx, tmpDir, true, git.Sha1ObjectFormat.Name()); err != nil { log.Error("Failed to init bare repo for git-receive-pack cache: %v", err) diff --git a/services/pull/patch.go b/services/pull/patch.go index 6e2540d9c4616..42fa661b721bc 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -355,23 +356,22 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * } // 3b. Create a plain patch from head to base - tmpPatchFile, err := os.CreateTemp(setting.TempDir(), "patch") + tmpPatchFile, err := temp.CreateTemp("patch") if err != nil { log.Error("Unable to create temporary patch file! Error: %v", err) return false, fmt.Errorf("unable to create temporary patch file! Error: %w", err) } defer func() { + tmpPatchFile.Close() _ = util.Remove(tmpPatchFile.Name()) }() if err := gitRepo.GetDiffBinary(pr.MergeBase+"...tracking", tmpPatchFile); err != nil { - tmpPatchFile.Close() log.Error("Unable to get patch file from %s to %s in %s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err) return false, fmt.Errorf("unable to get patch file from %s to %s in %s Error: %w", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err) } stat, err := tmpPatchFile.Stat() if err != nil { - tmpPatchFile.Close() return false, fmt.Errorf("unable to stat patch file: %w", err) } patchPath := tmpPatchFile.Name() diff --git a/services/repository/create.go b/services/repository/create.go index 4bb3a950ba403..c9c9afc43365e 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -27,8 +27,8 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/templates/vars" - "code.gitea.io/gitea/modules/util" ) // CreateRepoOptions contains the create repository options @@ -138,10 +138,6 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir return nil } -func createRepoTempDir(repoName string) (string, error) { - return os.MkdirTemp(setting.TempDir(), "repos-"+repoName) -} - // InitRepository initializes README and .gitignore if needed. func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Repository, opts CreateRepoOptions) (err error) { if err = repo_module.CheckInitRepository(ctx, repo); err != nil { @@ -150,15 +146,11 @@ func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Re // Initialize repository according to user's choice. if opts.AutoInit { - tmpDir, err := createRepoTempDir(repo.Name) + tmpDir, cleanup, err := temp.MkdirTemp("repos-" + repo.Name) if err != nil { return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Warn("Unable to remove temporary directory: %s: Error: %v", tmpDir, err) - } - }() + defer cleanup() if err = prepareRepoCommit(ctx, repo, tmpDir, opts); err != nil { return fmt.Errorf("prepareRepoCommit: %w", err) diff --git a/services/repository/generate.go b/services/repository/generate.go index c2d89f42460fb..2bdf1936b32f9 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -22,7 +22,7 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -257,16 +257,11 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { - tmpDir, err := os.MkdirTemp(setting.TempDir(), "gitea-"+repo.Name) + tmpDir, cleanup, err := temp.MkdirTemp("gitea-" + repo.Name) if err != nil { - return fmt.Errorf("Failed to create temp dir for repository %s: %w", repo.FullName(), err) + return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } - - defer func() { - if err := util.RemoveAll(tmpDir); err != nil { - log.Error("RemoveAll: %v", err) - } - }() + defer cleanup() if err = generateRepoCommit(ctx, repo, templateRepo, generateRepo, tmpDir); err != nil { return fmt.Errorf("generateRepoCommit: %w", err) From 3382a56cbac99cc9afb72fa4bf51e7c532c9249e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 16:31:03 -0700 Subject: [PATCH 14/30] improvements --- models/migrations/base/tests.go | 6 ++---- models/unittest/testdb.go | 15 ++++++--------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 013a98f020a61..873ff1142dc39 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -115,10 +115,11 @@ func MainTest(m *testing.M) { setting.CustomConf = giteaConf } - tmpDataPath, _, err := temp.MkdirTemp("data") + tmpDataPath, cleanup, err := temp.MkdirTemp("data") if err != nil { testlogger.Fatalf("Unable to create temporary data path %v\n", err) } + defer cleanup() setting.CustomPath = filepath.Join(setting.AppWorkPath, "custom") setting.AppDataPath = tmpDataPath @@ -135,8 +136,5 @@ func MainTest(m *testing.M) { if err := removeAllWithRetry(setting.RepoRootPath); err != nil { fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) } - if err := removeAllWithRetry(tmpDataPath); err != nil { - fmt.Fprintf(os.Stderr, "os.RemoveAll: %v\n", err) - } os.Exit(exitStatus) } diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 60f6b0052b9ec..975eb1a47ed19 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -93,15 +93,19 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { setting.SSH.Domain = "try.gitea.io" setting.Database.Type = "sqlite3" setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - repoRootPath, _, err := temp.MkdirTemp("repos") + repoRootPath, cleanup1, err := temp.MkdirTemp("repos") if err != nil { fatalTestError("TempDir: %v\n", err) } + defer cleanup1() + setting.RepoRootPath = repoRootPath - appDataPath, _, err := temp.MkdirTemp("appdata") + appDataPath, cleanup2, err := temp.MkdirTemp("appdata") if err != nil { fatalTestError("TempDir: %v\n", err) } + defer cleanup2() + setting.AppDataPath = appDataPath setting.AppWorkPath = giteaRoot setting.StaticRootPath = giteaRoot @@ -154,13 +158,6 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { fatalTestError("tear down failed: %v\n", err) } } - - if err = util.RemoveAll(repoRootPath); err != nil { - fatalTestError("util.RemoveAll: %v\n", err) - } - if err = util.RemoveAll(appDataPath); err != nil { - fatalTestError("util.RemoveAll: %v\n", err) - } os.Exit(exitStatus) } From 3fe5cd75850431de75dde51123b9697ab959df92 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 17:20:32 -0700 Subject: [PATCH 15/30] Fix bug --- modules/setting/setting.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 36cbf56f3d739..0a227549f076c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -186,10 +186,10 @@ func loadRunModeFrom(rootCfg ConfigProvider) { func loadTempDir(rootCfg ConfigProvider) { rootSec := rootCfg.Section("") - tempDir := rootSec.Key("TEMP_DIR").MustString(filepath.Join(os.TempDir(), "gitea")) - if tempDir != "" { - if err := os.MkdirAll(tempDir, os.ModePerm); err != nil { - log.Fatal("Failed to create temp directory %s: %v", tempDir, err) + TempPath = rootSec.Key("TEMP_PATH").MustString(filepath.Join(os.TempDir(), "gitea")) + if TempPath != "" { + if err := os.MkdirAll(TempPath, os.ModePerm); err != nil { + log.Fatal("Failed to create temp directory %s: %v", TempPath, err) } } } From 9afad7562887898d57de01cbedae3264af75d61a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 17:40:00 -0700 Subject: [PATCH 16/30] Fix test --- modules/setting/global.go | 9 +++++++++ modules/setting/setting.go | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/modules/setting/global.go b/modules/setting/global.go index f57e04ef352a2..d92e42dbf9f59 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -3,6 +3,11 @@ package setting +import ( + "os" + "path/filepath" +) + // Global settings var ( // RunUser is the OS user that Gitea is running as. ini:"RUN_USER" @@ -19,3 +24,7 @@ var ( // TempPath is the directory used for temporary files. ini: "TEMP_PATH" TempPath string ) + +func init() { + TempPath = filepath.Join(os.TempDir(), "gitea") +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0a227549f076c..f11e6ca5ecf6c 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -7,7 +7,6 @@ package setting import ( "fmt" "os" - "path/filepath" "runtime" "strings" "time" @@ -186,7 +185,10 @@ func loadRunModeFrom(rootCfg ConfigProvider) { func loadTempDir(rootCfg ConfigProvider) { rootSec := rootCfg.Section("") - TempPath = rootSec.Key("TEMP_PATH").MustString(filepath.Join(os.TempDir(), "gitea")) + tempPath := rootSec.Key("TEMP_PATH").String() + if tempPath != "" { + TempPath = tempPath + } if TempPath != "" { if err := os.MkdirAll(TempPath, os.ModePerm); err != nil { log.Fatal("Failed to create temp directory %s: %v", TempPath, err) From 13a4c9ba053ee74fc8ee0f972f7ba7d2a31a93dc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 17:50:31 -0700 Subject: [PATCH 17/30] some improvements --- modules/repository/temp.go | 12 ++++++------ routers/web/repo/setting/lfs.go | 4 ++-- services/pull/temp_repo.go | 4 ++-- services/repository/files/temp_repo.go | 10 +++++----- services/repository/repository.go | 6 ++---- services/wiki/wiki.go | 8 ++++---- tests/test_utils.go | 2 ++ 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/modules/repository/temp.go b/modules/repository/temp.go index d51c038867afa..1a6794f889464 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -14,8 +14,8 @@ import ( "code.gitea.io/gitea/modules/util" ) -// localCopyPath returns the local repository temporary copy path. -func localCopyPath() string { +// LocalCopyPath returns the local repository temporary copy path. +func LocalCopyPath() string { if setting.Repository.Local.LocalCopyPath == "" { return filepath.Join(setting.TempPath, "local-repo") } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { @@ -26,11 +26,11 @@ func localCopyPath() string { // CreateTemporaryPath creates a temporary path func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { - if err := os.MkdirAll(localCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", localCopyPath(), err) - return "", func() {}, fmt.Errorf("failed to create localcopypath directory %s: %w", localCopyPath(), err) + if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { + log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) + return "", func() {}, fmt.Errorf("failed to create localcopypath directory %s: %w", LocalCopyPath(), err) } - basePath, err := os.MkdirTemp(localCopyPath(), prefix+".git") + basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) return "", func() {}, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) diff --git a/routers/web/repo/setting/lfs.go b/routers/web/repo/setting/lfs.go index f2a5320f6b692..efda9bda58bd5 100644 --- a/routers/web/repo/setting/lfs.go +++ b/routers/web/repo/setting/lfs.go @@ -109,13 +109,13 @@ func LFSLocks(ctx *context.Context) { } // Clone base repo. - tmpBasePath, cancel, err := repo_module.CreateTemporaryPath("locks") + tmpBasePath, cleanup, err := repo_module.CreateTemporaryPath("locks") if err != nil { log.Error("Failed to create temporary path: %v", err) ctx.ServerError("LFSLocks", err) return } - defer cancel() + defer cleanup() if err := git.Clone(ctx, ctx.Repo.Repository.RepoPath(), tmpBasePath, git.CloneRepoOptions{ Bare: true, diff --git a/services/pull/temp_repo.go b/services/pull/temp_repo.go index 1dbaa231b999e..139c8d518f0dc 100644 --- a/services/pull/temp_repo.go +++ b/services/pull/temp_repo.go @@ -74,12 +74,12 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) } // Clone base repo. - tmpBasePath, cancel, err := repo_module.CreateTemporaryPath("pull") + tmpBasePath, cleanup, err := repo_module.CreateTemporaryPath("pull") if err != nil { log.Error("CreateTemporaryPath[%-v]: %v", pr, err) return nil, nil, err } - defer cancel() + defer cleanup() prCtx = &prContext{ Context: ctx, diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 494927ff3e469..c08c7a366b163 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -29,24 +29,24 @@ type TemporaryUploadRepository struct { repo *repo_model.Repository gitRepo *git.Repository basePath string - cancel func() + cleanup func() } // NewTemporaryUploadRepository creates a new temporary upload repository func NewTemporaryUploadRepository(repo *repo_model.Repository) (*TemporaryUploadRepository, error) { - basePath, cancel, err := repo_module.CreateTemporaryPath("upload") + basePath, cleanup, err := repo_module.CreateTemporaryPath("upload") if err != nil { return nil, err } - t := &TemporaryUploadRepository{repo: repo, basePath: basePath, cancel: cancel} + t := &TemporaryUploadRepository{repo: repo, basePath: basePath, cleanup: cleanup} return t, nil } // Close the repository cleaning up all files func (t *TemporaryUploadRepository) Close() { defer t.gitRepo.Close() - if t.cancel != nil { - t.cancel() + if t.cleanup != nil { + t.cleanup() } } diff --git a/services/repository/repository.go b/services/repository/repository.go index fbf00f0072557..fcc617979ef29 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -101,10 +101,8 @@ func Init(ctx context.Context) error { if err := repo_module.LoadRepoConfig(); err != nil { return err } - - if err := system_model.CreateNotice(db.DefaultContext, system_model.NoticeRepository, "Clean up temporary repositories"); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } + system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) + system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) if err := initPushQueue(); err != nil { return err } diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index f94838970129e..45a08dc5d686d 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -102,11 +102,11 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model hasDefaultBranch := gitrepo.IsBranchExist(ctx, repo.WikiStorageRepo(), repo.DefaultWikiBranch) - basePath, cancel, err := repo_module.CreateTemporaryPath("update-wiki") + basePath, cleanup, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { return err } - defer cancel() + defer cleanup() cloneOpts := git.CloneRepoOptions{ Bare: true, @@ -260,11 +260,11 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("InitWiki: %w", err) } - basePath, cancel, err := repo_module.CreateTemporaryPath("update-wiki") + basePath, cleanup, err := repo_module.CreateTemporaryPath("update-wiki") if err != nil { return err } - defer cancel() + defer cleanup() if err := git.Clone(ctx, repo.WikiPath(), basePath, git.CloneRepoOptions{ Bare: true, diff --git a/tests/test_utils.go b/tests/test_utils.go index 14ed759103fcb..96eb5731b4ac1 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/test" @@ -68,6 +69,7 @@ func InitTest(requireGitea bool) { unittest.InitSettings() setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" + _ = util.RemoveAll(repo_module.LocalCopyPath()) if err := git.InitFull(context.Background()); err != nil { log.Fatal("git.InitOnceWithSync: %v", err) From 058edb9fd885da9c690b93a486525f79c51f48f4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 17:54:45 -0700 Subject: [PATCH 18/30] some improvements --- modules/repository/temp.go | 10 +--------- services/pull/temp_repo.go | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 1a6794f889464..33948b3444d48 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -36,16 +36,8 @@ func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { return "", func() {}, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } return basePath, func() { - if err := removeTemporaryPath(basePath); err != nil { + if err := util.RemoveAll(basePath); err != nil { log.Error("Unable to remove temporary directory: %s (%v)", basePath, err) } }, nil } - -// removeTemporaryPath removes the temporary path -func removeTemporaryPath(basePath string) error { - if _, err := os.Stat(basePath); !os.IsNotExist(err) { - return util.RemoveAll(basePath) - } - return nil -} diff --git a/services/pull/temp_repo.go b/services/pull/temp_repo.go index 139c8d518f0dc..d543e3d4a3681 100644 --- a/services/pull/temp_repo.go +++ b/services/pull/temp_repo.go @@ -79,7 +79,7 @@ func createTemporaryRepoForPR(ctx context.Context, pr *issues_model.PullRequest) log.Error("CreateTemporaryPath[%-v]: %v", pr, err) return nil, nil, err } - defer cleanup() + cancel = cleanup prCtx = &prContext{ Context: ctx, From 6187ce0f7acd0ad7ec7dd8649f036e0187fc221b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 21:13:08 -0700 Subject: [PATCH 19/30] some improvements --- custom/conf/app.example.ini | 13 +++++++------ modules/setting/packages.go | 4 ++-- modules/setting/repository.go | 10 ++++++---- modules/setting/ssh.go | 5 ++++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 63e703308d96d..cc65a0ab97260 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -54,6 +54,7 @@ RUN_USER = ; git ;WORK_PATH = ;; The temporary directory, defaults to a directory named gitea under the system temporary directory +;; All other temporary directories are relative to this directory by default ;TEMP_PATH = ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1075,8 +1076,8 @@ LEVEL = Info ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; -;; Path for local repository copy. Defaults to `tmp/local-repo` (content gets deleted on gitea restart) -;LOCAL_COPY_PATH = tmp/local-repo +;; Path for local repository copy. Defaults to `local-repo` under `TEMP_PATH` except it's an absolute path (content gets deleted on gitea restart) +;LOCAL_COPY_PATH = local-repo ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1087,8 +1088,8 @@ LEVEL = Info ;; Whether repository file uploads are enabled. Defaults to `true` ;ENABLED = true ;; -;; Path for uploads. Defaults to `data/tmp/uploads` (content gets deleted on gitea restart) -;TEMP_PATH = data/tmp/uploads +;; Path for uploads. Defaults to `uploads` under `TEMP_PATH` except it's an absolute path (content gets deleted on gitea restart) +;TEMP_PATH = uploads ;; ;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. ;ALLOWED_TYPES = @@ -2588,8 +2589,8 @@ LEVEL = Info ;; Currently, only `minio` and `azureblob` is supported. ;SERVE_DIRECT = false ;; -;; Path for chunked uploads. Defaults to APP_DATA_PATH + `tmp/package-upload` -;CHUNKED_UPLOAD_PATH = tmp/package-upload +;; Path for chunked uploads. Defaults to `package-upload` under `TEMP_PATH` except it's an absolute path. +;CHUNKED_UPLOAD_PATH = package-upload ;; ;; Maximum count of package versions a single owner can have (`-1` means no limits) ;LIMIT_TOTAL_OWNER_COUNT = -1 diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 3f618cfd64115..b79a476390328 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -67,9 +67,9 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { return err } - Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("tmp/package-upload")) + Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("package-upload")) if !filepath.IsAbs(Packages.ChunkedUploadPath) { - Packages.ChunkedUploadPath = filepath.ToSlash(filepath.Join(AppDataPath, Packages.ChunkedUploadPath)) + Packages.ChunkedUploadPath = filepath.Join(TempPath, Packages.ChunkedUploadPath) } if HasInstallLock(rootCfg) { diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 43bfb3256d0aa..474670b27e1e8 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -187,7 +187,7 @@ var ( MaxFiles int }{ Enabled: true, - TempPath: "data/tmp/uploads", + TempPath: "uploads", AllowedTypes: "", FileMaxSize: 50, MaxFiles: 5, @@ -197,7 +197,7 @@ var ( Local: struct { LocalCopyPath string }{ - LocalCopyPath: "tmp/local-repo", + LocalCopyPath: "local-repo", }, // Pull request settings @@ -361,8 +361,10 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } } - if !filepath.IsAbs(Repository.Upload.TempPath) { - Repository.Upload.TempPath = filepath.Join(AppWorkPath, Repository.Upload.TempPath) + if Repository.Upload.TempPath == "" { + Repository.Upload.TempPath = filepath.Join(TempPath, "uploads") + } else if !filepath.IsAbs(Repository.Upload.TempPath) { + Repository.Upload.TempPath = filepath.Join(TempPath, Repository.Upload.TempPath) } if err := loadRepoArchiveFrom(rootCfg); err != nil { diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index e1acfc76ae2f0..d5a2fa1cf8a83 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -122,7 +122,10 @@ func loadSSHFrom(rootCfg ConfigProvider) { if len(serverMACs) > 0 { SSH.ServerMACs = serverMACs } - SSH.KeyTestPath = TempPath + SSH.KeyTestPath = sec.Key("SSH_KEY_TEST_PATH").MustString("ssh_key_test") + if !filepath.IsAbs(SSH.KeyTestPath) { + SSH.KeyTestPath = filepath.Join(TempPath, SSH.KeyTestPath) + } if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } From d6c9ed35af19ec6d3a75f8ab7d04b57dff2acd80 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 27 Mar 2025 23:28:37 -0700 Subject: [PATCH 20/30] Fix test --- modules/setting/ssh.go | 15 +++++++++++---- modules/ssh/init.go | 5 ----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index d5a2fa1cf8a83..8c920c9cd5c05 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,6 +4,7 @@ package setting import ( + "os" "path/filepath" "strings" "text/template" @@ -60,6 +61,7 @@ var SSH = struct { MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, + KeyTestPath: "ssh_key_test", AuthorizedKeysCommandTemplate: "{{.AppPath}} --config={{.CustomConf}} serv key-{{.Key.ID}}", PerWriteTimeout: PerWriteTimeout, PerWritePerKbTimeout: PerWritePerKbTimeout, @@ -122,10 +124,6 @@ func loadSSHFrom(rootCfg ConfigProvider) { if len(serverMACs) > 0 { SSH.ServerMACs = serverMACs } - SSH.KeyTestPath = sec.Key("SSH_KEY_TEST_PATH").MustString("ssh_key_test") - if !filepath.IsAbs(SSH.KeyTestPath) { - SSH.KeyTestPath = filepath.Join(TempPath, SSH.KeyTestPath) - } if err = sec.MapTo(&SSH); err != nil { log.Fatal("Failed to map SSH settings: %v", err) } @@ -135,6 +133,15 @@ func loadSSHFrom(rootCfg ConfigProvider) { } } + SSH.KeyTestPath = sec.Key("SSH_KEY_TEST_PATH").MustString("ssh_key_test") + if !filepath.IsAbs(SSH.KeyTestPath) { + SSH.KeyTestPath = filepath.Join(TempPath, SSH.KeyTestPath) + } + // FIXME: why 0o644 for a directory ..... + if err := os.MkdirAll(SSH.KeyTestPath, 0o644); err != nil { + log.Fatal("failed to create directory %q for ssh key test: %w", SSH.KeyTestPath, err) + } + SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) diff --git a/modules/ssh/init.go b/modules/ssh/init.go index 21d4f8993611f..fdc11632e23e5 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -32,11 +32,6 @@ func Init() error { builtinUnused() - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(setting.SSH.KeyTestPath, 0o644); err != nil { - return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.SSH.KeyTestPath, err) - } - if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) From 69c9a1eb9d5b543286be7f7c2096199799c2d3e8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 28 Mar 2025 11:35:08 -0700 Subject: [PATCH 21/30] Fix test --- modules/setting/ssh.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index 8c920c9cd5c05..2a4ebf8dd5a89 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -137,8 +137,7 @@ func loadSSHFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(SSH.KeyTestPath) { SSH.KeyTestPath = filepath.Join(TempPath, SSH.KeyTestPath) } - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(SSH.KeyTestPath, 0o644); err != nil { + if err := os.MkdirAll(SSH.KeyTestPath, os.ModePerm); err != nil { log.Fatal("failed to create directory %q for ssh key test: %w", SSH.KeyTestPath, err) } From b88c3a6af37cb3785688b4a0c0beb232d93fad3b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 4 Apr 2025 12:37:27 -0700 Subject: [PATCH 22/30] improvements --- custom/conf/app.example.ini | 6 +++--- modules/repository/temp.go | 14 +++++++------- modules/setting/setting.go | 15 +++++++++++---- modules/storage/local.go | 2 +- modules/util/path.go | 8 ++++++++ modules/util/path_test.go | 17 +++++++++++++++++ services/repository/repository.go | 3 --- tests/test_utils.go | 2 -- 8 files changed, 47 insertions(+), 20 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 7296c6185404f..a0cdc8d79d3c0 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1079,7 +1079,7 @@ LEVEL = Info ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; -;; Path for local repository copy. Defaults to `local-repo` under `TEMP_PATH` except it's an absolute path (content gets deleted on gitea restart) +;; Path for local repository copy. Defaults to TEMP_PATH + `local-repo` ;LOCAL_COPY_PATH = local-repo ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1091,7 +1091,7 @@ LEVEL = Info ;; Whether repository file uploads are enabled. Defaults to `true` ;ENABLED = true ;; -;; Path for uploads. Defaults to `uploads` under `TEMP_PATH` except it's an absolute path (content gets deleted on gitea restart) +;; Path for uploads. Defaults to TEMP_PATH + `uploads` ;TEMP_PATH = uploads ;; ;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. @@ -2592,7 +2592,7 @@ LEVEL = Info ;; Currently, only `minio` and `azureblob` is supported. ;SERVE_DIRECT = false ;; -;; Path for chunked uploads. Defaults to `package-upload` under `TEMP_PATH` except it's an absolute path. +;; Path for chunked uploads. Defaults it's `package-upload` under `TEMP_PATH` unless it's an absolute path. ;CHUNKED_UPLOAD_PATH = package-upload ;; ;; Maximum count of package versions a single owner can have (`-1` means no limits) diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 33948b3444d48..d2151e553122b 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -14,8 +14,8 @@ import ( "code.gitea.io/gitea/modules/util" ) -// LocalCopyPath returns the local repository temporary copy path. -func LocalCopyPath() string { +// localCopyPath returns the local repository temporary copy path. +func localCopyPath() string { if setting.Repository.Local.LocalCopyPath == "" { return filepath.Join(setting.TempPath, "local-repo") } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { @@ -26,14 +26,14 @@ func LocalCopyPath() string { // CreateTemporaryPath creates a temporary path func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { - if err := os.MkdirAll(LocalCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", LocalCopyPath(), err) - return "", func() {}, fmt.Errorf("failed to create localcopypath directory %s: %w", LocalCopyPath(), err) + if err := os.MkdirAll(localCopyPath(), os.ModePerm); err != nil { + log.Error("Unable to create localcopypath directory: %s (%v)", localCopyPath(), err) + return "", nil, fmt.Errorf("failed to create localcopypath directory %s: %w", localCopyPath(), err) } - basePath, err := os.MkdirTemp(LocalCopyPath(), prefix+".git") + basePath, err := os.MkdirTemp(localCopyPath(), prefix+".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) - return "", func() {}, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) + return "", nil, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } return basePath, func() { if err := util.RemoveAll(basePath); err != nil { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index f11e6ca5ecf6c..03f515fa5194b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -7,6 +7,7 @@ package setting import ( "fmt" "os" + "path/filepath" "runtime" "strings" "time" @@ -187,12 +188,18 @@ func loadTempDir(rootCfg ConfigProvider) { rootSec := rootCfg.Section("") tempPath := rootSec.Key("TEMP_PATH").String() if tempPath != "" { + if !filepath.IsAbs(tempPath) { + tempPath = filepath.Join(os.TempDir(), tempPath) + } TempPath = tempPath } - if TempPath != "" { - if err := os.MkdirAll(TempPath, os.ModePerm); err != nil { - log.Fatal("Failed to create temp directory %s: %v", TempPath, err) - } + // TempPath has been initialized in init function of global.go + if TempPath == "" { + log.Fatal("It's impossible that TEMP_PATH is empty") + } + + if err := os.MkdirAll(TempPath, os.ModePerm); err != nil { + log.Fatal("Failed to create temp directory %s: %v", TempPath, err) } } diff --git a/modules/storage/local.go b/modules/storage/local.go index 27d83f78bf742..eb4f0ff7461d0 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -36,7 +36,7 @@ func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorag } if config.TemporaryPath == "" { - config.TemporaryPath = filepath.Join(setting.TempPath, filepath.Base(config.Path)) + config.TemporaryPath = filepath.Join(setting.TempPath, "storage", util.SanitizeDirName(config.Path)) } if !filepath.IsAbs(config.TemporaryPath) { return nil, fmt.Errorf("LocalStorageConfig.TemporaryPath should be an absolute path, but not: %q", config.TemporaryPath) diff --git a/modules/util/path.go b/modules/util/path.go index 0e56348978092..57f91fcfca73c 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -291,3 +291,11 @@ func IsReadmeFileExtension(name string, ext ...string) (int, bool) { return 0, false } + +// SanitizeDirName replaces illegal characters in a directory name with "_" +func SanitizeDirName(path string) string { + p := filepath.Clean(path) + re := regexp.MustCompile(`[<>:"/\\|?*]`) + sanitized := re.ReplaceAllString(p, "_") + return sanitized +} diff --git a/modules/util/path_test.go b/modules/util/path_test.go index 79c37e55f7a57..45fe25657df74 100644 --- a/modules/util/path_test.go +++ b/modules/util/path_test.go @@ -230,3 +230,20 @@ func TestListDirRecursively(t *testing.T) { require.NoError(t, err) assert.ElementsMatch(t, []string{"d1/f-d1", "d1/s1/f-d1s1"}, res) } + +func TestSanitizeDirName(t *testing.T) { + cases := []struct { + path string + expected string + }{ + {"a", "a"}, + {"a/b", "a_b"}, + {"/a/b", "_a_b"}, + {"a/b/c", "a_b_c"}, + {"c:\\a\\b", "c__a_b"}, + {"c:\\a/b", "c__a_b"}, + } + for _, c := range cases { + assert.Equal(t, c.expected, SanitizeDirName(c.path)) + } +} diff --git a/services/repository/repository.go b/services/repository/repository.go index fcc617979ef29..10c8a0cf6cec0 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -12,7 +12,6 @@ import ( issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/organization" repo_model "code.gitea.io/gitea/models/repo" - system_model "code.gitea.io/gitea/models/system" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" @@ -101,8 +100,6 @@ func Init(ctx context.Context) error { if err := repo_module.LoadRepoConfig(); err != nil { return err } - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repository uploads", setting.Repository.Upload.TempPath) - system_model.RemoveAllWithNotice(ctx, "Clean up temporary repositories", repo_module.LocalCopyPath()) if err := initPushQueue(); err != nil { return err } diff --git a/tests/test_utils.go b/tests/test_utils.go index 96eb5731b4ac1..14ed759103fcb 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/test" @@ -69,7 +68,6 @@ func InitTest(requireGitea bool) { unittest.InitSettings() setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - _ = util.RemoveAll(repo_module.LocalCopyPath()) if err := git.InitFull(context.Background()); err != nil { log.Fatal("git.InitOnceWithSync: %v", err) From 85d5fc66f3f41b373b0ec6a077dd32ebc0cf287c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 5 Apr 2025 14:28:07 -0700 Subject: [PATCH 23/30] Remove sub temporary path configurations with a fixed sub path --- cmd/dump.go | 6 ++--- custom/conf/app.example.ini | 19 -------------- models/asymkey/ssh_key_parse.go | 2 +- models/repo/upload.go | 2 +- modules/repository/temp.go | 7 +---- modules/setting/packages.go | 18 ++++++------- modules/setting/repository.go | 27 +++----------------- modules/setting/ssh.go | 15 +++-------- modules/ssh/init.go | 5 ++++ services/packages/container/blob_uploader.go | 2 +- 10 files changed, 28 insertions(+), 75 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 270e82d8706a8..6ebbf58909dd0 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -48,7 +48,7 @@ var CmdDump = &cli.Command{ &cli.StringFlag{ Name: "tempdir", Aliases: []string{"t"}, - Value: setting.TempPath, + Value: filepath.Join(setting.TempPath, "dump"), Usage: "Temporary dir path", }, &cli.StringFlag{ @@ -194,8 +194,8 @@ func runDump(ctx *cli.Context) error { log.Info("Skipping database") } else { tmpDir := ctx.String("tempdir") - if _, err := os.Stat(tmpDir); os.IsNotExist(err) { - fatal("Path does not exist: %s", tmpDir) + if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil { + fatal("Unable to create temporary directory: %s (%v)", tmpDir, err) } dbDump, err := os.CreateTemp(tmpDir, "gitea-db.sql") diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index a0cdc8d79d3c0..040361219549c 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -201,10 +201,6 @@ RUN_USER = ; git ;; relative paths are made absolute relative to the APP_DATA_PATH ;SSH_SERVER_HOST_KEYS=ssh/gitea.rsa, ssh/gogs.rsa ;; -;; Directory to create temporary files in when testing public keys using ssh-keygen, -;; default is the system temporary directory. -;SSH_KEY_TEST_PATH = -;; ;; Use `ssh-keygen` to parse public SSH keys. The value is passed to the shell. By default, Gitea does the parsing itself. ;SSH_KEYGEN_PATH = ;; @@ -1073,15 +1069,6 @@ LEVEL = Info ;; Separate extensions with a comma. To line wrap files without an extension, just put a comma ;LINE_WRAP_EXTENSIONS = .txt,.md,.markdown,.mdown,.mkd,.livemd, -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;[repository.local] -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; -;; Path for local repository copy. Defaults to TEMP_PATH + `local-repo` -;LOCAL_COPY_PATH = local-repo - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;[repository.upload] @@ -1091,9 +1078,6 @@ LEVEL = Info ;; Whether repository file uploads are enabled. Defaults to `true` ;ENABLED = true ;; -;; Path for uploads. Defaults to TEMP_PATH + `uploads` -;TEMP_PATH = uploads -;; ;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types. ;ALLOWED_TYPES = ;; @@ -2592,9 +2576,6 @@ LEVEL = Info ;; Currently, only `minio` and `azureblob` is supported. ;SERVE_DIRECT = false ;; -;; Path for chunked uploads. Defaults it's `package-upload` under `TEMP_PATH` unless it's an absolute path. -;CHUNKED_UPLOAD_PATH = package-upload -;; ;; Maximum count of package versions a single owner can have (`-1` means no limits) ;LIMIT_TOTAL_OWNER_COUNT = -1 ;; Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/models/asymkey/ssh_key_parse.go b/models/asymkey/ssh_key_parse.go index c84352571832d..ef92af6fa2313 100644 --- a/models/asymkey/ssh_key_parse.go +++ b/models/asymkey/ssh_key_parse.go @@ -262,7 +262,7 @@ func SSHNativeParsePublicKey(keyLine string) (string, int, error) { // writeTmpKeyFile writes key content to a temporary file // and returns the name of that file, along with any possible errors. func writeTmpKeyFile(content string) (string, error) { - tmpFile, err := os.CreateTemp(setting.SSH.KeyTestPath, "gitea_keytest") + tmpFile, err := os.CreateTemp(setting.GetSSHKeyTestPath(), "gitea_keytest") if err != nil { return "", fmt.Errorf("TempFile: %w", err) } diff --git a/models/repo/upload.go b/models/repo/upload.go index cae11df96a3f3..858ffd18df58c 100644 --- a/models/repo/upload.go +++ b/models/repo/upload.go @@ -53,7 +53,7 @@ func init() { // UploadLocalPath returns where uploads is stored in local file system based on given UUID. func UploadLocalPath(uuid string) string { - return filepath.Join(setting.Repository.Upload.TempPath, uuid[0:1], uuid[1:2], uuid) + return filepath.Join(setting.GetRepositoryUploadTempPath(), uuid[0:1], uuid[1:2], uuid) } // LocalPath returns where uploads are temporarily stored in local file system. diff --git a/modules/repository/temp.go b/modules/repository/temp.go index d2151e553122b..1befe4a5c5ebe 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -16,12 +16,7 @@ import ( // localCopyPath returns the local repository temporary copy path. func localCopyPath() string { - if setting.Repository.Local.LocalCopyPath == "" { - return filepath.Join(setting.TempPath, "local-repo") - } else if !filepath.IsAbs(setting.Repository.Local.LocalCopyPath) { - return filepath.Join(setting.TempPath, setting.Repository.Local.LocalCopyPath) - } - return setting.Repository.Local.LocalCopyPath + return filepath.Join(setting.TempPath, "local-repo") } // CreateTemporaryPath creates a temporary path diff --git a/modules/setting/packages.go b/modules/setting/packages.go index b79a476390328..6c8c7a308111b 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -15,9 +15,8 @@ import ( // Package registry settings var ( Packages = struct { - Storage *Storage - Enabled bool - ChunkedUploadPath string + Storage *Storage + Enabled bool LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 @@ -51,6 +50,10 @@ var ( } ) +func GetPacakgeUploadTempPath() string { + return filepath.Join(TempPath, "package-upload") +} + func loadPackagesFrom(rootCfg ConfigProvider) (err error) { sec, _ := rootCfg.GetSection("packages") if sec == nil { @@ -67,14 +70,9 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { return err } - Packages.ChunkedUploadPath = filepath.ToSlash(sec.Key("CHUNKED_UPLOAD_PATH").MustString("package-upload")) - if !filepath.IsAbs(Packages.ChunkedUploadPath) { - Packages.ChunkedUploadPath = filepath.Join(TempPath, Packages.ChunkedUploadPath) - } - if HasInstallLock(rootCfg) { - if err := os.MkdirAll(Packages.ChunkedUploadPath, os.ModePerm); err != nil { - return fmt.Errorf("unable to create chunked upload directory: %s (%v)", Packages.ChunkedUploadPath, err) + if err := os.MkdirAll(GetPacakgeUploadTempPath(), os.ModePerm); err != nil { + return fmt.Errorf("unable to create chunked upload directory: %s (%v)", GetPacakgeUploadTempPath(), err) } } diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 474670b27e1e8..8a0dd791314c0 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -62,17 +62,11 @@ var ( // Repository upload settings Upload struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int } `ini:"-"` - // Repository local settings - Local struct { - LocalCopyPath string - } `ini:"-"` - // Pull request settings PullRequest struct { WorkInProgressPrefixes []string @@ -181,25 +175,16 @@ var ( // Repository upload settings Upload: struct { Enabled bool - TempPath string AllowedTypes string FileMaxSize int64 MaxFiles int }{ Enabled: true, - TempPath: "uploads", AllowedTypes: "", FileMaxSize: 50, MaxFiles: 5, }, - // Repository local settings - Local: struct { - LocalCopyPath string - }{ - LocalCopyPath: "local-repo", - }, - // Pull request settings PullRequest: struct { WorkInProgressPrefixes []string @@ -308,8 +293,6 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { log.Fatal("Failed to map Repository.Editor settings: %v", err) } else if err = rootCfg.Section("repository.upload").MapTo(&Repository.Upload); err != nil { log.Fatal("Failed to map Repository.Upload settings: %v", err) - } else if err = rootCfg.Section("repository.local").MapTo(&Repository.Local); err != nil { - log.Fatal("Failed to map Repository.Local settings: %v", err) } else if err = rootCfg.Section("repository.pull-request").MapTo(&Repository.PullRequest); err != nil { log.Fatal("Failed to map Repository.PullRequest settings: %v", err) } @@ -361,13 +344,11 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { } } - if Repository.Upload.TempPath == "" { - Repository.Upload.TempPath = filepath.Join(TempPath, "uploads") - } else if !filepath.IsAbs(Repository.Upload.TempPath) { - Repository.Upload.TempPath = filepath.Join(TempPath, Repository.Upload.TempPath) - } - if err := loadRepoArchiveFrom(rootCfg); err != nil { log.Fatal("loadRepoArchiveFrom: %v", err) } } + +func GetRepositoryUploadTempPath() string { + return filepath.Join(TempPath, "uploads") +} diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index 2a4ebf8dd5a89..770593df4b802 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,7 +4,6 @@ package setting import ( - "os" "path/filepath" "strings" "text/template" @@ -31,7 +30,6 @@ var SSH = struct { ServerKeyExchanges []string `ini:"SSH_SERVER_KEY_EXCHANGES"` ServerMACs []string `ini:"SSH_SERVER_MACS"` ServerHostKeys []string `ini:"SSH_SERVER_HOST_KEYS"` - KeyTestPath string `ini:"SSH_KEY_TEST_PATH"` KeygenPath string `ini:"SSH_KEYGEN_PATH"` AuthorizedKeysBackup bool `ini:"SSH_AUTHORIZED_KEYS_BACKUP"` AuthorizedPrincipalsBackup bool `ini:"SSH_AUTHORIZED_PRINCIPALS_BACKUP"` @@ -61,7 +59,6 @@ var SSH = struct { MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, - KeyTestPath: "ssh_key_test", AuthorizedKeysCommandTemplate: "{{.AppPath}} --config={{.CustomConf}} serv key-{{.Key.ID}}", PerWriteTimeout: PerWriteTimeout, PerWritePerKbTimeout: PerWritePerKbTimeout, @@ -99,6 +96,10 @@ func parseAuthorizedPrincipalsAllow(values []string) ([]string, bool) { return authorizedPrincipalsAllow, true } +func GetSSHKeyTestPath() string { + return filepath.Join(TempPath, "ssh_key_test") +} + func loadSSHFrom(rootCfg ConfigProvider) { sec := rootCfg.Section("server") if len(SSH.Domain) == 0 { @@ -133,14 +134,6 @@ func loadSSHFrom(rootCfg ConfigProvider) { } } - SSH.KeyTestPath = sec.Key("SSH_KEY_TEST_PATH").MustString("ssh_key_test") - if !filepath.IsAbs(SSH.KeyTestPath) { - SSH.KeyTestPath = filepath.Join(TempPath, SSH.KeyTestPath) - } - if err := os.MkdirAll(SSH.KeyTestPath, os.ModePerm); err != nil { - log.Fatal("failed to create directory %q for ssh key test: %w", SSH.KeyTestPath, err) - } - SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) diff --git a/modules/ssh/init.go b/modules/ssh/init.go index fdc11632e23e5..1ea3dfe7e7172 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -32,6 +32,11 @@ func Init() error { builtinUnused() + // FIXME: why 0o644 for a directory ..... + if err := os.MkdirAll(setting.GetSSHKeyTestPath(), 0o644); err != nil { + return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.GetSSHKeyTestPath(), err) + } + if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) diff --git a/services/packages/container/blob_uploader.go b/services/packages/container/blob_uploader.go index bae2e2d6af667..81aed7e50ae09 100644 --- a/services/packages/container/blob_uploader.go +++ b/services/packages/container/blob_uploader.go @@ -31,7 +31,7 @@ type BlobUploader struct { } func buildFilePath(id string) string { - return util.FilePathJoinAbs(setting.Packages.ChunkedUploadPath, id) + return util.FilePathJoinAbs(setting.GetPacakgeUploadTempPath(), id) } // NewBlobUploader creates a new blob uploader for the given id From 7897bc60ee3d802a7ce43f54bc032929124dd264 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 5 Apr 2025 14:49:07 -0700 Subject: [PATCH 24/30] Fix bug --- modules/setting/ssh.go | 6 ++++++ modules/ssh/init.go | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index 770593df4b802..577aa839c059c 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,6 +4,7 @@ package setting import ( + "os" "path/filepath" "strings" "text/template" @@ -134,6 +135,11 @@ func loadSSHFrom(rootCfg ConfigProvider) { } } + // FIXME: why 0o644 for a directory ..... + if err := os.MkdirAll(GetSSHKeyTestPath(), 0o644); err != nil { + log.Fatal("failed to create directory %q for ssh key test: %w", GetSSHKeyTestPath(), err) + } + SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) diff --git a/modules/ssh/init.go b/modules/ssh/init.go index 1ea3dfe7e7172..fdc11632e23e5 100644 --- a/modules/ssh/init.go +++ b/modules/ssh/init.go @@ -32,11 +32,6 @@ func Init() error { builtinUnused() - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(setting.GetSSHKeyTestPath(), 0o644); err != nil { - return fmt.Errorf("failed to create directory %q for ssh key test: %w", setting.GetSSHKeyTestPath(), err) - } - if len(setting.SSH.TrustedUserCAKeys) > 0 && setting.SSH.AuthorizedPrincipalsEnabled { caKeysFileName := setting.SSH.TrustedUserCAKeysFile caKeysFileDir := filepath.Dir(caKeysFileName) From 4d89d5607db4f7d91c3f2c6e2dcdb1ac5919c41c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 5 Apr 2025 16:19:44 -0700 Subject: [PATCH 25/30] Fix bug --- modules/setting/ssh.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index 577aa839c059c..b30987f9cc4c1 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -135,8 +135,7 @@ func loadSSHFrom(rootCfg ConfigProvider) { } } - // FIXME: why 0o644 for a directory ..... - if err := os.MkdirAll(GetSSHKeyTestPath(), 0o644); err != nil { + if err := os.MkdirAll(GetSSHKeyTestPath(), os.ModePerm); err != nil { log.Fatal("failed to create directory %q for ssh key test: %w", GetSSHKeyTestPath(), err) } From 834bb59842026b81013921fdb35dde1d871b8997 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 7 Apr 2025 09:43:11 +0800 Subject: [PATCH 26/30] revert dump tmp dir, remove SSH_KEYGEN_PATH --- cmd/dump.go | 6 +-- custom/conf/app.example.ini | 3 -- models/asymkey/ssh_key_fingerprint.go | 44 +---------------- models/asymkey/ssh_key_parse.go | 71 +-------------------------- models/asymkey/ssh_key_test.go | 30 ----------- modules/setting/ssh.go | 12 ----- options/locale/locale_en-US.ini | 2 - templates/admin/config.tmpl | 4 -- 8 files changed, 7 insertions(+), 165 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 6ebbf58909dd0..7d640b78fdfc0 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -48,7 +48,7 @@ var CmdDump = &cli.Command{ &cli.StringFlag{ Name: "tempdir", Aliases: []string{"t"}, - Value: filepath.Join(setting.TempPath, "dump"), + Value: os.TempDir(), Usage: "Temporary dir path", }, &cli.StringFlag{ @@ -194,8 +194,8 @@ func runDump(ctx *cli.Context) error { log.Info("Skipping database") } else { tmpDir := ctx.String("tempdir") - if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil { - fatal("Unable to create temporary directory: %s (%v)", tmpDir, err) + if _, err := os.Stat(tmpDir); os.IsNotExist(err) { + fatal("Path does not exist: %s", tmpDir) } dbDump, err := os.CreateTemp(tmpDir, "gitea-db.sql") diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 3733772bd22dd..a76ed586e546e 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -201,9 +201,6 @@ RUN_USER = ; git ;; relative paths are made absolute relative to the APP_DATA_PATH ;SSH_SERVER_HOST_KEYS=ssh/gitea.rsa, ssh/gogs.rsa ;; -;; Use `ssh-keygen` to parse public SSH keys. The value is passed to the shell. By default, Gitea does the parsing itself. -;SSH_KEYGEN_PATH = -;; ;; Enable SSH Authorized Key Backup when rewriting all keys, default is false ;SSH_AUTHORIZED_KEYS_BACKUP = false ;; diff --git a/models/asymkey/ssh_key_fingerprint.go b/models/asymkey/ssh_key_fingerprint.go index 1ed3b5df2a19e..4dcfe1f27925a 100644 --- a/models/asymkey/ssh_key_fingerprint.go +++ b/models/asymkey/ssh_key_fingerprint.go @@ -6,27 +6,13 @@ package asymkey import ( "context" "fmt" - "strings" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "golang.org/x/crypto/ssh" "xorm.io/builder" ) -// ___________.__ .__ __ -// \_ _____/|__| ____ ____ ________________________|__| _____/ |_ -// | __) | |/ \ / ___\_/ __ \_ __ \____ \_ __ \ |/ \ __\ -// | \ | | | \/ /_/ > ___/| | \/ |_> > | \/ | | \ | -// \___ / |__|___| /\___ / \___ >__| | __/|__| |__|___| /__| -// \/ \//_____/ \/ |__| \/ -// -// This file contains functions for fingerprinting SSH keys -// // The database is used in checkKeyFingerprint however most of these functions probably belong in a module // checkKeyFingerprint only checks if key fingerprint has been used as public key, @@ -41,29 +27,6 @@ func checkKeyFingerprint(ctx context.Context, fingerprint string) error { return nil } -func calcFingerprintSSHKeygen(publicKeyContent string) (string, error) { - // Calculate fingerprint. - tmpPath, err := writeTmpKeyFile(publicKeyContent) - if err != nil { - return "", err - } - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary key file: %s: Error: %v", tmpPath, err) - } - }() - stdout, stderr, err := process.GetManager().Exec("AddPublicKey", "ssh-keygen", "-lf", tmpPath) - if err != nil { - if strings.Contains(stderr, "is not a public key file") { - return "", ErrKeyUnableVerify{stderr} - } - return "", util.NewInvalidArgumentErrorf("'ssh-keygen -lf %s' failed with error '%s': %s", tmpPath, err, stderr) - } else if len(stdout) < 2 { - return "", util.NewInvalidArgumentErrorf("not enough output for calculating fingerprint: %s", stdout) - } - return strings.Split(stdout, " ")[1], nil -} - func calcFingerprintNative(publicKeyContent string) (string, error) { // Calculate fingerprint. pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKeyContent)) @@ -75,15 +38,12 @@ func calcFingerprintNative(publicKeyContent string) (string, error) { // CalcFingerprint calculate public key's fingerprint func CalcFingerprint(publicKeyContent string) (string, error) { - // Call the method based on configuration - useNative := setting.SSH.KeygenPath == "" - calcFn := util.Iif(useNative, calcFingerprintNative, calcFingerprintSSHKeygen) - fp, err := calcFn(publicKeyContent) + fp, err := calcFingerprintNative(publicKeyContent) if err != nil { if IsErrKeyUnableVerify(err) { return "", err } - return "", fmt.Errorf("CalcFingerprint(%s): %w", util.Iif(useNative, "native", "ssh-keygen"), err) + return "", fmt.Errorf("CalcFingerprint: %w", err) } return fp, nil } diff --git a/models/asymkey/ssh_key_parse.go b/models/asymkey/ssh_key_parse.go index ef92af6fa2313..46dcf4d89486a 100644 --- a/models/asymkey/ssh_key_parse.go +++ b/models/asymkey/ssh_key_parse.go @@ -13,12 +13,9 @@ import ( "errors" "fmt" "math/big" - "os" - "strconv" "strings" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -175,20 +172,9 @@ func CheckPublicKeyString(content string) (_ string, err error) { return content, nil } - var ( - fnName string - keyType string - length int - ) - if len(setting.SSH.KeygenPath) == 0 { - fnName = "SSHNativeParsePublicKey" - keyType, length, err = SSHNativeParsePublicKey(content) - } else { - fnName = "SSHKeyGenParsePublicKey" - keyType, length, err = SSHKeyGenParsePublicKey(content) - } + keyType, length, err := SSHNativeParsePublicKey(content) if err != nil { - return "", fmt.Errorf("%s: %w", fnName, err) + return "", fmt.Errorf("SSHNativeParsePublicKey: %w", err) } log.Trace("Key info [native: %v]: %s-%d", setting.SSH.StartBuiltinServer, keyType, length) @@ -258,56 +244,3 @@ func SSHNativeParsePublicKey(keyLine string) (string, int, error) { } return "", 0, fmt.Errorf("unsupported key length detection for type: %s", pkey.Type()) } - -// writeTmpKeyFile writes key content to a temporary file -// and returns the name of that file, along with any possible errors. -func writeTmpKeyFile(content string) (string, error) { - tmpFile, err := os.CreateTemp(setting.GetSSHKeyTestPath(), "gitea_keytest") - if err != nil { - return "", fmt.Errorf("TempFile: %w", err) - } - defer tmpFile.Close() - - if _, err = tmpFile.WriteString(content); err != nil { - return "", fmt.Errorf("WriteString: %w", err) - } - return tmpFile.Name(), nil -} - -// SSHKeyGenParsePublicKey extracts key type and length using ssh-keygen. -func SSHKeyGenParsePublicKey(key string) (string, int, error) { - tmpName, err := writeTmpKeyFile(key) - if err != nil { - return "", 0, fmt.Errorf("writeTmpKeyFile: %w", err) - } - defer func() { - if err := util.Remove(tmpName); err != nil { - log.Warn("Unable to remove temporary key file: %s: Error: %v", tmpName, err) - } - }() - - keygenPath := setting.SSH.KeygenPath - if len(keygenPath) == 0 { - keygenPath = "ssh-keygen" - } - - stdout, stderr, err := process.GetManager().Exec("SSHKeyGenParsePublicKey", keygenPath, "-lf", tmpName) - if err != nil { - return "", 0, fmt.Errorf("fail to parse public key: %s - %s", err, stderr) - } - if strings.Contains(stdout, "is not a public key file") { - return "", 0, ErrKeyUnableVerify{stdout} - } - - fields := strings.Split(stdout, " ") - if len(fields) < 4 { - return "", 0, fmt.Errorf("invalid public key line: %s", stdout) - } - - keyType := strings.Trim(fields[len(fields)-1], "()\r\n") - length, err := strconv.ParseInt(fields[0], 10, 32) - if err != nil { - return "", 0, err - } - return strings.ToLower(keyType), int(length), nil -} diff --git a/models/asymkey/ssh_key_test.go b/models/asymkey/ssh_key_test.go index b33d16030d57a..21e4ddf62eced 100644 --- a/models/asymkey/ssh_key_test.go +++ b/models/asymkey/ssh_key_test.go @@ -18,7 +18,6 @@ import ( "github.com/42wim/sshsig" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func Test_SSHParsePublicKey(t *testing.T) { @@ -45,27 +44,6 @@ func Test_SSHParsePublicKey(t *testing.T) { assert.Equal(t, tc.keyType, keyTypeN) assert.Equal(t, tc.length, lengthN) }) - if tc.skipSSHKeygen { - return - } - t.Run("SSHKeygen", func(t *testing.T) { - keyTypeK, lengthK, err := SSHKeyGenParsePublicKey(tc.content) - if err != nil { - // Some servers do not support ecdsa format. - if !strings.Contains(err.Error(), "line 1 too long:") { - require.NoError(t, err) - } - } - assert.Equal(t, tc.keyType, keyTypeK) - assert.Equal(t, tc.length, lengthK) - }) - t.Run("SSHParseKeyNative", func(t *testing.T) { - keyTypeK, lengthK, err := SSHNativeParsePublicKey(tc.content) - require.NoError(t, err) - - assert.Equal(t, tc.keyType, keyTypeK) - assert.Equal(t, tc.length, lengthK) - }) }) } } @@ -186,14 +164,6 @@ func Test_calcFingerprint(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tc.fp, fpN) }) - if tc.skipSSHKeygen { - return - } - t.Run("SSHKeygen", func(t *testing.T) { - fpK, err := calcFingerprintSSHKeygen(tc.content) - assert.NoError(t, err) - assert.Equal(t, tc.fp, fpK) - }) }) } } diff --git a/modules/setting/ssh.go b/modules/setting/ssh.go index b30987f9cc4c1..da8cdf58d2579 100644 --- a/modules/setting/ssh.go +++ b/modules/setting/ssh.go @@ -4,7 +4,6 @@ package setting import ( - "os" "path/filepath" "strings" "text/template" @@ -31,7 +30,6 @@ var SSH = struct { ServerKeyExchanges []string `ini:"SSH_SERVER_KEY_EXCHANGES"` ServerMACs []string `ini:"SSH_SERVER_MACS"` ServerHostKeys []string `ini:"SSH_SERVER_HOST_KEYS"` - KeygenPath string `ini:"SSH_KEYGEN_PATH"` AuthorizedKeysBackup bool `ini:"SSH_AUTHORIZED_KEYS_BACKUP"` AuthorizedPrincipalsBackup bool `ini:"SSH_AUTHORIZED_PRINCIPALS_BACKUP"` AuthorizedKeysCommandTemplate string `ini:"SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE"` @@ -56,7 +54,6 @@ var SSH = struct { ServerCiphers: []string{"chacha20-poly1305@openssh.com", "aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "aes256-gcm@openssh.com"}, ServerKeyExchanges: []string{"curve25519-sha256", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521", "diffie-hellman-group14-sha256", "diffie-hellman-group14-sha1"}, ServerMACs: []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256", "hmac-sha1"}, - KeygenPath: "", MinimumKeySizeCheck: true, MinimumKeySizes: map[string]int{"ed25519": 256, "ed25519-sk": 256, "ecdsa": 256, "ecdsa-sk": 256, "rsa": 3071}, ServerHostKeys: []string{"ssh/gitea.rsa", "ssh/gogs.rsa"}, @@ -97,10 +94,6 @@ func parseAuthorizedPrincipalsAllow(values []string) ([]string, bool) { return authorizedPrincipalsAllow, true } -func GetSSHKeyTestPath() string { - return filepath.Join(TempPath, "ssh_key_test") -} - func loadSSHFrom(rootCfg ConfigProvider) { sec := rootCfg.Section("server") if len(SSH.Domain) == 0 { @@ -135,11 +128,6 @@ func loadSSHFrom(rootCfg ConfigProvider) { } } - if err := os.MkdirAll(GetSSHKeyTestPath(), os.ModePerm); err != nil { - log.Fatal("failed to create directory %q for ssh key test: %w", GetSSHKeyTestPath(), err) - } - - SSH.KeygenPath = sec.Key("SSH_KEYGEN_PATH").String() SSH.Port = sec.Key("SSH_PORT").MustInt(22) SSH.ListenPort = sec.Key("SSH_LISTEN_PORT").MustInt(SSH.Port) SSH.UseProxyProtocol = sec.Key("SSH_SERVER_USE_PROXY_PROTOCOL").MustBool(false) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 96c99615f5d52..d7da975a217c3 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3287,8 +3287,6 @@ config.ssh_domain = SSH Server Domain config.ssh_port = Port config.ssh_listen_port = Listen Port config.ssh_root_path = Root Path -config.ssh_key_test_path = Key Test Path -config.ssh_keygen_path = Keygen ('ssh-keygen') Path config.ssh_minimum_key_size_check = Minimum Key Size Check config.ssh_minimum_key_sizes = Minimum Key Sizes diff --git a/templates/admin/config.tmpl b/templates/admin/config.tmpl index 88dadeb3ee5db..806347c72049d 100644 --- a/templates/admin/config.tmpl +++ b/templates/admin/config.tmpl @@ -69,10 +69,6 @@ {{if not .SSH.StartBuiltinServer}}
{{ctx.Locale.Tr "admin.config.ssh_root_path"}}
{{.SSH.RootPath}}
-
{{ctx.Locale.Tr "admin.config.ssh_key_test_path"}}
-
{{.SSH.KeyTestPath}}
-
{{ctx.Locale.Tr "admin.config.ssh_keygen_path"}}
-
{{.SSH.KeygenPath}}
{{ctx.Locale.Tr "admin.config.ssh_minimum_key_size_check"}}
{{svg (Iif .SSH.MinimumKeySizeCheck "octicon-check" "octicon-x")}}
{{if .SSH.MinimumKeySizeCheck}} From e40bfb888b213049cabeeb0ca489cbb5ab2e26c2 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 7 Apr 2025 12:54:50 +0800 Subject: [PATCH 27/30] refactor --- cmd/migrate_storage_test.go | 3 +- cmd/web.go | 2 + custom/conf/app.example.ini | 4 - models/issues/issue_update.go | 1 + models/migrations/base/tests.go | 6 +- models/repo/upload.go | 10 +-- models/unittest/testdb.go | 12 +-- modules/git/blame.go | 69 +++++++------- modules/git/blame_sha256_test.go | 3 + modules/git/blame_test.go | 3 + modules/git/git_test.go | 4 +- modules/git/repo.go | 4 +- modules/git/repo_index.go | 4 +- modules/git/repo_language_stats_test.go | 3 + modules/markup/external/external.go | 12 +-- modules/packages/hashed_buffer.go | 5 +- modules/packages/hashed_buffer_test.go | 3 + .../packages/nuget/symbol_extractor_test.go | 3 + modules/repository/temp.go | 20 +---- modules/setting/global.go | 12 --- modules/setting/packages.go | 16 ++-- modules/setting/path.go | 8 ++ modules/setting/repository.go | 4 - modules/setting/server.go | 7 ++ modules/setting/setting.go | 21 ----- modules/storage/local.go | 2 +- modules/temp/temp.go | 28 ------ modules/tempdir/tempdir.go | 89 +++++++++++++++++++ modules/tempdir/tempdir_test.go | 19 ++++ modules/templates/util_render_test.go | 2 +- modules/util/filebuffer/file_backed_buffer.go | 37 ++------ .../filebuffer/file_backed_buffer_test.go | 3 +- modules/util/path.go | 8 -- modules/util/path_test.go | 17 ---- routers/web/repo/githttp.go | 3 +- services/packages/container/blob_uploader.go | 2 +- services/pull/patch.go | 8 +- services/repository/create.go | 3 +- services/repository/generate.go | 4 +- .../migration-test/migration_test.go | 2 +- tests/test_utils.go | 2 +- 41 files changed, 230 insertions(+), 238 deletions(-) delete mode 100644 modules/temp/temp.go create mode 100644 modules/tempdir/tempdir.go create mode 100644 modules/tempdir/tempdir_test.go diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index e00bf5acf218a..6817867e28807 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -68,6 +68,7 @@ func TestMigratePackages(t *testing.T) { entries, err := os.ReadDir(p) assert.NoError(t, err) - assert.Len(t, entries, 1) // tmp directory should not be under storage any more + assert.Len(t, entries, 2) assert.Equal(t, "01", entries[0].Name()) + assert.Equal(t, "tmp", entries[1].Name()) } diff --git a/cmd/web.go b/cmd/web.go index dc5c6de48a309..76f3e96ea343c 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -213,6 +213,8 @@ func serveInstalled(ctx *cli.Context) error { log.Fatal("Can not find APP_DATA_PATH %q", setting.AppDataPath) } + setting.AppDataTempDir("").RemoveOutdated() + // Override the provided port number within the configuration if ctx.IsSet("port") { if err := setPort(ctx.String("port")); err != nil { diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index a76ed586e546e..328b50f9ce72f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -53,10 +53,6 @@ RUN_USER = ; git ;; The working directory, see the comment of AppWorkPath above ;WORK_PATH = -;; The temporary directory, defaults to a directory named gitea under the system temporary directory -;; All other temporary directories are relative to this directory by default -;TEMP_PATH = - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; [server] diff --git a/models/issues/issue_update.go b/models/issues/issue_update.go index 746a59c6fdc5d..1841da61e0adf 100644 --- a/models/issues/issue_update.go +++ b/models/issues/issue_update.go @@ -846,6 +846,7 @@ func DeleteOrphanedIssues(ctx context.Context) error { // Remove issue attachment files. for i := range attachmentPaths { + // FIXME: it's not right, because the attachment might not be on local filesystem system_model.RemoveAllWithNotice(ctx, "Delete issue attachment", attachmentPaths[i]) } return nil diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 873ff1142dc39..7da426fef0890 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -15,7 +15,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/temp" + "code.gitea.io/gitea/modules/tempdir" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/testlogger" @@ -115,7 +115,7 @@ func MainTest(m *testing.M) { setting.CustomConf = giteaConf } - tmpDataPath, cleanup, err := temp.MkdirTemp("data") + tmpDataPath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("data") if err != nil { testlogger.Fatalf("Unable to create temporary data path %v\n", err) } @@ -124,7 +124,7 @@ func MainTest(m *testing.M) { setting.CustomPath = filepath.Join(setting.AppWorkPath, "custom") setting.AppDataPath = tmpDataPath - unittest.InitSettings() + unittest.InitSettingsForTesting() if err = git.InitFull(context.Background()); err != nil { testlogger.Fatalf("Unable to InitFull: %v\n", err) } diff --git a/models/repo/upload.go b/models/repo/upload.go index 858ffd18df58c..fb57fb6c513a8 100644 --- a/models/repo/upload.go +++ b/models/repo/upload.go @@ -51,14 +51,10 @@ func init() { db.RegisterModel(new(Upload)) } -// UploadLocalPath returns where uploads is stored in local file system based on given UUID. -func UploadLocalPath(uuid string) string { - return filepath.Join(setting.GetRepositoryUploadTempPath(), uuid[0:1], uuid[1:2], uuid) -} - -// LocalPath returns where uploads are temporarily stored in local file system. +// LocalPath returns where uploads are temporarily stored in local file system based on given UUID. func (upload *Upload) LocalPath() string { - return UploadLocalPath(upload.UUID) + uuid := upload.UUID + return setting.AppDataTempDir("repo-uploads").JoinPath(uuid[0:1], uuid[1:2], uuid) } // NewUpload creates a new upload object. diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index 975eb1a47ed19..cb60cf5f85482 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -20,7 +20,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting/config" "code.gitea.io/gitea/modules/storage" - "code.gitea.io/gitea/modules/temp" + "code.gitea.io/gitea/modules/tempdir" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" @@ -36,8 +36,8 @@ func fatalTestError(fmtStr string, args ...any) { os.Exit(1) } -// InitSettings initializes config provider and load common settings for tests -func InitSettings() { +// InitSettingsForTesting initializes config provider and load common settings for tests +func InitSettingsForTesting() { setting.IsInTesting = true log.OsExiter = func(code int) { if code != 0 { @@ -76,7 +76,7 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { testOpts := util.OptionalArg(testOptsArg, &TestOptions{}) giteaRoot = test.SetupGiteaRoot() setting.CustomPath = filepath.Join(giteaRoot, "custom") - InitSettings() + InitSettingsForTesting() fixturesOpts := FixturesOptions{Dir: filepath.Join(giteaRoot, "models", "fixtures"), Files: testOpts.FixtureFiles} if err := CreateTestEngine(fixturesOpts); err != nil { @@ -93,14 +93,14 @@ func MainTest(m *testing.M, testOptsArg ...*TestOptions) { setting.SSH.Domain = "try.gitea.io" setting.Database.Type = "sqlite3" setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" - repoRootPath, cleanup1, err := temp.MkdirTemp("repos") + repoRootPath, cleanup1, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("repos") if err != nil { fatalTestError("TempDir: %v\n", err) } defer cleanup1() setting.RepoRootPath = repoRootPath - appDataPath, cleanup2, err := temp.MkdirTemp("appdata") + appDataPath, cleanup2, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("appdata") if err != nil { fatalTestError("TempDir: %v\n", err) } diff --git a/modules/git/blame.go b/modules/git/blame.go index a36f8a01c2768..6eb583a6b9c44 100644 --- a/modules/git/blame.go +++ b/modules/git/blame.go @@ -11,8 +11,7 @@ import ( "os" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/temp" - "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/setting" ) // BlamePart represents block of blame - continuous lines with one sha @@ -30,12 +29,13 @@ type BlameReader struct { bufferedReader *bufio.Reader done chan error lastSha *string - ignoreRevsFile *string + ignoreRevsFile string objectFormat ObjectFormat + cleanupFuncs []func() } func (r *BlameReader) UsesIgnoreRevs() bool { - return r.ignoreRevsFile != nil + return r.ignoreRevsFile != "" } // NextPart returns next part of blame (sequential code lines with the same commit) @@ -123,36 +123,37 @@ func (r *BlameReader) Close() error { r.bufferedReader = nil _ = r.reader.Close() _ = r.output.Close() - if r.ignoreRevsFile != nil { - _ = util.Remove(*r.ignoreRevsFile) + for _, cleanup := range r.cleanupFuncs { + if cleanup != nil { + cleanup() + } } return err } // CreateBlameReader creates reader for given repository, commit and file func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (*BlameReader, error) { - var ignoreRevsFile *string - if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { - ignoreRevsFile = tryCreateBlameIgnoreRevsFile(commit) + reader, stdout, err := os.Pipe() + if err != nil { + return nil, err } cmd := NewCommandNoGlobals("blame", "--porcelain") - if ignoreRevsFile != nil { - // Possible improvement: use --ignore-revs-file /dev/stdin on unix - // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. - cmd.AddOptionValues("--ignore-revs-file", *ignoreRevsFile) - } - cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) - reader, stdout, err := os.Pipe() - if err != nil { - if ignoreRevsFile != nil { - _ = util.Remove(*ignoreRevsFile) + + var ignoreRevsFileName string + var ignoreRevsFileCleanup func() // TODO: maybe it should check the returned err in a defer func to make sure the cleanup could always be executed correctly + if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore { + ignoreRevsFileName, ignoreRevsFileCleanup = tryCreateBlameIgnoreRevsFile(commit) + if ignoreRevsFileName != "" { + // Possible improvement: use --ignore-revs-file /dev/stdin on unix + // There is no equivalent on Windows. May be implemented if Gitea uses an external git backend. + cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName) } - return nil, err } - done := make(chan error, 1) + cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file) + done := make(chan error, 1) go func() { stderr := bytes.Buffer{} // TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close" @@ -170,40 +171,44 @@ func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath }() bufferedReader := bufio.NewReader(reader) - return &BlameReader{ output: stdout, reader: reader, bufferedReader: bufferedReader, done: done, - ignoreRevsFile: ignoreRevsFile, + ignoreRevsFile: ignoreRevsFileName, objectFormat: objectFormat, + cleanupFuncs: []func(){ignoreRevsFileCleanup}, }, nil } -func tryCreateBlameIgnoreRevsFile(commit *Commit) *string { +func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func()) { entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs") if err != nil { - return nil + log.Error("Unable to get .git-blame-ignore-revs file: GetTreeEntryByPath: %v", err) + return "", nil } r, err := entry.Blob().DataAsync() if err != nil { - return nil + log.Error("Unable to get .git-blame-ignore-revs file data: DataAsync: %v", err) + return "", nil } defer r.Close() - f, err := temp.CreateTemp("gitea_git-blame-ignore-revs") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs") if err != nil { - return nil + log.Error("Unable to get .git-blame-ignore-revs file data: CreateTempFileRandom: %v", err) + return "", nil } - + filename := f.Name() _, err = io.Copy(f, r) _ = f.Close() if err != nil { - _ = util.Remove(f.Name()) - return nil + cleanup() + log.Error("Unable to get .git-blame-ignore-revs file data: Copy: %v", err) + return "", nil } - return util.ToPointer(f.Name()) + return filename, cleanup } diff --git a/modules/git/blame_sha256_test.go b/modules/git/blame_sha256_test.go index 99c23429e2d2c..c0a97bed3bd35 100644 --- a/modules/git/blame_sha256_test.go +++ b/modules/git/blame_sha256_test.go @@ -7,10 +7,13 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutputSha256(t *testing.T) { + setting.AppDataPath = t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() diff --git a/modules/git/blame_test.go b/modules/git/blame_test.go index 36b5fb934951a..809d6fbcf7381 100644 --- a/modules/git/blame_test.go +++ b/modules/git/blame_test.go @@ -7,10 +7,13 @@ import ( "context" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestReadingBlameOutput(t *testing.T) { + setting.AppDataPath = t.TempDir() ctx, cancel := context.WithCancel(t.Context()) defer cancel() diff --git a/modules/git/git_test.go b/modules/git/git_test.go index cf6fa87c0c4dd..58ba01cabcb0e 100644 --- a/modules/git/git_test.go +++ b/modules/git/git_test.go @@ -10,14 +10,14 @@ import ( "testing" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/temp" + "code.gitea.io/gitea/modules/tempdir" "github.com/hashicorp/go-version" "github.com/stretchr/testify/assert" ) func testRun(m *testing.M) error { - gitHomePath, cleanup, err := temp.MkdirTemp("git-home") + gitHomePath, cleanup, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("git-home") if err != nil { return fmt.Errorf("unable to create temp dir: %w", err) } diff --git a/modules/git/repo.go b/modules/git/repo.go index be01c4104947a..45937a8d5fa54 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -18,7 +18,7 @@ import ( "time" "code.gitea.io/gitea/modules/proxy" - "code.gitea.io/gitea/modules/temp" + "code.gitea.io/gitea/modules/setting" ) // GPGSettings represents the default GPG settings for this repository @@ -267,7 +267,7 @@ func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch // CreateBundle create bundle content to the target path func (repo *Repository) CreateBundle(ctx context.Context, commit string, out io.Writer) error { - tmp, cleanup, err := temp.MkdirTemp("gitea-bundle") + tmp, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-bundle") if err != nil { return err } diff --git a/modules/git/repo_index.go b/modules/git/repo_index.go index 5bfd6b2d9dcc4..443a3a20d175c 100644 --- a/modules/git/repo_index.go +++ b/modules/git/repo_index.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - "code.gitea.io/gitea/modules/temp" + "code.gitea.io/gitea/modules/setting" ) // ReadTreeToIndex reads a treeish to the index @@ -58,7 +58,7 @@ func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (tmpIndexFilena } }() - tmpDir, cancel, err = temp.MkdirTemp("index") + tmpDir, cancel, err = setting.AppDataTempDir("git-repo-content").MkdirTempRandom("index") if err != nil { return "", "", nil, err } diff --git a/modules/git/repo_language_stats_test.go b/modules/git/repo_language_stats_test.go index 81f130bacb6cd..12ce958c6e556 100644 --- a/modules/git/repo_language_stats_test.go +++ b/modules/git/repo_language_stats_test.go @@ -9,11 +9,14 @@ import ( "path/filepath" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRepository_GetLanguageStats(t *testing.T) { + setting.AppDataPath = t.TempDir() repoPath := filepath.Join(testReposDir, "language_stats_repo") gitRepo, err := openRepositoryWithDefaultContext(repoPath) require.NoError(t, err) diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 7f09f64d06e71..39861ade121ee 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,12 +12,9 @@ import ( "runtime" "strings" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/temp" - "code.gitea.io/gitea/modules/util" ) // RegisterRenderers registers all supported third part renderers according settings @@ -89,16 +86,11 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. if p.IsInputFile { // write to temp file - f, err := temp.CreateTemp("gitea_input") + f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("gitea_input") if err != nil { return fmt.Errorf("%s create temp file when rendering %s failed: %w", p.Name(), p.Command, err) } - tmpPath := f.Name() - defer func() { - if err := util.Remove(tmpPath); err != nil { - log.Warn("Unable to remove temporary file: %s: Error: %v", tmpPath, err) - } - }() + defer cleanup() _, err = io.Copy(f, input) if err != nil { diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index 4ab45edcecaf8..c89d732ba4aad 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -6,6 +6,7 @@ package packages import ( "io" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util/filebuffer" ) @@ -34,11 +35,11 @@ func NewHashedBuffer() (*HashedBuffer, error) { // NewHashedBufferWithSize creates a hashed buffer with a specific memory size func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { - b, err := filebuffer.New(maxMemorySize) + tempDir, err := setting.AppDataTempDir("package-hashed-buffer").Mkdir("") if err != nil { return nil, err } - + b := filebuffer.New(maxMemorySize, tempDir) hash := NewMultiHasher() combinedWriter := io.MultiWriter(b, hash) diff --git a/modules/packages/hashed_buffer_test.go b/modules/packages/hashed_buffer_test.go index 564e782f18f0a..5104c1fb25f78 100644 --- a/modules/packages/hashed_buffer_test.go +++ b/modules/packages/hashed_buffer_test.go @@ -9,10 +9,13 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) func TestHashedBuffer(t *testing.T) { + setting.AppDataPath = t.TempDir() cases := []struct { MaxMemorySize int Data string diff --git a/modules/packages/nuget/symbol_extractor_test.go b/modules/packages/nuget/symbol_extractor_test.go index fa1b80ee82e36..711ad6d096a18 100644 --- a/modules/packages/nuget/symbol_extractor_test.go +++ b/modules/packages/nuget/symbol_extractor_test.go @@ -9,6 +9,8 @@ import ( "encoding/base64" "testing" + "code.gitea.io/gitea/modules/setting" + "github.com/stretchr/testify/assert" ) @@ -17,6 +19,7 @@ fgAA3AEAAAQAAAAjU3RyaW5ncwAAAADgAQAABAAAACNVUwDkAQAAMAAAACNHVUlEAAAAFAIAACgB AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==` func TestExtractPortablePdb(t *testing.T) { + setting.AppDataPath = t.TempDir() createArchive := func(name string, content []byte) []byte { var buf bytes.Buffer archive := zip.NewWriter(&buf) diff --git a/modules/repository/temp.go b/modules/repository/temp.go index 1befe4a5c5ebe..d7253d9e021b4 100644 --- a/modules/repository/temp.go +++ b/modules/repository/temp.go @@ -6,33 +6,17 @@ package repository import ( "context" "fmt" - "os" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -// localCopyPath returns the local repository temporary copy path. -func localCopyPath() string { - return filepath.Join(setting.TempPath, "local-repo") -} - // CreateTemporaryPath creates a temporary path func CreateTemporaryPath(prefix string) (string, context.CancelFunc, error) { - if err := os.MkdirAll(localCopyPath(), os.ModePerm); err != nil { - log.Error("Unable to create localcopypath directory: %s (%v)", localCopyPath(), err) - return "", nil, fmt.Errorf("failed to create localcopypath directory %s: %w", localCopyPath(), err) - } - basePath, err := os.MkdirTemp(localCopyPath(), prefix+".git") + basePath, cleanup, err := setting.AppDataTempDir("local-repo").MkdirTempRandom(prefix + ".git") if err != nil { log.Error("Unable to create temporary directory: %s-*.git (%v)", prefix, err) return "", nil, fmt.Errorf("failed to create dir %s-*.git: %w", prefix, err) } - return basePath, func() { - if err := util.RemoveAll(basePath); err != nil { - log.Error("Unable to remove temporary directory: %s (%v)", basePath, err) - } - }, nil + return basePath, cleanup, nil } diff --git a/modules/setting/global.go b/modules/setting/global.go index d92e42dbf9f59..55dfe485b21d1 100644 --- a/modules/setting/global.go +++ b/modules/setting/global.go @@ -3,11 +3,6 @@ package setting -import ( - "os" - "path/filepath" -) - // Global settings var ( // RunUser is the OS user that Gitea is running as. ini:"RUN_USER" @@ -20,11 +15,4 @@ var ( // AppName is the Application name, used in the page title. ini: "APP_NAME" AppName string - - // TempPath is the directory used for temporary files. ini: "TEMP_PATH" - TempPath string ) - -func init() { - TempPath = filepath.Join(os.TempDir(), "gitea") -} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 6c8c7a308111b..c56fec333d3bf 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -6,8 +6,6 @@ package setting import ( "fmt" "math" - "os" - "path/filepath" "github.com/dustin/go-humanize" ) @@ -15,8 +13,9 @@ import ( // Package registry settings var ( Packages = struct { - Storage *Storage - Enabled bool + Storage *Storage + Enabled bool + ChunkedUploadPath string LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 @@ -50,10 +49,6 @@ var ( } ) -func GetPacakgeUploadTempPath() string { - return filepath.Join(TempPath, "package-upload") -} - func loadPackagesFrom(rootCfg ConfigProvider) (err error) { sec, _ := rootCfg.GetSection("packages") if sec == nil { @@ -71,8 +66,9 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { } if HasInstallLock(rootCfg) { - if err := os.MkdirAll(GetPacakgeUploadTempPath(), os.ModePerm); err != nil { - return fmt.Errorf("unable to create chunked upload directory: %s (%v)", GetPacakgeUploadTempPath(), err) + Packages.ChunkedUploadPath, err = AppDataTempDir("package-upload").Mkdir("") + if err != nil { + return fmt.Errorf("unable to create chunked upload directory: %w", err) } } diff --git a/modules/setting/path.go b/modules/setting/path.go index 0fdc305aa160c..fe77531c86765 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -11,6 +11,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/tempdir" ) var ( @@ -196,3 +197,10 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP CustomPath = tmpCustomPath.Value CustomConf = tmpCustomConf.Value } + +func AppDataTempDir(sub string) *tempdir.TempDir { + if AppDataPath == "" { + panic("setting.AppDataPath is not set") + } + return tempdir.New(AppDataPath, "tmp/"+sub) +} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 8a0dd791314c0..f99c854e4f89f 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -348,7 +348,3 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { log.Fatal("loadRepoArchiveFrom: %v", err) } } - -func GetRepositoryUploadTempPath() string { - return filepath.Join(TempPath, "uploads") -} diff --git a/modules/setting/server.go b/modules/setting/server.go index e15b790906738..6d21d8c9005c6 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "net" "net/url" + "os" "path/filepath" "strconv" "strings" @@ -330,6 +331,12 @@ func loadServerFrom(rootCfg ConfigProvider) { if !filepath.IsAbs(AppDataPath) { AppDataPath = filepath.ToSlash(filepath.Join(AppWorkPath, AppDataPath)) } + if IsInTesting && HasInstallLock(rootCfg) { + // FIXME: in testing, the "app data" directory is not correctly initialized before loading settings + if _, err := os.Stat(AppDataPath); err != nil { + _ = os.MkdirAll(AppDataPath, os.ModePerm) + } + } EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 03f515fa5194b..e14997801fed4 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -7,7 +7,6 @@ package setting import ( "fmt" "os" - "path/filepath" "runtime" "strings" "time" @@ -111,7 +110,6 @@ func LoadCommonSettings() { func loadCommonSettingsFrom(cfg ConfigProvider) error { // WARNING: don't change the sequence except you know what you are doing. loadRunModeFrom(cfg) - loadTempDir(cfg) loadLogGlobalFrom(cfg) loadServerFrom(cfg) loadSSHFrom(cfg) @@ -184,25 +182,6 @@ func loadRunModeFrom(rootCfg ConfigProvider) { } } -func loadTempDir(rootCfg ConfigProvider) { - rootSec := rootCfg.Section("") - tempPath := rootSec.Key("TEMP_PATH").String() - if tempPath != "" { - if !filepath.IsAbs(tempPath) { - tempPath = filepath.Join(os.TempDir(), tempPath) - } - TempPath = tempPath - } - // TempPath has been initialized in init function of global.go - if TempPath == "" { - log.Fatal("It's impossible that TEMP_PATH is empty") - } - - if err := os.MkdirAll(TempPath, os.ModePerm); err != nil { - log.Fatal("Failed to create temp directory %s: %v", TempPath, err) - } -} - // HasInstallLock checks the install-lock in ConfigProvider directly, because sometimes the config file is not loaded into setting variables yet. func HasInstallLock(rootCfg ConfigProvider) bool { return rootCfg.Section("security").Key("INSTALL_LOCK").MustBool(false) diff --git a/modules/storage/local.go b/modules/storage/local.go index eb4f0ff7461d0..00c7f668aa2c3 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -36,7 +36,7 @@ func NewLocalStorage(ctx context.Context, config *setting.Storage) (ObjectStorag } if config.TemporaryPath == "" { - config.TemporaryPath = filepath.Join(setting.TempPath, "storage", util.SanitizeDirName(config.Path)) + config.TemporaryPath = filepath.Join(config.Path, "tmp") } if !filepath.IsAbs(config.TemporaryPath) { return nil, fmt.Errorf("LocalStorageConfig.TemporaryPath should be an absolute path, but not: %q", config.TemporaryPath) diff --git a/modules/temp/temp.go b/modules/temp/temp.go deleted file mode 100644 index d053dfe7e9e2d..0000000000000 --- a/modules/temp/temp.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2025 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package temp - -import ( - "os" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" -) - -func MkdirTemp(pattern string) (string, func(), error) { - dir, err := os.MkdirTemp(setting.TempPath, pattern) - if err != nil { - return "", nil, err - } - return dir, func() { - if err := util.RemoveAll(dir); err != nil { - log.Error("Failed to remove temp directory %s: %v", dir, err) - } - }, nil -} - -func CreateTemp(pattern string) (*os.File, error) { - return os.CreateTemp(setting.TempPath, pattern) -} diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go new file mode 100644 index 0000000000000..e6d0a53c91035 --- /dev/null +++ b/modules/tempdir/tempdir.go @@ -0,0 +1,89 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "os" + "path/filepath" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" +) + +type TempDir struct { + base, sub string +} + +func (td *TempDir) JoinPath(elems ...string) string { + return filepath.Join(append([]string{td.base, td.sub}, elems...)...) +} + +func (td *TempDir) Mkdir(dir string) (string, error) { + if _, err := os.Stat(td.base); err != nil { + return "", err + } + full := filepath.Join(td.base, td.sub, dir) + if err := os.MkdirAll(full, os.ModePerm); err != nil { + return "", err + } + return full, nil +} + +func (td *TempDir) prepareDirWithPattern(elems ...string) (dir, pattern string, err error) { + if _, err = os.Stat(td.base); err != nil { + return "", "", err + } + dir, pattern = filepath.Split(filepath.Join(append([]string{td.base, td.sub}, elems...)...)) + if err = os.MkdirAll(dir, os.ModePerm); err != nil { + return "", "", err + } + return dir, pattern, nil +} + +func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return "", nil, err + } + dir, err = os.MkdirTemp(dir, pattern) + if err != nil { + return "", nil, err + } + return dir, func() { + if err := util.RemoveAll(dir); err != nil { + log.Error("Failed to remove temp directory %s: %v", dir, err) + } + }, nil +} + +func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), error) { + dir, pattern, err := td.prepareDirWithPattern(elems...) + if err != nil { + return nil, nil, err + } + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, nil, err + } + filename := f.Name() + return f, func() { + _ = f.Close() + if err := util.Remove(filename); err != nil { + log.Error("Unable to remove temporary file: %s: Error: %v", filename, err) + } + }, err +} + +func (td *TempDir) RemoveOutdated() { + // TODO: remove the out-dated temp files + log.Error("TODO: remove the out-dated temp files, not implemented yet") +} + +func OsTempDir(sub string) *TempDir { + return &TempDir{base: os.TempDir(), sub: sub} +} + +func New(base, sub string) *TempDir { + return &TempDir{base: base, sub: sub} +} diff --git a/modules/tempdir/tempdir_test.go b/modules/tempdir/tempdir_test.go new file mode 100644 index 0000000000000..f7e6e81e621f8 --- /dev/null +++ b/modules/tempdir/tempdir_test.go @@ -0,0 +1,19 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package tempdir + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTempDir(t *testing.T) { + base := t.TempDir() + td := New(base, "sub") + assert.Equal(t, filepath.Join(base, "sub"), td.JoinPath("")) + + // TODO: add some tests +} diff --git a/modules/templates/util_render_test.go b/modules/templates/util_render_test.go index 26cd1eb3489ca..460b9dc190bf4 100644 --- a/modules/templates/util_render_test.go +++ b/modules/templates/util_render_test.go @@ -56,7 +56,7 @@ var testMetas = map[string]string{ } func TestMain(m *testing.M) { - unittest.InitSettings() + unittest.InitSettingsForTesting() if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } diff --git a/modules/util/filebuffer/file_backed_buffer.go b/modules/util/filebuffer/file_backed_buffer.go index 46af89165e693..0731ba30c8b27 100644 --- a/modules/util/filebuffer/file_backed_buffer.go +++ b/modules/util/filebuffer/file_backed_buffer.go @@ -7,18 +7,10 @@ import ( "bytes" "errors" "io" - "math" "os" - - "code.gitea.io/gitea/modules/setting" ) -var ( - // ErrInvalidMemorySize occurs if the memory size is not in a valid range - ErrInvalidMemorySize = errors.New("Memory size must be greater 0 and lower math.MaxInt32") - // ErrWriteAfterRead occurs if Write is called after a read operation - ErrWriteAfterRead = errors.New("Write is unsupported after a read operation") -) +var ErrWriteAfterRead = errors.New("write is unsupported after a read operation") // occurs if Write is called after a read operation type readAtSeeker interface { io.ReadSeeker @@ -32,34 +24,17 @@ type FileBackedBuffer struct { maxMemorySize int64 size int64 buffer bytes.Buffer + tempDir string file *os.File reader readAtSeeker } // New creates a file backed buffer with a specific maximum memory size -func New(maxMemorySize int) (*FileBackedBuffer, error) { - if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { - return nil, ErrInvalidMemorySize - } - +func New(maxMemorySize int, tempDir string) *FileBackedBuffer { return &FileBackedBuffer{ maxMemorySize: int64(maxMemorySize), - }, nil -} - -// CreateFromReader creates a file backed buffer and copies the provided reader data into it. -func CreateFromReader(r io.Reader, maxMemorySize int) (*FileBackedBuffer, error) { - b, err := New(maxMemorySize) - if err != nil { - return nil, err + tempDir: tempDir, } - - _, err = io.Copy(b, r) - if err != nil { - return nil, err - } - - return b, nil } // Write implements io.Writer @@ -75,7 +50,7 @@ func (b *FileBackedBuffer) Write(p []byte) (int, error) { n, err = b.file.Write(p) } else { if b.size+int64(len(p)) > b.maxMemorySize { - b.file, err = os.CreateTemp(setting.TempPath, "gitea-buffer-") + b.file, err = os.CreateTemp(b.tempDir, "gitea-buffer-") if err != nil { return 0, err } @@ -150,7 +125,7 @@ func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { func (b *FileBackedBuffer) Close() error { if b.file != nil { err := b.file.Close() - os.Remove(b.file.Name()) + _ = os.Remove(b.file.Name()) b.file = nil return err } diff --git a/modules/util/filebuffer/file_backed_buffer_test.go b/modules/util/filebuffer/file_backed_buffer_test.go index 16d5a1965f694..3f13c6ac7bb92 100644 --- a/modules/util/filebuffer/file_backed_buffer_test.go +++ b/modules/util/filebuffer/file_backed_buffer_test.go @@ -21,7 +21,8 @@ func TestFileBackedBuffer(t *testing.T) { } for _, c := range cases { - buf, err := CreateFromReader(strings.NewReader(c.Data), c.MaxMemorySize) + buf := New(c.MaxMemorySize, t.TempDir()) + _, err := io.Copy(buf, strings.NewReader(c.Data)) assert.NoError(t, err) assert.EqualValues(t, len(c.Data), buf.Size()) diff --git a/modules/util/path.go b/modules/util/path.go index 57f91fcfca73c..0e56348978092 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -291,11 +291,3 @@ func IsReadmeFileExtension(name string, ext ...string) (int, bool) { return 0, false } - -// SanitizeDirName replaces illegal characters in a directory name with "_" -func SanitizeDirName(path string) string { - p := filepath.Clean(path) - re := regexp.MustCompile(`[<>:"/\\|?*]`) - sanitized := re.ReplaceAllString(p, "_") - return sanitized -} diff --git a/modules/util/path_test.go b/modules/util/path_test.go index 45fe25657df74..79c37e55f7a57 100644 --- a/modules/util/path_test.go +++ b/modules/util/path_test.go @@ -230,20 +230,3 @@ func TestListDirRecursively(t *testing.T) { require.NoError(t, err) assert.ElementsMatch(t, []string{"d1/f-d1", "d1/s1/f-d1s1"}, res) } - -func TestSanitizeDirName(t *testing.T) { - cases := []struct { - path string - expected string - }{ - {"a", "a"}, - {"a/b", "a_b"}, - {"/a/b", "_a_b"}, - {"a/b/c", "a_b_c"}, - {"c:\\a\\b", "c__a_b"}, - {"c:\\a/b", "c__a_b"}, - } - for _, c := range cases { - assert.Equal(t, c.expected, SanitizeDirName(c.path)) - } -} diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index e21500f68b428..61606f8c5f1ee 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -29,7 +29,6 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/services/context" repo_service "code.gitea.io/gitea/services/repository" @@ -303,7 +302,7 @@ var ( func dummyInfoRefs(ctx *context.Context) { infoRefsOnce.Do(func() { - tmpDir, cleanup, err := temp.MkdirTemp("gitea-info-refs-cache") + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-info-refs-cache") if err != nil { log.Error("Failed to create temp dir for git-receive-pack cache: %v", err) return diff --git a/services/packages/container/blob_uploader.go b/services/packages/container/blob_uploader.go index 81aed7e50ae09..bae2e2d6af667 100644 --- a/services/packages/container/blob_uploader.go +++ b/services/packages/container/blob_uploader.go @@ -31,7 +31,7 @@ type BlobUploader struct { } func buildFilePath(id string) string { - return util.FilePathJoinAbs(setting.GetPacakgeUploadTempPath(), id) + return util.FilePathJoinAbs(setting.Packages.ChunkedUploadPath, id) } // NewBlobUploader creates a new blob uploader for the given id diff --git a/services/pull/patch.go b/services/pull/patch.go index 1467fff71cff1..7a242377249d3 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -23,7 +23,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -356,15 +355,12 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * } // 3b. Create a plain patch from head to base - tmpPatchFile, err := temp.CreateTemp("patch") + tmpPatchFile, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("patch") if err != nil { log.Error("Unable to create temporary patch file! Error: %v", err) return false, fmt.Errorf("unable to create temporary patch file! Error: %w", err) } - defer func() { - tmpPatchFile.Close() - _ = util.Remove(tmpPatchFile.Name()) - }() + defer cleanup() if err := gitRepo.GetDiffBinary(pr.MergeBase+"...tracking", tmpPatchFile); err != nil { log.Error("Unable to get patch file from %s to %s in %s Error: %v", pr.MergeBase, pr.HeadBranch, pr.BaseRepo.FullName(), err) diff --git a/services/repository/create.go b/services/repository/create.go index f0997dfcda4bc..699b338e9ada1 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -27,7 +27,6 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/temp" "code.gitea.io/gitea/modules/templates/vars" ) @@ -146,7 +145,7 @@ func initRepository(ctx context.Context, u *user_model.User, repo *repo_model.Re // Initialize repository according to user's choice. if opts.AutoInit { - tmpDir, cleanup, err := temp.MkdirTemp("repos-" + repo.Name) + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("repos-" + repo.Name) if err != nil { return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } diff --git a/services/repository/generate.go b/services/repository/generate.go index 2bdf1936b32f9..e81856be6a3c3 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -22,7 +22,7 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" - "code.gitea.io/gitea/modules/temp" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "github.com/gobwas/glob" @@ -257,7 +257,7 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } func generateGitContent(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository) (err error) { - tmpDir, cleanup, err := temp.MkdirTemp("gitea-" + repo.Name) + tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-" + repo.Name) if err != nil { return fmt.Errorf("failed to create temp dir for repository %s: %w", repo.FullName(), err) } diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index ffb8afa9c5875..0fc8a6e24de7d 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -52,7 +52,7 @@ func initMigrationTest(t *testing.T) func() { setting.CustomConf = giteaConf } - unittest.InitSettings() + unittest.InitSettingsForTesting() assert.NotEmpty(t, setting.RepoRootPath) assert.NoError(t, unittest.SyncDirs(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) diff --git a/tests/test_utils.go b/tests/test_utils.go index 2d3b66c62c379..4d8a8635d6480 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -66,7 +66,7 @@ func InitTest(requireGitea bool) { setting.CustomConf = giteaConf } - unittest.InitSettings() + unittest.InitSettingsForTesting() setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master" if err := git.InitFull(context.Background()); err != nil { From 47ea42666db4273787748b89e21114ff8455446e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 7 Apr 2025 15:04:57 +0800 Subject: [PATCH 28/30] add APP_TEMP_PATH --- custom/conf/app.example.ini | 3 +++ modules/setting/path.go | 3 +++ modules/setting/server.go | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 328b50f9ce72f..d1124371e0299 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -287,6 +287,9 @@ RUN_USER = ; git ;; Default path for App data ;APP_DATA_PATH = data ; relative paths will be made absolute with _`AppWorkPath`_ ;; +;; Base path for App's temp files, leave empty to use the managed tmp directory in APP_DATA_PATH +;APP_TEMP_PATH = +;; ;; Enable gzip compression for runtime-generated content, static resources excluded ;ENABLE_GZIP = false ;; diff --git a/modules/setting/path.go b/modules/setting/path.go index fe77531c86765..6752038800ad8 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -199,6 +199,9 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP } func AppDataTempDir(sub string) *tempdir.TempDir { + if appTempPathInternal != "" { + return tempdir.New(appTempPathInternal, "gitea-tmp/"+sub) + } if AppDataPath == "" { panic("setting.AppDataPath is not set") } diff --git a/modules/setting/server.go b/modules/setting/server.go index 6d21d8c9005c6..ca635c8abeff4 100644 --- a/modules/setting/server.go +++ b/modules/setting/server.go @@ -60,6 +60,8 @@ var ( // AssetVersion holds a opaque value that is used for cache-busting assets AssetVersion string + appTempPathInternal string // the temporary path for the app, it is only an internal variable, do not use it, always use AppDataTempDir + Protocol Scheme UseProxyProtocol bool // `ini:"USE_PROXY_PROTOCOL"` ProxyProtocolTLSBridging bool //`ini:"PROXY_PROTOCOL_TLS_BRIDGING"` @@ -338,6 +340,13 @@ func loadServerFrom(rootCfg ConfigProvider) { } } + appTempPathInternal = sec.Key("APP_TEMP_PATH").String() + if appTempPathInternal != "" { + if _, err := os.Stat(appTempPathInternal); err != nil { + log.Fatal("APP_TEMP_PATH %q is not accessible: %v", appTempPathInternal, err) + } + } + EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(filepath.Join(AppWorkPath, "data/tmp/pprof")) From 8483d03e130bd5d43c5e4c00e2fd0d3fce709305 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 7 Apr 2025 16:18:41 +0800 Subject: [PATCH 29/30] rename --- modules/packages/hashed_buffer.go | 2 +- modules/setting/packages.go | 2 +- modules/tempdir/tempdir.go | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/packages/hashed_buffer.go b/modules/packages/hashed_buffer.go index c89d732ba4aad..0cd657cd44b15 100644 --- a/modules/packages/hashed_buffer.go +++ b/modules/packages/hashed_buffer.go @@ -35,7 +35,7 @@ func NewHashedBuffer() (*HashedBuffer, error) { // NewHashedBufferWithSize creates a hashed buffer with a specific memory size func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { - tempDir, err := setting.AppDataTempDir("package-hashed-buffer").Mkdir("") + tempDir, err := setting.AppDataTempDir("package-hashed-buffer").MkdirAllSub("") if err != nil { return nil, err } diff --git a/modules/setting/packages.go b/modules/setting/packages.go index c56fec333d3bf..845f6d4e12bea 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -66,7 +66,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) { } if HasInstallLock(rootCfg) { - Packages.ChunkedUploadPath, err = AppDataTempDir("package-upload").Mkdir("") + Packages.ChunkedUploadPath, err = AppDataTempDir("package-upload").MkdirAllSub("") if err != nil { return fmt.Errorf("unable to create chunked upload directory: %w", err) } diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go index e6d0a53c91035..91e7fe4f0e293 100644 --- a/modules/tempdir/tempdir.go +++ b/modules/tempdir/tempdir.go @@ -12,6 +12,8 @@ import ( ) type TempDir struct { + // base is the base directory for temporary files, it must exist before accessing and won't be created automatically. + // for example: base="/system-tmpdir", sub="gitea-tmp" base, sub string } @@ -19,7 +21,8 @@ func (td *TempDir) JoinPath(elems ...string) string { return filepath.Join(append([]string{td.base, td.sub}, elems...)...) } -func (td *TempDir) Mkdir(dir string) (string, error) { +// MkdirAllSub works like os.MkdirAll, but the base directory must exist +func (td *TempDir) MkdirAllSub(dir string) (string, error) { if _, err := os.Stat(td.base); err != nil { return "", err } @@ -41,6 +44,7 @@ func (td *TempDir) prepareDirWithPattern(elems ...string) (dir, pattern string, return dir, pattern, nil } +// MkdirTempRandom works like os.MkdirTemp, the last path field is the "pattern" func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { dir, pattern, err := td.prepareDirWithPattern(elems...) if err != nil { @@ -57,6 +61,7 @@ func (td *TempDir) MkdirTempRandom(elems ...string) (string, func(), error) { }, nil } +// CreateTempFileRandom works like os.CreateTemp, the last path field is the "pattern" func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), error) { dir, pattern, err := td.prepareDirWithPattern(elems...) if err != nil { From a7371fa370cf825828b505f4ad1620e8400aa522 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 8 Apr 2025 21:44:34 +0800 Subject: [PATCH 30/30] add tests --- cmd/web.go | 4 ++- modules/setting/path.go | 5 +++ modules/tempdir/tempdir.go | 32 +++++++++++++---- modules/tempdir/tempdir_test.go | 62 +++++++++++++++++++++++++++++++-- 4 files changed, 92 insertions(+), 11 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 76f3e96ea343c..e47b171455c44 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -213,7 +213,9 @@ func serveInstalled(ctx *cli.Context) error { log.Fatal("Can not find APP_DATA_PATH %q", setting.AppDataPath) } - setting.AppDataTempDir("").RemoveOutdated() + // the AppDataTempDir is fully managed by us with a safe sub-path + // so it's safe to automatically remove the outdated files + setting.AppDataTempDir("").RemoveOutdated(3 * 24 * time.Hour) // Override the provided port number within the configuration if ctx.IsSet("port") { diff --git a/modules/setting/path.go b/modules/setting/path.go index 6752038800ad8..f51457a620eba 100644 --- a/modules/setting/path.go +++ b/modules/setting/path.go @@ -198,6 +198,11 @@ func InitWorkPathAndCfgProvider(getEnvFn func(name string) string, args ArgWorkP CustomConf = tmpCustomConf.Value } +// AppDataTempDir returns a managed temporary directory for the application data. +// Using empty sub will get the managed base temp directory, and it's safe to delete it. +// Gitea only creates subdirectories under it, but not the APP_TEMP_PATH directory itself. +// * When APP_TEMP_PATH="/tmp": the managed temp directory is "/tmp/gitea-tmp" +// * When APP_TEMP_PATH is not set: the managed temp directory is "/{APP_DATA_PATH}/tmp" func AppDataTempDir(sub string) *tempdir.TempDir { if appTempPathInternal != "" { return tempdir.New(appTempPathInternal, "gitea-tmp/"+sub) diff --git a/modules/tempdir/tempdir.go b/modules/tempdir/tempdir.go index 91e7fe4f0e293..22c2e4ea165f2 100644 --- a/modules/tempdir/tempdir.go +++ b/modules/tempdir/tempdir.go @@ -6,6 +6,7 @@ package tempdir import ( "os" "path/filepath" + "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" @@ -80,15 +81,32 @@ func (td *TempDir) CreateTempFileRandom(elems ...string) (*os.File, func(), erro }, err } -func (td *TempDir) RemoveOutdated() { - // TODO: remove the out-dated temp files - log.Error("TODO: remove the out-dated temp files, not implemented yet") -} - -func OsTempDir(sub string) *TempDir { - return &TempDir{base: os.TempDir(), sub: sub} +func (td *TempDir) RemoveOutdated(d time.Duration) { + var remove func(path string) + remove = func(path string) { + entries, _ := os.ReadDir(path) + for _, entry := range entries { + full := filepath.Join(path, entry.Name()) + if entry.IsDir() { + remove(full) + _ = os.Remove(full) + continue + } + info, err := entry.Info() + if err == nil && time.Since(info.ModTime()) > d { + _ = os.Remove(full) + } + } + } + remove(td.JoinPath("")) } +// New create a new TempDir instance, "base" must be an existing directory, +// "sub" could be a multi-level directory and will be created if not exist func New(base, sub string) *TempDir { return &TempDir{base: base, sub: sub} } + +func OsTempDir(sub string) *TempDir { + return New(os.TempDir(), sub) +} diff --git a/modules/tempdir/tempdir_test.go b/modules/tempdir/tempdir_test.go index f7e6e81e621f8..d6afcb7beda19 100644 --- a/modules/tempdir/tempdir_test.go +++ b/modules/tempdir/tempdir_test.go @@ -4,16 +4,72 @@ package tempdir import ( + "os" "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" ) func TestTempDir(t *testing.T) { base := t.TempDir() - td := New(base, "sub") - assert.Equal(t, filepath.Join(base, "sub"), td.JoinPath("")) - // TODO: add some tests + t.Run("Create", func(t *testing.T) { + td := New(base, "sub1/sub2") // make sure the sub dir supports "/" in the path + assert.Equal(t, filepath.Join(base, "sub1", "sub2"), td.JoinPath()) + assert.Equal(t, filepath.Join(base, "sub1", "sub2/test"), td.JoinPath("test")) + + t.Run("MkdirTempRandom", func(t *testing.T) { + s, cleanup, err := td.MkdirTempRandom("foo") + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(s, filepath.Join(base, "sub1/sub2", "foo"))) + + _, err = os.Stat(s) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(s) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("CreateTempFileRandom", func(t *testing.T) { + f, cleanup, err := td.CreateTempFileRandom("foo", "bar") + filename := f.Name() + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(filename, filepath.Join(base, "sub1/sub2", "foo", "bar"))) + _, err = os.Stat(filename) + assert.NoError(t, err) + cleanup() + _, err = os.Stat(filename) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + + t.Run("RemoveOutDated", func(t *testing.T) { + fa1, _, err := td.CreateTempFileRandom("dir-a", "f1") + assert.NoError(t, err) + fa2, _, err := td.CreateTempFileRandom("dir-a", "f2") + assert.NoError(t, err) + _ = os.Chtimes(fa2.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + fb1, _, err := td.CreateTempFileRandom("dir-b", "f1") + assert.NoError(t, err) + _ = os.Chtimes(fb1.Name(), time.Now().Add(-time.Hour), time.Now().Add(-time.Hour)) + _, _, _ = fa1.Close(), fa2.Close(), fb1.Close() + + td.RemoveOutdated(time.Minute) + + _, err = os.Stat(fa1.Name()) + assert.NoError(t, err) + _, err = os.Stat(fa2.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(fb1.Name()) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + }) + + t.Run("BaseNotExist", func(t *testing.T) { + td := New(filepath.Join(base, "not-exist"), "sub") + _, _, err := td.MkdirTempRandom("foo") + assert.ErrorIs(t, err, os.ErrNotExist) + }) }