From 6fba428a8e5da92409b8548299c8b6f0a7cb5a1c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 8 May 2021 20:23:13 +0800 Subject: [PATCH 1/4] Move restore repo to internal router and invoke from command to avoid open the same db file or queues files --- cmd/restore_repo.go | 77 +++++---------------------------- modules/migrations/dump.go | 44 +++++++++++++++++-- modules/private/restore_repo.go | 58 +++++++++++++++++++++++++ routers/private/internal.go | 1 + routers/private/restore_repo.go | 52 ++++++++++++++++++++++ 5 files changed, 163 insertions(+), 69 deletions(-) create mode 100644 modules/private/restore_repo.go create mode 100644 routers/private/restore_repo.go diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 541995879bf4c..b832471928f4a 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -5,15 +5,12 @@ package cmd import ( - "context" - "strings" + "errors" + "net/http" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/migrations" - "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/storage" - pull_service "code.gitea.io/gitea/services/pull" "github.com/urfave/cli" ) @@ -50,70 +47,18 @@ wiki, issues, labels, releases, release_assets, milestones, pull_requests, comme } func runRestoreRepository(ctx *cli.Context) error { - if err := initDB(); err != nil { - return err - } - - log.Trace("AppPath: %s", setting.AppPath) - log.Trace("AppWorkPath: %s", setting.AppWorkPath) - log.Trace("Custom path: %s", setting.CustomPath) - log.Trace("Log path: %s", setting.LogRootPath) - setting.InitDBConfig() - - if err := storage.Init(); err != nil { - return err - } - - if err := pull_service.Init(); err != nil { - return err - } - - var opts = base.MigrateOptions{ - RepoName: ctx.String("repo_name"), - } - - if len(ctx.String("units")) == 0 { - opts.Wiki = true - opts.Issues = true - opts.Milestones = true - opts.Labels = true - opts.Releases = true - opts.Comments = true - opts.PullRequests = true - opts.ReleaseAssets = true - } else { - units := strings.Split(ctx.String("units"), ",") - for _, unit := range units { - switch strings.ToLower(unit) { - case "wiki": - opts.Wiki = true - case "issues": - opts.Issues = true - case "milestones": - opts.Milestones = true - case "labels": - opts.Labels = true - case "releases": - opts.Releases = true - case "release_assets": - opts.ReleaseAssets = true - case "comments": - opts.Comments = true - case "pull_requests": - opts.PullRequests = true - } - } - } + setting.NewContext() - if err := migrations.RestoreRepository( - context.Background(), + statusCode, errStr := private.RestoreRepo( ctx.String("repo_dir"), ctx.String("owner_name"), ctx.String("repo_name"), - ); err != nil { - log.Fatal("Failed to restore repository: %v", err) - return err + ctx.StringSlice("units"), + ) + if statusCode == http.StatusOK { + return nil } - return nil + log.Fatal("Failed to restore repository: %v", errStr) + return errors.New(errStr) } diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index 297095883be4b..eb032d041bd3e 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" "code.gitea.io/gitea/models" @@ -563,8 +564,42 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi return nil } +func updateOptionsUnits(opts *base.MigrateOptions, units []string) { + if len(units) == 0 { + opts.Wiki = true + opts.Issues = true + opts.Milestones = true + opts.Labels = true + opts.Releases = true + opts.Comments = true + opts.PullRequests = true + opts.ReleaseAssets = true + } else { + for _, unit := range units { + switch strings.ToLower(unit) { + case "wiki": + opts.Wiki = true + case "issues": + opts.Issues = true + case "milestones": + opts.Milestones = true + case "labels": + opts.Labels = true + case "releases": + opts.Releases = true + case "release_assets": + opts.ReleaseAssets = true + case "comments": + opts.Comments = true + case "pull_requests": + opts.PullRequests = true + } + } + } +} + // RestoreRepository restore a repository from the disk directory -func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error { +func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string, units []string) error { doer, err := models.GetAdminUser() if err != nil { return err @@ -580,7 +615,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName } tp, _ := strconv.Atoi(opts["service_type"]) - if err = migrateRepository(downloader, uploader, base.MigrateOptions{ + var migrateOpts = base.MigrateOptions{ Wiki: true, Issues: true, Milestones: true, @@ -590,7 +625,10 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName PullRequests: true, ReleaseAssets: true, GitServiceType: structs.GitServiceType(tp), - }); err != nil { + } + updateOptionsUnits(&migrateOpts, units) + + if err = migrateRepository(downloader, uploader, migrateOpts); err != nil { if err1 := uploader.Rollback(); err1 != nil { log.Error("rollback failed: %v", err1) } diff --git a/modules/private/restore_repo.go b/modules/private/restore_repo.go new file mode 100644 index 0000000000000..c9b33b42a6fd3 --- /dev/null +++ b/modules/private/restore_repo.go @@ -0,0 +1,58 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "fmt" + "io/ioutil" + "net/http" + + "code.gitea.io/gitea/modules/setting" + jsoniter "github.com/json-iterator/go" +) + +// RestoreParams structure holds a data for restore repository +type RestoreParams struct { + RepoDir string + OwnerName string + RepoName string + Units []string +} + +// RestoreRepo calls the internal RestoreRepo function +func RestoreRepo(repoDir, ownerName, repoName string, units []string) (int, string) { + reqURL := setting.LocalURL + "api/internal/restore_repo" + + req := newInternalRequest(reqURL, "POST") + req = req.Header("Content-Type", "application/json") + json := jsoniter.ConfigCompatibleWithStandardLibrary + jsonBytes, _ := json.Marshal(RestoreParams{ + RepoDir: repoDir, + OwnerName: ownerName, + RepoName: repoName, + Units: units, + }) + req.Body(jsonBytes) + resp, err := req.Response() + if err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v, could you confirm it's running?", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + var ret = struct { + Err string `json:"err"` + }{} + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Response body error: %v", err.Error()) + } + if err := json.Unmarshal(body, &ret); err != nil { + return http.StatusInternalServerError, fmt.Sprintf("Response body Unmarshal error: %v", err.Error()) + } + } + + return http.StatusOK, fmt.Sprintf("Restore repo %s/%s successfully", ownerName, repoName) +} diff --git a/routers/private/internal.go b/routers/private/internal.go index e541591a3840b..15a393c530678 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -69,6 +69,7 @@ func Routes() *web.Route { r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger) r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger) r.Post("/mail/send", SendEmail) + r.Post("/restore_repo", RestoreRepo) return r } diff --git a/routers/private/restore_repo.go b/routers/private/restore_repo.go new file mode 100644 index 0000000000000..8b6e670ff564a --- /dev/null +++ b/routers/private/restore_repo.go @@ -0,0 +1,52 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package private + +import ( + "context" + "io/ioutil" + + myCtx "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/migrations" + jsoniter "github.com/json-iterator/go" +) + +// RestoreRepo restore a repository from data +func RestoreRepo(ctx *myCtx.PrivateContext) { + json := jsoniter.ConfigCompatibleWithStandardLibrary + bs, err := ioutil.ReadAll(ctx.Req.Body) + if err != nil { + ctx.JSON(500, map[string]string{ + "err": err.Error(), + }) + return + } + var params = struct { + RepoDir string + OwnerName string + RepoName string + Units []string + }{} + if err = json.Unmarshal(bs, ¶ms); err != nil { + ctx.JSON(500, map[string]string{ + "err": err.Error(), + }) + return + } + + if err := migrations.RestoreRepository( + context.Background(), + params.RepoDir, + params.OwnerName, + params.RepoName, + params.Units, + ); err != nil { + ctx.JSON(500, map[string]string{ + "err": err.Error(), + }) + } else { + ctx.Status(200) + } +} From 2f06f4f2898ef66edb2ce9176e66183bd380aab8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 9 May 2021 14:55:18 +0800 Subject: [PATCH 2/4] Follow @zeripath's review --- modules/migrations/dump.go | 8 -------- routers/private/restore_repo.go | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go index eb032d041bd3e..4a18c47ae5bec 100644 --- a/modules/migrations/dump.go +++ b/modules/migrations/dump.go @@ -616,14 +616,6 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName tp, _ := strconv.Atoi(opts["service_type"]) var migrateOpts = base.MigrateOptions{ - Wiki: true, - Issues: true, - Milestones: true, - Labels: true, - Releases: true, - Comments: true, - PullRequests: true, - ReleaseAssets: true, GitServiceType: structs.GitServiceType(tp), } updateOptionsUnits(&migrateOpts, units) diff --git a/routers/private/restore_repo.go b/routers/private/restore_repo.go index 8b6e670ff564a..cf0831a011fab 100644 --- a/routers/private/restore_repo.go +++ b/routers/private/restore_repo.go @@ -5,10 +5,10 @@ package private import ( - "context" "io/ioutil" myCtx "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/migrations" jsoniter "github.com/json-iterator/go" ) @@ -37,7 +37,7 @@ func RestoreRepo(ctx *myCtx.PrivateContext) { } if err := migrations.RestoreRepository( - context.Background(), + graceful.GetManager().HammerContext(), params.RepoDir, params.OwnerName, params.RepoName, From 13df0cca2a76824628705c302d6efec974b652fc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 May 2021 09:01:32 +0800 Subject: [PATCH 3/4] set no timeout for resotre repo private request --- modules/private/restore_repo.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/private/restore_repo.go b/modules/private/restore_repo.go index c9b33b42a6fd3..6fe2e6844b986 100644 --- a/modules/private/restore_repo.go +++ b/modules/private/restore_repo.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" "net/http" + "time" "code.gitea.io/gitea/modules/setting" jsoniter "github.com/json-iterator/go" @@ -26,6 +27,7 @@ func RestoreRepo(repoDir, ownerName, repoName string, units []string) (int, stri reqURL := setting.LocalURL + "api/internal/restore_repo" req := newInternalRequest(reqURL, "POST") + req.SetTimeout(3*time.Second, 0) // since the request will spend much time, don't timeout req = req.Header("Content-Type", "application/json") json := jsoniter.ConfigCompatibleWithStandardLibrary jsonBytes, _ := json.Marshal(RestoreParams{ From aa24186682f258de8decb1c651bf9d04b90cc6af Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 10 May 2021 15:13:41 +0800 Subject: [PATCH 4/4] make restore repo cancelable --- routers/private/restore_repo.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routers/private/restore_repo.go b/routers/private/restore_repo.go index cf0831a011fab..c002de874a96e 100644 --- a/routers/private/restore_repo.go +++ b/routers/private/restore_repo.go @@ -8,7 +8,6 @@ import ( "io/ioutil" myCtx "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/migrations" jsoniter "github.com/json-iterator/go" ) @@ -37,7 +36,7 @@ func RestoreRepo(ctx *myCtx.PrivateContext) { } if err := migrations.RestoreRepository( - graceful.GetManager().HammerContext(), + ctx.Req.Context(), params.RepoDir, params.OwnerName, params.RepoName,