From 5014204f6be1035fb3d0e266cfa0ac3ce2aeeb90 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 15 Nov 2017 17:15:34 +0800 Subject: [PATCH 1/5] Reimplement dump and implment restore command --- cmd/dump.go | 45 ++++++++------ cmd/restore.go | 148 +++++++++++++++++++++++++++++++++++++++++++++++ main.go | 1 + models/models.go | 80 +++++++++++++++++++++++++ 4 files changed, 256 insertions(+), 18 deletions(-) create mode 100644 cmd/restore.go diff --git a/cmd/dump.go b/cmd/dump.go index a895785295939..2b5bc6a6bcbd2 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -1,5 +1,5 @@ // Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2016 The Gitea Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -44,10 +44,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`, Value: os.TempDir(), Usage: "Temporary dir path", }, - cli.StringFlag{ - Name: "database, d", - Usage: "Specify the database SQL syntax", - }, cli.BoolFlag{ Name: "skip-repository, R", Usage: "Skip the repository dumping", @@ -83,10 +79,9 @@ func runDump(ctx *cli.Context) error { os.Setenv("TMPDIR", tmpWorkDir) } - dbDump := path.Join(tmpWorkDir, "gitea-db.sql") + log.Printf("Packing dump files...") fileName := fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()) - log.Printf("Packing dump files...") z, err := zip.Create(fileName) if err != nil { log.Fatalf("Failed to create %s: %v", fileName, err) @@ -106,20 +101,23 @@ func runDump(ctx *cli.Context) error { } } - targetDBType := ctx.String("database") - if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type { - log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) - } else { - log.Printf("Dumping database...") + log.Printf("Dumping database...") + + dbDump := path.Join(tmpWorkDir, "database") + if err := os.MkdirAll(dbDump, os.ModePerm); err != nil { + log.Fatalf("Failed to create database dir: %v", err) } - if err := models.DumpDatabase(dbDump, targetDBType); err != nil { + if err := models.DumpDatabaseFixtures(dbDump); err != nil { log.Fatalf("Failed to dump database: %v", err) } - if err := z.AddFile("gitea-db.sql", dbDump); err != nil { - log.Fatalf("Failed to include gitea-db.sql: %v", err) + if err := z.AddDir("database", dbDump); err != nil { + log.Fatalf("Failed to include database: %v", err) } + + log.Printf("Dumping custom directory ... %s", setting.CustomPath) + customDir, err := os.Stat(setting.CustomPath) if err == nil && customDir.IsDir() { if err := z.AddDir("custom", setting.CustomPath); err != nil { @@ -130,7 +128,7 @@ func runDump(ctx *cli.Context) error { } if com.IsExist(setting.AppDataPath) { - log.Printf("Packing data directory...%s", setting.AppDataPath) + log.Printf("Dumping data directory ... %s", setting.AppDataPath) var sessionAbsPath string if setting.SessionConfig.Provider == "file" { @@ -141,8 +139,19 @@ func runDump(ctx *cli.Context) error { } } - if err := z.AddDir("log", setting.LogRootPath); err != nil { - log.Fatalf("Failed to include log: %v", err) + verPath := filepath.Join(tmpWorkDir, "VERSION") + verf, err := os.Create(verPath) + if err != nil { + log.Fatalf("Failed to create version file: %v", err) + } + _, err = verf.WriteString(setting.AppVer) + verf.Close() + if err != nil { + log.Fatalf("Failed to write version to file: %v", err) + } + + if err = z.AddFile("VERSION", verPath); err != nil { + log.Fatalf("Failed to add version file: %v", err) } if err = z.Close(); err != nil { diff --git a/cmd/restore.go b/cmd/restore.go new file mode 100644 index 0000000000000..00341c0c3ff13 --- /dev/null +++ b/cmd/restore.go @@ -0,0 +1,148 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "errors" + "io/ioutil" + "log" + "os" + "path/filepath" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + + "github.com/Unknwon/cae/zip" + "github.com/Unknwon/com" + "github.com/urfave/cli" +) + +// CmdRestore represents the available restore sub-command. +var CmdRestore = cli.Command{ + Name: "restore", + Usage: "Restore Gitea files and database", + Description: `Restore will restore all data from zip file which dumped from gitea. It will use +the custom config in this dump zip file, this operation will remove all the dest database and repositories.`, + Action: runRestore, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Value: "custom/conf/app.ini", + Usage: "Custom configuration file path, if empty will use dumped config file", + }, + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Show process details", + }, + cli.StringFlag{ + Name: "tempdir, t", + Value: os.TempDir(), + Usage: "Temporary dir path", + }, + }, +} + +func runRestore(ctx *cli.Context) error { + if len(os.Args) < 3 { + return errors.New("need zip file path") + } + + tmpDir := ctx.String("tempdir") + if _, err := os.Stat(tmpDir); os.IsNotExist(err) { + log.Fatalf("Path does not exist: %s", tmpDir) + } + tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-") + if err != nil { + log.Fatalf("Failed to create tmp work directory: %v", err) + } + log.Printf("Creating tmp work dir: %s", tmpWorkDir) + + // work-around #1103 + if os.Getenv("TMPDIR") == "" { + os.Setenv("TMPDIR", tmpWorkDir) + } + + srcPath := os.Args[2] + + zip.Verbose = ctx.Bool("verbose") + log.Printf("Extracting %s to tmp work dir", srcPath) + err = zip.ExtractTo(srcPath, tmpWorkDir) + if err != nil { + log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) + } + + verData, err := ioutil.ReadFile(filepath.Join(tmpWorkDir, "VERSION")) + if err != nil { + log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) + } + + if setting.AppVer != string(verData) { + log.Fatalf("Expected gitea version to restore is %s, but get %s", string(verData), setting.AppVer) + } + + if ctx.IsSet("config") { + setting.CustomConf = ctx.String("config") + } else { + setting.CustomConf = filepath.Join(tmpWorkDir, "custom", "conf", "app.ini") + } + if !com.IsExist(setting.CustomConf) { + log.Fatalf("Failed to load ini config file from %s", setting.CustomConf) + } + + setting.NewContext() + //setting.CustomPath = filepath.Join(tmpWorkDir, "custom") + setting.NewXORMLogService(false) + models.LoadConfigs() + + err = models.SetEngine() + if err != nil { + log.Fatalf("Failed to SetEngine: %v", err) + } + + log.Printf("Restoring repo dir %s ...", setting.RepoRootPath) + repoPath := filepath.Join(tmpWorkDir, "repositories") + err = os.RemoveAll(setting.RepoRootPath) + if err != nil { + log.Fatalf("Failed to Remove repo root path %s: %v", setting.RepoRootPath, err) + } + + err = os.Rename(repoPath, setting.RepoRootPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err) + } + + log.Printf("Restoring custom dir %s ...", setting.CustomPath) + customPath := filepath.Join(tmpWorkDir, "custom") + err = os.RemoveAll(setting.CustomPath) + if err != nil { + log.Fatalf("Failed to Remove repo root path %s: %v", setting.CustomPath, err) + } + + err = os.Rename(customPath, setting.CustomPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err) + } + + log.Printf("Restoring data dir %s ...", setting.AppDataPath) + dataPath := filepath.Join(tmpWorkDir, "data") + err = os.RemoveAll(setting.AppDataPath) + if err != nil { + log.Fatalf("Failed to Remove data root path %s: %v", setting.AppDataPath, err) + } + + err = os.Rename(dataPath, setting.AppDataPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err) + } + + log.Printf("Restoring database from ...") + dbPath := filepath.Join(tmpWorkDir, "database") + err = models.RestoreDatabaseFixtures(dbPath) + if err != nil { + log.Fatalf("Failed to restore database dir %s: %v", dbPath, err) + } + + return nil +} diff --git a/main.go b/main.go index 976bbdf1f7669..d4ce8371c86ed 100644 --- a/main.go +++ b/main.go @@ -46,6 +46,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdServ, cmd.CmdHook, cmd.CmdDump, + cmd.CmdRestore, cmd.CmdCert, cmd.CmdAdmin, cmd.CmdGenerate, diff --git a/models/models.go b/models/models.go index e7ecc67fc5538..b154589de5dba 100644 --- a/models/models.go +++ b/models/models.go @@ -9,10 +9,12 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" "net/url" "os" "path" "path/filepath" + "reflect" "strings" "code.gitea.io/gitea/modules/log" @@ -22,6 +24,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + "gopkg.in/yaml.v2" // Needed for the Postgresql driver _ "github.com/lib/pq" @@ -360,3 +363,80 @@ func DumpDatabase(filePath string, dbType string) error { } return x.DumpTablesToFile(tbs, filePath) } + +// DumpDatabaseFixtures dumps all data from database to fixtures files on dirPath +func DumpDatabaseFixtures(dirPath string) error { + for _, t := range tables { + if err := dumpTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func dumpTableFixtures(bean interface{}, dirPath string) error { + table := x.TableInfo(bean) + f, err := os.Create(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + defer f.Close() + var bufferSize = 100 + var objs = make([]interface{}, 0, bufferSize) + err = x.BufferSize(bufferSize).Iterate(bean, func(idx int, obj interface{}) error { + objs = append(objs, obj) + if len(objs) == bufferSize { + // BLOCK: need yaml support gonic name mapper + data, err := yaml.Marshal(objs) + if err != nil { + return err + } + _, err = f.Write(data) + if err != nil { + return err + } + objs = make([]interface{}, 0, bufferSize) + } + return err + }) + if err != nil { + return err + } + if len(objs) > 0 { + data, err := yaml.Marshal(objs) + if err != nil { + return err + } + _, err = f.Write(data) + } + return err +} + +// RestoreDatabaseFixtures restores all data from dir to database +func RestoreDatabaseFixtures(dirPath string) error { + for _, t := range tables { + if err := restoreTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func restoreTableFixtures(bean interface{}, dirPath string) error { + table := x.TableInfo(bean) + data, err := ioutil.ReadFile(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + + var bufferSize = 100 + v := reflect.MakeSlice(table.Type, 0, bufferSize) + // BLOCK: need yaml support gonic name mapper + err = yaml.Unmarshal(data, v.Interface()) + if err != nil { + return err + } + + _, err = x.Insert(v.Interface()) + return err +} From 4fdb5fadf0fa4ca884ff83c65a5f92d242afb38a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 07:07:16 +0800 Subject: [PATCH 2/5] fix name convert --- models/models.go | 84 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/models/models.go b/models/models.go index b154589de5dba..68a8046691d4c 100644 --- a/models/models.go +++ b/models/models.go @@ -14,7 +14,7 @@ import ( "os" "path" "path/filepath" - "reflect" + "sort" "strings" "code.gitea.io/gitea/modules/log" @@ -381,35 +381,33 @@ func dumpTableFixtures(bean interface{}, dirPath string) error { return err } defer f.Close() - var bufferSize = 100 - var objs = make([]interface{}, 0, bufferSize) - err = x.BufferSize(bufferSize).Iterate(bean, func(idx int, obj interface{}) error { - objs = append(objs, obj) - if len(objs) == bufferSize { - // BLOCK: need yaml support gonic name mapper - data, err := yaml.Marshal(objs) - if err != nil { - return err - } - _, err = f.Write(data) - if err != nil { - return err - } - objs = make([]interface{}, 0, bufferSize) + + const bufferSize = 100 + var start = 0 + for { + objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface() + if err != nil { + return err } - return err - }) - if err != nil { - return err - } - if len(objs) > 0 { + if len(objs) == 0 { + break + } + data, err := yaml.Marshal(objs) if err != nil { return err } _, err = f.Write(data) + if err != nil { + return err + } + if len(objs) < bufferSize { + break + } + start += len(objs) } - return err + + return nil } // RestoreDatabaseFixtures restores all data from dir to database @@ -429,14 +427,44 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { return err } - var bufferSize = 100 - v := reflect.MakeSlice(table.Type, 0, bufferSize) - // BLOCK: need yaml support gonic name mapper - err = yaml.Unmarshal(data, v.Interface()) + const bufferSize = 100 + var records = make([]map[string]interface{}, 0, bufferSize*10) + err = yaml.Unmarshal(data, records) if err != nil { return err } - _, err = x.Insert(v.Interface()) + if len(records) == 0 { + return nil + } + + var columns = make([]string, 0, len(records[0])) + for k, _ := range records[0] { + columns = append(columns, k) + } + sort.Strings(columns) + + qm := strings.Repeat("?,", len(columns)) + qm = "(" + qm[:len(qm)-1] + ")" + + var sql = "INSERT INTO " + table.Name + "(" + strings.Join(columns, ",") + ") VALUES " + var args = make([]interface{}, 0, bufferSize) + var insertSQLs = make([]string, 0, bufferSize) + for i, vals := range records { + insertSQLs = append(insertSQLs, qm) + for _, colName := range columns { + args = append(args, vals[colName]) + } + + if i+1%100 == 0 || i == len(records)-1 { + _, err = x.Exec(sql+strings.Join(insertSQLs, ","), args...) + if err != nil { + return err + } + insertSQLs = make([]string, 0, bufferSize) + args = make([]interface{}, 0, bufferSize) + } + } + return err } From 7302de86a8fcb31536ac3afcb2234c3c801db8d7 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 16:29:57 +0800 Subject: [PATCH 3/5] fix lint --- cmd/dump.go | 2 +- models/models.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 2b5bc6a6bcbd2..3530c63e545c8 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -1,5 +1,5 @@ // Copyright 2014 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. +// Copyright 2016 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/models/models.go b/models/models.go index 68a8046691d4c..03d41762d6f7d 100644 --- a/models/models.go +++ b/models/models.go @@ -439,7 +439,7 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { } var columns = make([]string, 0, len(records[0])) - for k, _ := range records[0] { + for k := range records[0] { columns = append(columns, k) } sort.Strings(columns) From dcb0811fd77f4bae4c2103905bd7997412e49d27 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 17:37:52 +0800 Subject: [PATCH 4/5] fix typo --- cmd/restore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restore.go b/cmd/restore.go index 00341c0c3ff13..cdba8b8bb6459 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -53,7 +53,7 @@ func runRestore(ctx *cli.Context) error { if _, err := os.Stat(tmpDir); os.IsNotExist(err) { log.Fatalf("Path does not exist: %s", tmpDir) } - tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-") + tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-restore-") if err != nil { log.Fatalf("Failed to create tmp work directory: %v", err) } From c85526deb80c8a503e12e928f30bb317956b813d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 8 May 2018 18:08:26 +0800 Subject: [PATCH 5/5] fix restore --- cmd/restore.go | 15 ++++++++++----- models/models.go | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cmd/restore.go b/cmd/restore.go index cdba8b8bb6459..f6d26047afac9 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -67,7 +67,7 @@ func runRestore(ctx *cli.Context) error { srcPath := os.Args[2] zip.Verbose = ctx.Bool("verbose") - log.Printf("Extracting %s to tmp work dir", srcPath) + log.Printf("Extracting %s to %s", srcPath, tmpWorkDir) err = zip.ExtractTo(srcPath, tmpWorkDir) if err != nil { log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) @@ -101,7 +101,12 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to SetEngine: %v", err) } - log.Printf("Restoring repo dir %s ...", setting.RepoRootPath) + err = models.SyncDBStructs() + if err != nil { + log.Fatalf("Failed to SyncDBStructs: %v", err) + } + + log.Printf("Restoring repo dir to %s ...", setting.RepoRootPath) repoPath := filepath.Join(tmpWorkDir, "repositories") err = os.RemoveAll(setting.RepoRootPath) if err != nil { @@ -113,7 +118,7 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err) } - log.Printf("Restoring custom dir %s ...", setting.CustomPath) + log.Printf("Restoring custom dir to %s ...", setting.CustomPath) customPath := filepath.Join(tmpWorkDir, "custom") err = os.RemoveAll(setting.CustomPath) if err != nil { @@ -125,7 +130,7 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err) } - log.Printf("Restoring data dir %s ...", setting.AppDataPath) + log.Printf("Restoring data dir to %s ...", setting.AppDataPath) dataPath := filepath.Join(tmpWorkDir, "data") err = os.RemoveAll(setting.AppDataPath) if err != nil { @@ -137,8 +142,8 @@ func runRestore(ctx *cli.Context) error { log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err) } - log.Printf("Restoring database from ...") dbPath := filepath.Join(tmpWorkDir, "database") + log.Printf("Restoring database from %s ...", dbPath) err = models.RestoreDatabaseFixtures(dbPath) if err != nil { log.Fatalf("Failed to restore database dir %s: %v", dbPath, err) diff --git a/models/models.go b/models/models.go index 03d41762d6f7d..8687da0cd67b8 100644 --- a/models/models.go +++ b/models/models.go @@ -304,6 +304,15 @@ func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) { return nil } +// SyncDBStructs will sync database structs +func SyncDBStructs() error { + if err := x.StoreEngine("InnoDB").Sync2(tables...); err != nil { + return fmt.Errorf("sync database struct error: %v", err) + } + + return nil +} + // Statistic contains the database statistics type Statistic struct { Counter struct { @@ -429,7 +438,7 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { const bufferSize = 100 var records = make([]map[string]interface{}, 0, bufferSize*10) - err = yaml.Unmarshal(data, records) + err = yaml.Unmarshal(data, &records) if err != nil { return err } @@ -447,7 +456,12 @@ func restoreTableFixtures(bean interface{}, dirPath string) error { qm := strings.Repeat("?,", len(columns)) qm = "(" + qm[:len(qm)-1] + ")" - var sql = "INSERT INTO " + table.Name + "(" + strings.Join(columns, ",") + ") VALUES " + _, err = x.Exec("DELETE FROM `" + table.Name + "`") + if err != nil { + return err + } + + var sql = "INSERT INTO `" + table.Name + "` (`" + strings.Join(columns, "`,`") + "`) VALUES " var args = make([]interface{}, 0, bufferSize) var insertSQLs = make([]string, 0, bufferSize) for i, vals := range records {