WIP: Add support to migrate from gogs

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561
 	github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14
+	github.com/gogs/go-gogs-client v0.0.0-20190710002546-4c3c18947c15
 	github.com/google/go-github/v24 v24.0.1
 	github.com/gorilla/context v1.1.1
 	github.com/issue9/assert v1.3.2 // indirect
@@ -69,8 +70,6 @@ require (
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/mcuadros/go-version v0.0.0-20190308113854-92cdf37c5b75
 	github.com/microcosm-cc/bluemonday v0.0.0-20161012083705-f77f16ffc87a
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect
 	github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
@@ -82,7 +81,6 @@ require (
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/procfs v0.0.4 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect
-	github.com/russross/blackfriday v2.0.0+incompatible // indirect
 	github.com/russross/blackfriday/v2 v2.0.1
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/satori/go.uuid v1.2.0

go.sum

@@ -243,9 +243,13 @@ github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561 h1:deE7ritpK04Pgtpy
 github.com/gogits/chardet v0.0.0-20150115103509-2404f7772561/go.mod h1:YgYOrVn3Nj9Tq0EvjmFbphRytDj7JNRoWSStJZWDJTQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8=
 github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3G5amEoiNbhVrmM+ZQEcnQvNQ2KaZk=
+github.com/gogs/go-gogs-client v0.0.0-20190710002546-4c3c18947c15 h1:tgEyCCe4+o8A2K/PEi9lF0QMA6XK+Y/j/WN01LnNbbo=
+github.com/gogs/go-gogs-client v0.0.0-20190710002546-4c3c18947c15/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -254,6 +258,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -423,8 +428,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
 github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
-github.com/niklasfasching/go-org v0.1.6 h1:F521WcqRNl8OJumlgAnekZgERaTA2HpfOYYfVEKOeI8=
-github.com/niklasfasching/go-org v0.1.6/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU=
 github.com/niklasfasching/go-org v0.1.7 h1:t3V+3XnS/7BhKv/7SlMUa8FvAiq577/a1T3D7mLIRXE=
 github.com/niklasfasching/go-org v0.1.7/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUvGZCCH1Nz0VdrU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -489,8 +492,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qq
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
@@ -501,8 +502,6 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b h1:4kg1wyftSKxLtnPAvcRWakIPpokB9w780/KwrNLnfPA=
 github.com/shurcooL/httpfs v0.0.0-20190527155220-6a4d4a70508b/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20160918041101-1dba4b3954bc h1:3wIrJvFb3Pf6B/2mDBnN1G5IfUVev4X5apadQlWOczE=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20160918041101-1dba4b3954bc/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0=
@@ -654,8 +653,6 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc=
-golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271 h1:N66aaryRB3Ax92gH0v3hp1QYZ3zWWCCUR/j8Ifh45Ss=
 golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -702,6 +699,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

modules/migrations/base/comment.go

@@ -14,6 +14,7 @@ type Comment struct {
 	PosterName  string
 	PosterEmail string
 	Created     time.Time
+	Updated     time.Time
 	Content     string
 	Reactions   *Reactions
 }

modules/migrations/base/milestone.go

@@ -15,5 +15,5 @@ type Milestone struct {
 	Created     time.Time
 	Updated     *time.Time
 	Closed      *time.Time
-	State       string
+	State       string // open, closed
 }

modules/migrations/gogs.go

@@ -0,0 +1,254 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
+
+	"github.com/gogs/go-gogs-client"
+)
+
+var (
+	_ base.Downloader        = &GogsDownloader{}
+	_ base.DownloaderFactory = &GogsDownloaderFactory{}
+)
+
+func init() {
+	RegisterDownloaderFactory(&GogsDownloaderFactory{})
+}
+
+// GogsDownloaderFactory defines a gogs downloader factory
+type GogsDownloaderFactory struct {
+}
+
+// Match returns ture if the migration remote URL matched this downloader factory
+func (f *GogsDownloaderFactory) Match(opts base.MigrateOptions) (bool, error) {
+	if opts.GitServiceType == structs.GogsService {
+		return true, nil
+	}
+	return false, nil
+}
+
+// New returns a Downloader related to this factory according MigrateOptions
+func (f *GogsDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader, error) {
+	u, err := url.Parse(opts.CloneAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	baseURL := u.Scheme + "://" + u.Host
+	fields := strings.Split(u.Path, "/")
+	oldOwner := fields[1]
+	oldName := strings.TrimSuffix(fields[2], ".git")
+
+	log.Trace("Create gogs downloader: %s/%s", oldOwner, oldName)
+
+	return NewGogsDownloader(baseURL, opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
+}
+
+// GitServiceType returns the type of git service
+func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType {
+	return structs.GogsService
+}
+
+// GogsDownloader implements a Downloader interface to get repository informations
+// from gogs via API
+type GogsDownloader struct {
+	client    *gogs.Client
+	baseURL   string
+	repoOwner string
+	repoName  string
+	userName  string
+	password  string
+}
+
+// NewGogsDownloader creates a gogs Downloader via gogs API
+func NewGogsDownloader(baseURL, userName, password, repoOwner, repoName string) *GogsDownloader {
+	var downloader = GogsDownloader{
+		baseURL:   baseURL,
+		userName:  userName,
+		password:  password,
+		repoOwner: repoOwner,
+		repoName:  repoName,
+	}
+
+	var client *gogs.Client
+	if userName != "" {
+		if password == "" {
+			client = gogs.NewClient(baseURL, userName)
+		} else {
+			client = gogs.NewClient(baseURL, "")
+			client.SetHTTPClient(&http.Client{
+				Transport: &http.Transport{
+					Proxy: func(req *http.Request) (*url.URL, error) {
+						req.SetBasicAuth(userName, password)
+						return nil, nil
+					},
+				},
+			})
+		}
+	}
+	downloader.client = client
+	return &downloader
+}
+
+// GetRepoInfo returns a repository information
+func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) {
+	gr, err := g.client.GetRepo(g.repoOwner, g.repoName)
+	if err != nil {
+		return nil, err
+	}
+
+	// convert github repo to stand Repo
+	return &base.Repository{
+		Owner:       g.repoOwner,
+		Name:        g.repoName,
+		IsPrivate:   gr.Private,
+		Description: gr.Description,
+		CloneURL:    gr.CloneURL,
+	}, nil
+}
+
+// GetTopics return github topics
+func (g *GogsDownloader) GetTopics() ([]string, error) {
+	return []string{}, nil
+}
+
+// GetMilestones returns milestones
+func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) {
+	var perPage = 100
+	var milestones = make([]*base.Milestone, 0, perPage)
+
+	ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName)
+	if err != nil {
+		return nil, err
+	}
+
+	t := time.Now()
+
+	for _, m := range ms {
+		milestones = append(milestones, &base.Milestone{
+			Title:       m.Title,
+			Description: m.Description,
+			Deadline:    m.Deadline,
+			State:       string(m.State),
+			Created:     t,
+			Updated:     &t,
+			Closed:      m.Closed,
+		})
+	}
+
+	return milestones, nil
+}
+
+func convertGogsLabel(label *gogs.Label) *base.Label {
+	return &base.Label{
+		Name:  label.Name,
+		Color: label.Color,
+	}
+}
+
+// GetLabels returns labels
+func (g *GogsDownloader) GetLabels() ([]*base.Label, error) {
+	var perPage = 100
+	var labels = make([]*base.Label, 0, perPage)
+	ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, label := range ls {
+		labels = append(labels, convertGogsLabel(label))
+	}
+
+	return labels, nil
+}
+
+// GetReleases returns releases
+// FIXME: gogs API haven't support get releases
+func (g *GogsDownloader) GetReleases() ([]*base.Release, error) {
+	return nil, ErrNotSupported
+}
+
+// GetIssues returns issues according start and limit, perPage is not supported
+func (g *GogsDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+	var allIssues = make([]*base.Issue, 0, perPage)
+
+	issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{
+		Page: page,
+	})
+	if err != nil {
+		return nil, false, fmt.Errorf("error while listing repos: %v", err)
+	}
+	for _, issue := range issues {
+		if issue.PullRequest != nil {
+			continue
+		}
+
+		var milestone string
+		if issue.Milestone != nil {
+			milestone = issue.Milestone.Title
+		}
+		var labels = make([]*base.Label, 0, len(issue.Labels))
+		for _, l := range issue.Labels {
+			labels = append(labels, convertGogsLabel(l))
+		}
+
+		var closed *time.Time
+		if issue.State == gogs.STATE_CLOSED {
+			// gogs client haven't provide closed, so we use updated instead
+			closed = &issue.Updated
+		}
+
+		allIssues = append(allIssues, &base.Issue{
+			Title:       issue.Title,
+			Number:      issue.Index,
+			PosterName:  issue.Poster.Login,
+			PosterEmail: issue.Poster.Email,
+			Content:     issue.Body,
+			Milestone:   milestone,
+			State:       string(issue.State),
+			Created:     issue.Created,
+			Labels:      labels,
+			Closed:      closed,
+		})
+	}
+
+	return allIssues, len(issues) == 0, nil
+}
+
+// GetComments returns comments according issueNumber
+func (g *GogsDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
+	var allComments = make([]*base.Comment, 0, 100)
+
+	comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber)
+	if err != nil {
+		return nil, fmt.Errorf("error while listing repos: %v", err)
+	}
+	for _, comment := range comments {
+		allComments = append(allComments, &base.Comment{
+			PosterName:  comment.Poster.Login,
+			PosterEmail: comment.Poster.Email,
+			Content:     comment.Body,
+			Created:     comment.Created,
+			Updated:     comment.Updated,
+		})
+	}
+
+	return allComments, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (g *GogsDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
+	return nil, ErrNotSupported
+}