feat(migrations): GraphQL fast path for GitHub migration + rate-limit resilience

modules/migration/options.go

@@ -39,6 +39,19 @@ type MigrateOptions struct {
 	MigrateToRepoID int64
 	MirrorInterval  string `json:"mirror_interval"`
 
+	// mirror-only: enable read-only metadata sync of issues/pull requests
+	// on each mirror update.
+	SyncIssues       bool
+	SyncPullRequests bool
+
+	// SkipReactions omits per-issue and per-comment reaction fetches. Reactions
+	// are an N+1 call storm (one request per issue and per comment) that
+	// dominates the API budget and, on a single transient failure, aborts the
+	// whole pass. The mirror metadata reflection sets this so large repositories
+	// complete in a sane window; reactions are the lowest-value metadata for a
+	// read-only mirror.
+	SkipReactions bool
+
 	AWSAccessKeyID     string
 	AWSSecretAccessKey string `json:",omitempty"`
 

modules/setting/migrations.go

@@ -11,6 +11,14 @@ var Migrations = struct {
 	BlockedDomains     string
 	AllowLocalNetworks bool
 	SkipTLSVerify      bool
+	// the batched GraphQL fast path (see services/migrations/github_graphql.go),
+	// cutting rate-limit pressure on large repositories. Off by default; set
+	// [migrations] USE_GRAPHQL_FOR_MIRROR = true to enable (e.g. for A/B timing).
+	// SyncReactionsForMirror opts metadata mirror syncs into fetching issue/PR/comment
+	// reactions too. Off by default because reactions are an N+1 call storm (the lowest
+	// -value, heaviest metadata for a read-only mirror); set
+	// [migrations] SYNC_REACTIONS_FOR_MIRROR = true to include them.
+	SyncReactionsForMirror bool
 }{
 	MaxAttempts:  3,
 	RetryBackoff: 3,
@@ -25,4 +33,5 @@ func loadMigrationsFrom(rootCfg ConfigProvider) {
 	Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").MustString("")
 	Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
 	Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
+	Migrations.SyncReactionsForMirror = sec.Key("SYNC_REACTIONS_FOR_MIRROR").MustBool(false)
 }

modules/structs/repo.go

@@ -418,6 +418,14 @@ type MigrateRepoOptions struct {
 	Releases       bool   `json:"releases"`
 	MirrorInterval string `json:"mirror_interval"`
 
+	// only used when mirror is true: keep the mirror's issues/pull requests in
+	// sync with the remote on each mirror update, shown read-only.
+	SyncIssues       bool `json:"sync_issues"`
+	SyncPullRequests bool `json:"sync_pull_requests"`
+
+	// opt into the batched GraphQL fast path for GitHub migrations (fewer API
+	// calls / less rate-limit pressure on large repos). REST is the default.
+
 	AWSAccessKeyID     string `json:"aws_access_key_id"`
 	AWSSecretAccessKey string `json:"aws_secret_access_key"`
 }

routers/api/v1/repo/migrate.go

@@ -28,6 +28,7 @@ import (
 	"gitea.dev/services/context"
 	"gitea.dev/services/convert"
 	"gitea.dev/services/migrations"
+	mirror_service "gitea.dev/services/mirror"
 	notify_service "gitea.dev/services/notify"
 	repo_service "gitea.dev/services/repository"
 )
@@ -163,6 +164,8 @@ func Migrate(ctx *context.APIContext) {
 		opts.Comments = false
 		opts.PullRequests = false
 		opts.Releases = false
+		opts.SyncIssues = form.SyncIssues
+		opts.SyncPullRequests = form.SyncPullRequests
 	}
 	if gitServiceType == api.CodeCommitService {
 		opts.AWSAccessKeyID = form.AWSAccessKeyID
@@ -201,6 +204,10 @@ func Migrate(ctx *context.APIContext) {
 			return nil, err
 		}
 		notify_service.MigrateRepository(ctx, doer, repoOwner, migratedRepo)
+		if opts.Mirror && (opts.SyncIssues || opts.SyncPullRequests) {
+			log.Info("migrate: initiating immediate metadata sync for mirror [%d] %s/%s", migratedRepo.ID, repoOwner.Name, opts.RepoName)
+			mirror_service.AddPullMirrorToQueue(migratedRepo.ID)
+		}
 		return migratedRepo, nil
 	}
 

services/forms/repo_form.go

@@ -79,6 +79,9 @@ type MigrateRepoForm struct {
 	Releases       bool   `json:"releases"`
 	MirrorInterval string `json:"mirror_interval"`
 
+	SyncIssues       bool `json:"sync_issues"`
+	SyncPullRequests bool `json:"sync_pull_requests"`
+
 	AWSAccessKeyID     string `json:"aws_access_key_id"`
 	AWSSecretAccessKey string `json:"aws_secret_access_key"`
 }
@@ -109,6 +112,12 @@ type RepoSettingForm struct {
 	Template               bool
 	EnablePrune            bool
 
+	// Pull-mirror metadata sync toggles. Issues and pull requests are the only
+	// choices; their comments, reviews, labels and milestones come along as
+	// part of the read-only reflection.
+	MirrorSyncIssues       bool `form:"mirror_sync_issues"`
+	MirrorSyncPullRequests bool `form:"mirror_sync_pull_requests"`
+
 	// Advanced settings
 	EnableCode bool
 

services/migrations/_mock_data/TestGitHubDownloadRepo/GET_%2Fapi%2Fv3%2Frepos%2Fgo-gitea%2Ftest_repo%2Fissues%3Fdirection%3Dasc%26per_page%3D2%26sort%3Dcreated%26state%3Dall

@@ -0,0 +1,24 @@
+Access-Control-Allow-Origin: *
+Access-Control-Expose-Headers: ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset
+Cache-Control: private, max-age=60, s-maxage=60
+Content-Security-Policy: default-src 'none'
+Content-Type: application/json; charset=utf-8
+Etag: W/"4d1d8ecb86bafe76a686d250018cc158745dd6bc794ef651eeeeecbfc1a947bc"
+Link: <https://api.github.com/repositories/220672974/issues?direction=asc&page=2&per_page=2&sort=created&state=all>; rel="next", <https://api.github.com/repositories/220672974/issues?direction=asc&page=3&per_page=2&sort=created&state=all>; rel="last"
+Referrer-Policy: origin-when-cross-origin, strict-origin-when-cross-origin
+Strict-Transport-Security: max-age=31536000; includeSubdomains; preload
+Vary: Accept, Authorization, Cookie, X-GitHub-OTP,Accept-Encoding, Accept, X-Requested-With
+X-Accepted-Github-Permissions: issues=read
+X-Content-Type-Options: nosniff
+X-Frame-Options: deny
+X-Github-Api-Version-Selected: 2022-11-28
+X-Github-Media-Type: github.v3; param=squirrel-girl-preview
+X-Github-Request-Id: C4F6:A93E1:6A3A9E:5B260C:69AF24E7
+X-Ratelimit-Limit: 5000
+X-Ratelimit-Remaining: 4931
+X-Ratelimit-Reset: 1773089427
+X-Ratelimit-Resource: core
+X-Ratelimit-Used: 69
+X-Xss-Protection: 0
+
+[{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/1","repository_url":"https://api.github.com/repos/go-gitea/test_repo","labels_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/labels{/name}","comments_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/comments","events_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/events","html_url":"https://github.com/go-gitea/test_repo/issues/1","id":520479843,"node_id":"MDU6SXNzdWU1MjA0Nzk4NDM=","number":1,"title":"Please add an animated gif icon to the merge button","user":{"login":"guillep2k","id":18600385,"node_id":"MDQ6VXNlcjE4NjAwMzg1","avatar_url":"https://avatars.githubusercontent.com/u/18600385?v=4","gravatar_id":"","url":"https://api.github.com/users/guillep2k","html_url":"https://github.com/guillep2k","followers_url":"https://api.github.com/users/guillep2k/followers","following_url":"https://api.github.com/users/guillep2k/following{/other_user}","gists_url":"https://api.github.com/users/guillep2k/gists{/gist_id}","starred_url":"https://api.github.com/users/guillep2k/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/guillep2k/subscriptions","organizations_url":"https://api.github.com/users/guillep2k/orgs","repos_url":"https://api.github.com/users/guillep2k/repos","events_url":"https://api.github.com/users/guillep2k/events{/privacy}","received_events_url":"https://api.github.com/users/guillep2k/received_events","type":"User","user_view_type":"public","site_admin":false},"labels":[{"id":1667254252,"node_id":"MDU6TGFiZWwxNjY3MjU0MjUy","url":"https://api.github.com/repos/go-gitea/test_repo/labels/bug","name":"bug","color":"d73a4a","default":true,"description":"Something isn't working"},{"id":1667254261,"node_id":"MDU6TGFiZWwxNjY3MjU0MjYx","url":"https://api.github.com/repos/go-gitea/test_repo/labels/good%20first%20issue","name":"good first issue","color":"7057ff","default":true,"description":"Good for newcomers"}],"state":"closed","locked":false,"assignees":[],"milestone":{"url":"https://api.github.com/repos/go-gitea/test_repo/milestones/1","html_url":"https://github.com/go-gitea/test_repo/milestone/1","labels_url":"https://api.github.com/repos/go-gitea/test_repo/milestones/1/labels","id":4839941,"node_id":"MDk6TWlsZXN0b25lNDgzOTk0MQ==","number":1,"title":"1.0.0","description":"Milestone 1.0.0","creator":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"open_issues":1,"closed_issues":1,"state":"closed","created_at":"2019-11-12T19:37:08Z","updated_at":"2019-11-12T21:56:17Z","due_on":"2019-11-11T00:00:00Z","closed_at":"2019-11-12T19:45:49Z"},"comments":0,"created_at":"2019-11-09T17:00:29Z","updated_at":"2019-11-12T20:29:53Z","closed_at":"2019-11-12T20:22:22Z","assignee":null,"author_association":"MEMBER","type":null,"active_lock_reason":null,"sub_issues_summary":{"total":0,"completed":0,"percent_completed":0},"issue_dependencies_summary":{"blocked_by":0,"total_blocked_by":0,"blocking":0,"total_blocking":0},"body":"I just want the merge button to hurt my eyes a little. 😝 ","closed_by":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"reactions":{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/reactions","total_count":1,"+1":1,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/go-gitea/test_repo/issues/1/timeline","performed_via_github_app":null,"state_reason":"completed","pinned_comment":null},{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/2","repository_url":"https://api.github.com/repos/go-gitea/test_repo","labels_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/labels{/name}","comments_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/comments","events_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/events","html_url":"https://github.com/go-gitea/test_repo/issues/2","id":521799485,"node_id":"MDU6SXNzdWU1MjE3OTk0ODU=","number":2,"title":"Test issue","user":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"labels":[{"id":1667254257,"node_id":"MDU6TGFiZWwxNjY3MjU0MjU3","url":"https://api.github.com/repos/go-gitea/test_repo/labels/duplicate","name":"duplicate","color":"cfd3d7","default":true,"description":"This issue or pull request already exists"}],"state":"closed","locked":false,"assignees":[],"milestone":{"url":"https://api.github.com/repos/go-gitea/test_repo/milestones/2","html_url":"https://github.com/go-gitea/test_repo/milestone/2","labels_url":"https://api.github.com/repos/go-gitea/test_repo/milestones/2/labels","id":4839942,"node_id":"MDk6TWlsZXN0b25lNDgzOTk0Mg==","number":2,"title":"1.1.0","description":"Milestone 1.1.0","creator":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"open_issues":0,"closed_issues":2,"state":"closed","created_at":"2019-11-12T19:37:25Z","updated_at":"2019-11-12T21:39:27Z","due_on":"2019-11-12T00:00:00Z","closed_at":"2019-11-12T19:45:46Z"},"comments":2,"created_at":"2019-11-12T21:00:06Z","updated_at":"2019-11-12T22:07:14Z","closed_at":"2019-11-12T21:01:31Z","assignee":null,"author_association":"MEMBER","type":null,"active_lock_reason":null,"sub_issues_summary":{"total":0,"completed":0,"percent_completed":0},"issue_dependencies_summary":{"blocked_by":0,"total_blocked_by":0,"blocking":0,"total_blocking":0},"body":"This is test issue 2, do not touch!","closed_by":{"login":"mrsdizzie","id":1669571,"node_id":"MDQ6VXNlcjE2Njk1NzE=","avatar_url":"https://avatars.githubusercontent.com/u/1669571?v=4","gravatar_id":"","url":"https://api.github.com/users/mrsdizzie","html_url":"https://github.com/mrsdizzie","followers_url":"https://api.github.com/users/mrsdizzie/followers","following_url":"https://api.github.com/users/mrsdizzie/following{/other_user}","gists_url":"https://api.github.com/users/mrsdizzie/gists{/gist_id}","starred_url":"https://api.github.com/users/mrsdizzie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrsdizzie/subscriptions","organizations_url":"https://api.github.com/users/mrsdizzie/orgs","repos_url":"https://api.github.com/users/mrsdizzie/repos","events_url":"https://api.github.com/users/mrsdizzie/events{/privacy}","received_events_url":"https://api.github.com/users/mrsdizzie/received_events","type":"User","user_view_type":"public","site_admin":false},"reactions":{"url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/reactions","total_count":6,"+1":1,"-1":1,"laugh":1,"hooray":1,"confused":1,"heart":1,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/go-gitea/test_repo/issues/2/timeline","performed_via_github_app":null,"state_reason":"completed","pinned_comment":null}]