Allow providing a token for accessing GitHub.com.

README.md

@@ -22,13 +22,15 @@ From a machine with access to both GitHub.com and GitHub Enterprise Server use t
 
 **Optional Arguments:**
 * `--cache-dir` - A temporary directory in which to store data downloaded from GitHub.com before it is uploaded to GitHub Enterprise Server. If not specified a directory next to the sync tool will be used.
+* `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. If provided, it should have the `public_repo` scope.
 * `--destination-repository` - The name of the repository in which to create or update the CodeQL Action. If not specified `github/codeql-action` will be used.
 
 ### I don't have a machine that can access both GitHub.com and GitHub Enterprise Server.
 From a machine with access to GitHub.com use the `./codeql-action-sync pull` command to download a copy of the CodeQL Action and bundles to a local folder.
 
 **Optional Arguments:**
 * `--cache-dir` - The directory in which to store data downloaded from GitHub.com. If not specified a directory next to the sync tool will be used.
+* `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. If provided, it should have the `public_repo` scope.
 
 Next copy the sync tool and cache directory to another machine which has access to GitHub Enterprise Server.
 

cmd/pull.go

@@ -13,12 +13,16 @@ var pullCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		version.LogVersion()
 		cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
-		return pull.Pull(cmd.Context(), cacheDirectory)
+		return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken)
 	},
 }
 
-type pullFlagFields struct{}
+type pullFlagFields struct {
+	sourceToken string
+}
 
 var pullFlags = pullFlagFields{}
 
-func (f *pullFlagFields) Init(cmd *cobra.Command) {}
+func (f *pullFlagFields) Init(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&f.sourceToken, "source-token", "", "A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting.")
+}

cmd/sync.go

@@ -14,7 +14,7 @@ var syncCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		version.LogVersion()
 		cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
-		err := pull.Pull(cmd.Context(), cacheDirectory)
+		err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken)
 		if err != nil {
 			return err
 		}

internal/pull/pull.go

@@ -13,13 +13,15 @@ import (
 
 	"github.com/github/codeql-action-sync/internal/actionconfiguration"
 	"github.com/mitchellh/ioprogress"
+	"golang.org/x/oauth2"
 
 	"github.com/github/codeql-action-sync/internal/cachedirectory"
 	"github.com/github/codeql-action-sync/internal/version"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/google/go-github/v32/github"
 	"github.com/pkg/errors"
 )
@@ -37,6 +39,7 @@ type pullService struct {
 	cacheDirectory     cachedirectory.CacheDirectory
 	gitCloneURL        string
 	githubDotComClient *github.Client
+	sourceToken        string
 }
 
 func (pullService *pullService) pullGit(fresh bool) error {
@@ -78,6 +81,14 @@ func (pullService *pullService) pullGit(fresh bool) error {
 		return errors.Wrap(err, "Error setting Git remote.")
 	}
 
+	var credentials *githttp.BasicAuth
+	if pullService.sourceToken != "" {
+		credentials = &githttp.BasicAuth{
+			Username: "x-access-token",
+			Password: pullService.sourceToken,
+		}
+	}
+
 	err = localRepository.FetchContext(pullService.ctx, &git.FetchOptions{
 		RemoteName: git.DefaultRemoteName,
 		RefSpecs: []config.RefSpec{
@@ -87,6 +98,7 @@ func (pullService *pullService) pullGit(fresh bool) error {
 		Progress: os.Stderr,
 		Tags:     git.NoTags,
 		Force:    true,
+		Auth:     credentials,
 	})
 	if err != nil && err != git.NoErrAlreadyUpToDate {
 		return errors.Wrap(err, "Error doing Git fetch.")
@@ -220,17 +232,26 @@ func (pullService *pullService) pullReleases() error {
 	return nil
 }
 
-func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory) error {
+func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string) error {
 	err := cacheDirectory.CheckOrCreateVersionFile(true, version.Version())
 	if err != nil {
 		return err
 	}
 
+	var tokenClient *http.Client
+	if sourceToken != "" {
+		tokenSource := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: sourceToken},
+		)
+		tokenClient = oauth2.NewClient(ctx, tokenSource)
+	}
+
 	pullService := pullService{
 		ctx:                ctx,
 		cacheDirectory:     cacheDirectory,
 		gitCloneURL:        sourceURL,
-		githubDotComClient: github.NewClient(nil),
+		githubDotComClient: github.NewClient(tokenClient),
+		sourceToken:        sourceToken,
 	}
 
 	err = pullService.pullGit(false)