Merge pull request #40 from github/pruning

Implement reference pruning on push.

Author: chrisgavin

internal/push/push.go

@@ -14,6 +14,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/go-git/go-git/v5/plumbing"
+
 	"github.com/github/codeql-action-sync/internal/githubapiutil"
 
 	log "github.com/sirupsen/logrus"
@@ -22,6 +24,7 @@ import (
 	"github.com/github/codeql-action-sync/internal/version"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/google/go-github/v32/github"
 	"github.com/mitchellh/ioprogress"
@@ -142,35 +145,53 @@ func (pushService *pushService) pushGit(repository *github.Repository, initialPu
 	}
 
 	refSpecBatches := [][]config.RefSpec{}
+	remoteReferences, err := remote.List(&git.ListOptions{Auth: credentials})
+	if err != nil && err != transport.ErrEmptyRemoteRepository {
+		return errors.Wrap(err, "Error listing remote references.")
+	}
+	deleteRefSpecs := []config.RefSpec{}
+	for _, remoteReference := range remoteReferences {
+		_, err := gitRepository.Reference(remoteReference.Name(), false)
+		if err != nil && err != plumbing.ErrReferenceNotFound {
+			return errors.Wrapf(err, "Error finding local reference %s.", remoteReference.Name())
+		}
+		if err == plumbing.ErrReferenceNotFound {
+			deleteRefSpecs = append(deleteRefSpecs, config.RefSpec(":"+remoteReference.Name().String()))
+		}
+	}
+	refSpecBatches = append(refSpecBatches, deleteRefSpecs)
+
 	if initialPush {
 		releasePathStats, err := ioutil.ReadDir(pushService.cacheDirectory.ReleasesPath())
 		if err != nil {
 			return errors.Wrap(err, "Error reading releases.")
 		}
-		refSpecBatches = append(refSpecBatches, []config.RefSpec{})
+		initialRefSpecs := []config.RefSpec{}
 		for _, releasePathStat := range releasePathStats {
-			refSpecBatches[0] = append(refSpecBatches[0], config.RefSpec("+refs/tags/"+releasePathStat.Name()+":refs/tags/"+releasePathStat.Name()))
+			initialRefSpecs = append(initialRefSpecs, config.RefSpec("+refs/tags/"+releasePathStat.Name()+":refs/tags/"+releasePathStat.Name()))
 		}
+		refSpecBatches = append(refSpecBatches, initialRefSpecs)
 	} else {
 		// We've got to push `main` on its own, so that it will be made the default branch if the repository has just been created. We then push everything else afterwards.
-		refSpecBatches = [][]config.RefSpec{
+		refSpecBatches = append(refSpecBatches,
 			[]config.RefSpec{
 				config.RefSpec("+refs/heads/main:refs/heads/main"),
 			},
 			[]config.RefSpec{
 				config.RefSpec("+refs/*:refs/*"),
 			},
-		}
+		)
 	}
 	for _, refSpecs := range refSpecBatches {
-		err = remote.PushContext(pushService.ctx, &git.PushOptions{
-			RefSpecs: refSpecs,
-			Auth:     credentials,
-			Progress: os.Stderr,
-			Force:    true,
-		})
-		if err != nil && errors.Cause(err) != git.NoErrAlreadyUpToDate {
-			return errors.Wrap(err, "Error pushing Action to GitHub Enterprise Server.")
+		if len(refSpecs) != 0 {
+			err = remote.PushContext(pushService.ctx, &git.PushOptions{
+				RefSpecs: refSpecs,
+				Auth:     credentials,
+				Progress: os.Stderr,
+			})
+			if err != nil && errors.Cause(err) != git.NoErrAlreadyUpToDate {
+				return errors.Wrap(err, "Error pushing Action to GitHub Enterprise Server.")
+			}
 		}
 	}
 

internal/push/push_test.go

@@ -150,6 +150,35 @@ func TestPushGit(t *testing.T) {
 		"bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch",
 		"bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too",
 		"26936381e619a01122ea33993e3cebc474496805 refs/tags/v2",
+		"26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning",
+	})
+
+	pushService = getTestPushService(t, "./push_test/action-cache-modified/", "")
+	err = pushService.pushGit(&repository, true)
+	require.NoError(t, err)
+	test.CheckExpectedReferencesInRepository(t, destinationPath, []string{
+		"26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200101",
+		"26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200630",
+		"b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main",
+		"26936381e619a01122ea33993e3cebc474496805 refs/heads/v1",
+		"e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3",
+		"bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch",
+		"bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too",
+		"26936381e619a01122ea33993e3cebc474496805 refs/tags/v2",
+	})
+
+	err = pushService.pushGit(&repository, false)
+	require.NoError(t, err)
+	test.CheckExpectedReferencesInRepository(t, destinationPath, []string{
+		"26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200101",
+		"26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200630",
+		"b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main",
+		"26936381e619a01122ea33993e3cebc474496805 refs/heads/v1",
+		"e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3",
+		"bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch",
+		"bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too",
+		"26936381e619a01122ea33993e3cebc474496805 refs/tags/v2",
+		"26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning/because-it-now-has-this-extra-bit",
 	})
 }
 

internal/push/push_test/action-cache-initial/git/packed-refs

@@ -1,4 +1,5 @@
 # pack-refs with: peeled fully-peeled sorted 
+26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning
 b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main
 26936381e619a01122ea33993e3cebc474496805 refs/heads/v1
 e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3

internal/push/push_test/action-cache-modified/git/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/main

internal/push/push_test/action-cache-modified/git/config

@@ -0,0 +1,4 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true

internal/push/push_test/action-cache-modified/git/objects/26/936381e619a01122ea33993e3cebc474496805

@@ -0,0 +1,10 @@
+# pack-refs with: peeled fully-peeled sorted 
+26936381e619a01122ea33993e3cebc474496805 refs/heads/a-ref-that-will-need-pruning/because-it-now-has-this-extra-bit
+b9f01aa2c50f49898d4c7845a66be8824499fe9d refs/heads/main
+26936381e619a01122ea33993e3cebc474496805 refs/heads/v1
+e529a54fad10a936308b2220e05f7f00757f8e7c refs/heads/v3
+bd82b85707bc13904e3526517677039d4da4a9bb refs/heads/very-ignored-branch
+bd82b85707bc13904e3526517677039d4da4a9bb refs/tags/an-ignored-tag-too
+26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200101
+26936381e619a01122ea33993e3cebc474496805 refs/tags/codeql-bundle-20200630
+26936381e619a01122ea33993e3cebc474496805 refs/tags/v2

internal/push/push_test/action-cache-modified/git/objects/28/509716746a9e77e3f752e9df740cb443bf9960

@@ -0,0 +1 @@
+This isn't really a CodeQL bundle either!

internal/push/push_test/action-cache-modified/git/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904

@@ -0,0 +1,4 @@
+{
+	"tag_name": "codeql-bundle-20200101",
+	"target_commitish": "main"
+}

internal/push/push_test/action-cache-modified/git/objects/61/8dc8ffee7af67e4e2dfdbe416c53cca6099fab

@@ -0,0 +1 @@
+This isn't really a CodeQL bundle!

internal/push/push_test/action-cache-modified/git/objects/80/8d4d2d6b7aa39089b99ca0d89f1fc6a2bc63ec

@@ -0,0 +1,4 @@
+{
+	"tag_name": "codeql-bundle-20200630",
+	"target_commitish": "main"
+}