From ae55d699dc3e2b32389ba21ef3d53a98fd5dff59 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 10 Aug 2023 14:51:59 +0800 Subject: [PATCH 01/13] Allow get release download files and lfs files with oauth2 token format --- services/auth/oauth2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index 0dd7a12d2c436..a8612912023d6 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -126,7 +126,8 @@ func (o *OAuth2) userIDFromToken(tokenSHA string, store DataStore) int64 { // If verification is successful returns an existing user object. // Returns nil if verification fails. func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { - if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) { + if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) && + !isGitRawReleaseOrLFSPath(req) { return nil, nil } From f91dd42ce283198759b6209c53154d276a344c23 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Aug 2023 00:17:21 +0800 Subject: [PATCH 02/13] Add a test to download a release attachment from a private repository --- models/fixtures/attachment.yml | 13 +++++++++++++ models/fixtures/release.yml | 14 ++++++++++++++ .../user2/repo2.git/refs/tags/v1.1 | 1 + tests/integration/release_test.go | 14 ++++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 tests/gitea-repositories-meta/user2/repo2.git/refs/tags/v1.1 diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index 9ad43fa2b7eb6..872cb71353901 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -140,3 +140,16 @@ download_count: 0 size: 0 created_unix: 946684800 + +- + id: 12 + uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 + repo_id: 40 + issue_id: 0 + release_id: 11 + uploader_id: 2 + comment_id: 0 + name: README.md + download_count: 0 + size: 0 + created_unix: 946684800 diff --git a/models/fixtures/release.yml b/models/fixtures/release.yml index 4ed7df440dbd3..372a79509f0a8 100644 --- a/models/fixtures/release.yml +++ b/models/fixtures/release.yml @@ -136,3 +136,17 @@ is_prerelease: false is_tag: false created_unix: 946684803 + +- id: 11 + repo_id: 2 + publisher_id: 2 + tag_name: "v1.1" + lower_tag_name: "v1.1" + target: "" + title: "v1.1" + sha1: "205ac761f3326a7ebe416e8673760016450b5cec" + num_commits: 2 + is_draft: false + is_prerelease: false + is_tag: false + created_unix: 946684803 diff --git a/tests/gitea-repositories-meta/user2/repo2.git/refs/tags/v1.1 b/tests/gitea-repositories-meta/user2/repo2.git/refs/tags/v1.1 new file mode 100644 index 0000000000000..334d09ca02155 --- /dev/null +++ b/tests/gitea-repositories-meta/user2/repo2.git/refs/tags/v1.1 @@ -0,0 +1 @@ +1032bbf17fbc0d9c95bb5418dabe8f8c99278700 diff --git a/tests/integration/release_test.go b/tests/integration/release_test.go index 8de761ea6cb6e..f48c4395e8e15 100644 --- a/tests/integration/release_test.go +++ b/tests/integration/release_test.go @@ -239,3 +239,17 @@ func TestViewTagsList(t *testing.T) { assert.EqualValues(t, []string{"v1.0", "delete-tag", "v1.1"}, tagNames) } + +func TestDownloadReleaseAttachment(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) + + url := repo.Link() + "/releases/download/v1.1/README.md" + + req := NewRequest(t, "GET", url) + MakeRequest(t, req, http.StatusNotFound) + + session := loginUser(t, "user2") + session.MakeRequest(t, req, http.StatusOK) +} From 494834e50a11472cd06f0a7c60134e75e1b7e178 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 11 Aug 2023 17:01:10 +0800 Subject: [PATCH 03/13] Fix test fixture --- models/fixtures/attachment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index 872cb71353901..12712478b42f0 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -143,7 +143,7 @@ - id: 12 - uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 + uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 repo_id: 40 issue_id: 0 release_id: 11 From 5ce2395a015807bafebd326358076444307adf8d Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 11 Sep 2023 11:53:58 +0800 Subject: [PATCH 04/13] narraw the change scope --- services/auth/oauth2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index 5ccbe6233fa51..45b4078630e9a 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -126,7 +126,7 @@ func (o *OAuth2) userIDFromToken(tokenSHA string, store DataStore) int64 { // Returns nil if verification fails. func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) && - !isGitRawReleaseOrLFSPath(req) { + !gitRawReleasePathRe.MatchString(req.URL.Path) { return nil, nil } From 5104af1d6aac465bca50b851a987a581e79fb872 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 28 Sep 2023 16:14:49 +0800 Subject: [PATCH 05/13] Fix test --- models/fixtures/attachment.yml | 2 +- routers/web/web.go | 8 +++----- tests/integration/release_test.go | 1 + tests/test_utils.go | 4 ++++ .../attachments/a/0/a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 | 1 + 5 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 tests/testdata/data/attachments/a/0/a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 diff --git a/models/fixtures/attachment.yml b/models/fixtures/attachment.yml index 12712478b42f0..7882d8bff2089 100644 --- a/models/fixtures/attachment.yml +++ b/models/fixtures/attachment.yml @@ -144,7 +144,7 @@ - id: 12 uuid: a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 - repo_id: 40 + repo_id: 2 issue_id: 0 release_id: 11 uploader_id: 2 diff --git a/routers/web/web.go b/routers/web/web.go index 99862505b48a1..215483872670d 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -978,9 +978,6 @@ func registerRoutes(m *web.Route) { }, reqUnitAccess(unit.TypeCode, perm.AccessModeRead, false)) }, ignSignIn, context_service.UserAssignmentWeb(), context.OrgAssignment()) // for "/{username}/-" (packages, projects, code) - // ***** Release Attachment Download without Signin - m.Get("/{username}/{reponame}/releases/download/{vTag}/{fileName}", ignSignIn, context.RepoAssignment, repo.MustBeNotEmpty, repo.RedirectDownload) - m.Group("/{username}/{reponame}", func() { m.Group("/settings", func() { m.Group("", func() { @@ -1240,8 +1237,9 @@ func registerRoutes(m *web.Route) { m.Get(".rss", feedEnabled, repo.ReleasesFeedRSS) m.Get(".atom", feedEnabled, repo.ReleasesFeedAtom) }, ctxDataSet("EnableFeed", setting.Other.EnableFeed), - repo.MustBeNotEmpty, reqRepoReleaseReader, context.RepoRefByType(context.RepoRefTag, true)) - m.Get("/releases/attachments/{uuid}", repo.MustBeNotEmpty, reqRepoReleaseReader, repo.GetAttachment) + repo.MustBeNotEmpty, context.RepoRefByType(context.RepoRefTag, true)) + m.Get("/releases/attachments/{uuid}", repo.MustBeNotEmpty, repo.GetAttachment) + m.Get("/releases/download/{vTag}/{fileName}", repo.MustBeNotEmpty, repo.RedirectDownload) m.Group("/releases", func() { m.Get("/new", repo.NewRelease) m.Post("/new", web.Bind(forms.NewReleaseForm{}), repo.NewReleasePost) diff --git a/tests/integration/release_test.go b/tests/integration/release_test.go index f48c4395e8e15..985dfe3ab3761 100644 --- a/tests/integration/release_test.go +++ b/tests/integration/release_test.go @@ -250,6 +250,7 @@ func TestDownloadReleaseAttachment(t *testing.T) { req := NewRequest(t, "GET", url) MakeRequest(t, req, http.StatusNotFound) + req = NewRequest(t, "GET", url) session := loginUser(t, "user2") session.MakeRequest(t, req, http.StatusOK) } diff --git a/tests/test_utils.go b/tests/test_utils.go index 089b4dce1cc1d..c47f5137484f6 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -213,6 +213,10 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { } } + // prepare attachments directory and files + assert.NoError(t, util.RemoveAll(setting.Attachment.Storage.Path)) + assert.NoError(t, unittest.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), setting.Attachment.Storage.Path)) + // load LFS object fixtures // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) lfsFixtures, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{ diff --git a/tests/testdata/data/attachments/a/0/a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 b/tests/testdata/data/attachments/a/0/a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 new file mode 100644 index 0000000000000..96fc98807f411 --- /dev/null +++ b/tests/testdata/data/attachments/a/0/a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a22 @@ -0,0 +1 @@ +# This is a release README From 3bc8a024d16b1acd164ad644bcaf86cd817b4813 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 28 Sep 2023 16:23:03 +0800 Subject: [PATCH 06/13] Add comment --- services/auth/oauth2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index 9e6c3cd2fdf5b..38b705cc5b8f7 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -125,6 +125,7 @@ func (o *OAuth2) userIDFromToken(ctx context.Context, tokenSHA string, store Dat // If verification is successful returns an existing user object. // Returns nil if verification fails. func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { + // These paths are not API paths, but we still want to check for tokens because they maybe in the API returned URLs if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) && !gitRawReleasePathRe.MatchString(req.URL.Path) { return nil, nil From dba042c05afeb5a4cd3589a42df516a09deae704 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 28 Sep 2023 21:39:36 +0800 Subject: [PATCH 07/13] Fix test for postgres which using minio --- tests/test_utils.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.go b/tests/test_utils.go index c47f5137484f6..bbe41a5c4bdfa 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -215,7 +215,15 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { // prepare attachments directory and files assert.NoError(t, util.RemoveAll(setting.Attachment.Storage.Path)) - assert.NoError(t, unittest.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), setting.Attachment.Storage.Path)) + if setting.Attachment.Storage.Type == setting.LocalStorageType { + assert.NoError(t, unittest.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), setting.Attachment.Storage.Path)) + } else { + assert.EqualValues(t, setting.MinioStorageType, setting.Attachment.Storage.Type) + s, err := storage.NewLocalStorage(context.Background(), setting.Attachment.Storage) + assert.NoError(t, err) + _, err = storage.Copy(storage.Attachments, "", s, "") + assert.NoError(t, err) + } // load LFS object fixtures // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) From 6a085473c14ae0f52503ce32e94216e89525e56a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 28 Sep 2023 22:29:23 +0800 Subject: [PATCH 08/13] Fix test --- tests/test_utils.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.go b/tests/test_utils.go index bbe41a5c4bdfa..d358663c49d0e 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -215,11 +215,14 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { // prepare attachments directory and files assert.NoError(t, util.RemoveAll(setting.Attachment.Storage.Path)) + attachPath := filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments") if setting.Attachment.Storage.Type == setting.LocalStorageType { - assert.NoError(t, unittest.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), setting.Attachment.Storage.Path)) + assert.NoError(t, unittest.CopyDir(attachPath, setting.Attachment.Storage.Path)) } else { assert.EqualValues(t, setting.MinioStorageType, setting.Attachment.Storage.Type) - s, err := storage.NewLocalStorage(context.Background(), setting.Attachment.Storage) + s, err := storage.NewLocalStorage(context.Background(), &setting.Storage{ + Path: attachPath, + }) assert.NoError(t, err) _, err = storage.Copy(storage.Attachments, "", s, "") assert.NoError(t, err) From ea6b3f29de42d6348a2ec5257a04591a9b67d48e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 29 Sep 2023 09:39:45 +0800 Subject: [PATCH 09/13] Fix test --- tests/test_utils.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.go b/tests/test_utils.go index d358663c49d0e..66083953f35e7 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -224,8 +224,10 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { Path: attachPath, }) assert.NoError(t, err) - _, err = storage.Copy(storage.Attachments, "", s, "") - assert.NoError(t, err) + assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error { + _, err = storage.Copy(storage.Attachments, p, s, p) + return err + })) } // load LFS object fixtures From 4d930a2c08b242873505e3d4c475c13e72f3ad00 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 29 Sep 2023 10:58:32 +0800 Subject: [PATCH 10/13] Make code simple --- tests/test_utils.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/test_utils.go b/tests/test_utils.go index 66083953f35e7..fdff866085a84 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -214,21 +214,16 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { } // prepare attachments directory and files - assert.NoError(t, util.RemoveAll(setting.Attachment.Storage.Path)) - attachPath := filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments") - if setting.Attachment.Storage.Type == setting.LocalStorageType { - assert.NoError(t, unittest.CopyDir(attachPath, setting.Attachment.Storage.Path)) - } else { - assert.EqualValues(t, setting.MinioStorageType, setting.Attachment.Storage.Type) - s, err := storage.NewLocalStorage(context.Background(), &setting.Storage{ - Path: attachPath, - }) - assert.NoError(t, err) - assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error { - _, err = storage.Copy(storage.Attachments, p, s, p) - return err - })) - } + assert.NoError(t, storage.Clean(storage.Attachments)) + + s, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{ + Path: filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), + }) + assert.NoError(t, err) + assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error { + _, err = storage.Copy(storage.Attachments, p, s, p) + return err + })) // load LFS object fixtures // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) From 180a0f81d7458d19179d10da60384b2abfa66ae9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 1 Oct 2023 10:14:58 +0800 Subject: [PATCH 11/13] Use common function --- services/auth/oauth2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index 38b705cc5b8f7..e2dc151c54ffb 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -127,7 +127,7 @@ func (o *OAuth2) userIDFromToken(ctx context.Context, tokenSHA string, store Dat func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { // These paths are not API paths, but we still want to check for tokens because they maybe in the API returned URLs if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) && - !gitRawReleasePathRe.MatchString(req.URL.Path) { + !isGitRawReleaseOrLFSPath(req) { return nil, nil } From a9cf69c170abaadb4c9a61128011abc1449677a2 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 1 Oct 2023 10:36:44 +0800 Subject: [PATCH 12/13] do the attachment directories copy only for the related tests --- tests/integration/release_test.go | 2 ++ tests/test_utils.go | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/integration/release_test.go b/tests/integration/release_test.go index 985dfe3ab3761..42d0d00e78698 100644 --- a/tests/integration/release_test.go +++ b/tests/integration/release_test.go @@ -243,6 +243,8 @@ func TestViewTagsList(t *testing.T) { func TestDownloadReleaseAttachment(t *testing.T) { defer tests.PrepareTestEnv(t)() + tests.PrepareAttachmentsStorage(t) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 2}) url := repo.Link() + "/releases/download/v1.1/README.md" diff --git a/tests/test_utils.go b/tests/test_utils.go index fdff866085a84..50049e73f01c9 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -179,6 +179,20 @@ func InitTest(requireGitea bool) { routers.InitWebInstalled(graceful.GetManager().HammerContext()) } +func PrepareAttachmentsStorage(t testing.TB) { + // prepare attachments directory and files + assert.NoError(t, storage.Clean(storage.Attachments)) + + s, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{ + Path: filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), + }) + assert.NoError(t, err) + assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error { + _, err = storage.Copy(storage.Attachments, p, s, p) + return err + })) +} + func PrepareTestEnv(t testing.TB, skip ...int) func() { t.Helper() ourSkip := 1 @@ -213,18 +227,6 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { } } - // prepare attachments directory and files - assert.NoError(t, storage.Clean(storage.Attachments)) - - s, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{ - Path: filepath.Join(filepath.Dir(setting.AppPath), "tests", "testdata", "data", "attachments"), - }) - assert.NoError(t, err) - assert.NoError(t, s.IterateObjects("", func(p string, obj storage.Object) error { - _, err = storage.Copy(storage.Attachments, p, s, p) - return err - })) - // load LFS object fixtures // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) lfsFixtures, err := storage.NewStorage(setting.LocalStorageType, &setting.Storage{ From 95d6c26c91ea12fb7684768ac050cfb319c1a068 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 1 Oct 2023 15:13:33 +0800 Subject: [PATCH 13/13] Fix test --- services/auth/oauth2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go index e2dc151c54ffb..38b705cc5b8f7 100644 --- a/services/auth/oauth2.go +++ b/services/auth/oauth2.go @@ -127,7 +127,7 @@ func (o *OAuth2) userIDFromToken(ctx context.Context, tokenSHA string, store Dat func (o *OAuth2) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { // These paths are not API paths, but we still want to check for tokens because they maybe in the API returned URLs if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) && - !isGitRawReleaseOrLFSPath(req) { + !gitRawReleasePathRe.MatchString(req.URL.Path) { return nil, nil }