Skip to content

Commit 9a70a12

Browse files
lunnylunny
authored
Merge db.Iterate and IterateObjects (#21641)
These two functions are similiar, merge them.
1 parent 4ae3f76 commit 9a70a12

File tree

12 files changed

+77
-51
lines changed

12 files changed

+77
-51
lines changed

‎cmd/migrate_storage.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -83,43 +83,43 @@ var CmdMigrateStorage = cli.Command{
8383
}
8484

8585
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
86-
return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error {
86+
return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error {
8787
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
8888
return err
8989
})
9090
}
9191

9292
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
93-
return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error {
93+
return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error {
9494
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
9595
return err
9696
})
9797
}
9898

9999
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
100-
return db.IterateObjects(ctx, func(user *user_model.User) error {
100+
return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error {
101101
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
102102
return err
103103
})
104104
}
105105

106106
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
107-
return db.IterateObjects(ctx, func(repo *repo_model.Repository) error {
107+
return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error {
108108
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
109109
return err
110110
})
111111
}
112112

113113
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
114-
return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error {
114+
return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error {
115115
p := archiver.RelativePath()
116116
_, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
117117
return err
118118
})
119119
}
120120

121121
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
122-
return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error {
122+
return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error {
123123
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
124124
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
125125
return err

‎models/db/context.go

-10
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,6 @@ import (
88
"context"
99
"database/sql"
1010

11-
"code.gitea.io/gitea/modules/setting"
12-
13-
"xorm.io/builder"
1411
"xorm.io/xorm/schemas"
1512
)
1613

@@ -121,13 +118,6 @@ func WithTx(f func(ctx context.Context) error, stdCtx ...context.Context) error
121118
return sess.Commit()
122119
}
123120

124-
// Iterate iterates the databases and doing something
125-
func Iterate(ctx context.Context, tableBean interface{}, cond builder.Cond, fun func(idx int, bean interface{}) error) error {
126-
return GetEngine(ctx).Where(cond).
127-
BufferSize(setting.Database.IterateBufferSize).
128-
Iterate(tableBean, fun)
129-
}
130-
131121
// Insert inserts records into database
132122
func Insert(ctx context.Context, beans ...interface{}) error {
133123
_, err := GetEngine(ctx).Insert(beans...)

‎models/db/iterate.go

+13-8
Original file line numberDiff line numberDiff line change
@@ -8,25 +8,30 @@ import (
88
"context"
99

1010
"code.gitea.io/gitea/modules/setting"
11+
12+
"xorm.io/builder"
1113
)
1214

13-
// IterateObjects iterate all the Bean object
14-
func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error {
15+
// Iterate iterate all the Bean object
16+
func Iterate[Bean any](ctx context.Context, cond builder.Cond, f func(ctx context.Context, bean *Bean) error) error {
1517
var start int
1618
batchSize := setting.Database.IterateBufferSize
1719
sess := GetEngine(ctx)
1820
for {
19-
repos := make([]*Object, 0, batchSize)
20-
if err := sess.Limit(batchSize, start).Find(&repos); err != nil {
21+
beans := make([]*Bean, 0, batchSize)
22+
if cond != nil {
23+
sess = sess.Where(cond)
24+
}
25+
if err := sess.Limit(batchSize, start).Find(&beans); err != nil {
2126
return err
2227
}
23-
if len(repos) == 0 {
28+
if len(beans) == 0 {
2429
return nil
2530
}
26-
start += len(repos)
31+
start += len(beans)
2732

28-
for _, repo := range repos {
29-
if err := f(repo); err != nil {
33+
for _, bean := range beans {
34+
if err := f(ctx, bean); err != nil {
3035
return err
3136
}
3237
}

‎models/db/iterate_test.go

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package db_test
6+
7+
import (
8+
"context"
9+
"testing"
10+
11+
"code.gitea.io/gitea/models/db"
12+
repo_model "code.gitea.io/gitea/models/repo"
13+
"code.gitea.io/gitea/models/unittest"
14+
15+
"github.com/stretchr/testify/assert"
16+
)
17+
18+
func TestIterate(t *testing.T) {
19+
assert.NoError(t, unittest.PrepareTestDatabase())
20+
xe := unittest.GetXORMEngine()
21+
assert.NoError(t, xe.Sync(&repo_model.RepoUnit{}))
22+
23+
var repoCnt int
24+
err := db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repo *repo_model.RepoUnit) error {
25+
repoCnt++
26+
return nil
27+
})
28+
assert.NoError(t, err)
29+
assert.EqualValues(t, 79, repoCnt)
30+
31+
err = db.Iterate(db.DefaultContext, nil, func(ctx context.Context, repoUnit *repo_model.RepoUnit) error {
32+
reopUnit2 := repo_model.RepoUnit{ID: repoUnit.ID}
33+
has, err := db.GetByBean(ctx, &reopUnit2)
34+
if err != nil {
35+
return err
36+
} else if !has {
37+
return db.ErrNotExist{Resource: "repo_unit", ID: repoUnit.ID}
38+
}
39+
assert.EqualValues(t, repoUnit.RepoID, repoUnit.RepoID)
40+
assert.EqualValues(t, repoUnit.CreatedUnix, repoUnit.CreatedUnix)
41+
return nil
42+
})
43+
assert.NoError(t, err)
44+
}

‎modules/doctor/breaking.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,9 @@ import (
1818
func iterateUserAccounts(ctx context.Context, each func(*user.User) error) error {
1919
err := db.Iterate(
2020
ctx,
21-
new(user.User),
2221
builder.Gt{"id": 0},
23-
func(idx int, bean interface{}) error {
24-
return each(bean.(*user.User))
22+
func(ctx context.Context, bean *user.User) error {
23+
return each(bean)
2524
},
2625
)
2726
return err

‎modules/doctor/fix16961.go

+1-4
Original file line numberDiff line numberDiff line change
@@ -269,13 +269,10 @@ func fixBrokenRepoUnits16961(ctx context.Context, logger log.Logger, autofix boo
269269

270270
err := db.Iterate(
271271
ctx,
272-
new(RepoUnit),
273272
builder.Gt{
274273
"id": 0,
275274
},
276-
func(idx int, bean interface{}) error {
277-
unit := bean.(*RepoUnit)
278-
275+
func(ctx context.Context, unit *RepoUnit) error {
279276
bs := unit.Config
280277
repoUnit := &repo_model.RepoUnit{
281278
ID: unit.ID,

‎modules/doctor/mergebase.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,9 @@ import (
2121
func iteratePRs(ctx context.Context, repo *repo_model.Repository, each func(*repo_model.Repository, *issues_model.PullRequest) error) error {
2222
return db.Iterate(
2323
ctx,
24-
new(issues_model.PullRequest),
2524
builder.Eq{"base_repo_id": repo.ID},
26-
func(idx int, bean interface{}) error {
27-
return each(repo, bean.(*issues_model.PullRequest))
25+
func(ctx context.Context, bean *issues_model.PullRequest) error {
26+
return each(repo, bean)
2827
},
2928
)
3029
}

‎modules/doctor/misc.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,9 @@ import (
3030
func iterateRepositories(ctx context.Context, each func(*repo_model.Repository) error) error {
3131
err := db.Iterate(
3232
ctx,
33-
new(repo_model.Repository),
3433
builder.Gt{"id": 0},
35-
func(idx int, bean interface{}) error {
36-
return each(bean.(*repo_model.Repository))
34+
func(ctx context.Context, bean *repo_model.Repository) error {
35+
return each(bean)
3736
},
3837
)
3938
return err

‎routers/private/mail.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package private
66

77
import (
8+
stdCtx "context"
89
"fmt"
910
"net/http"
1011
"strconv"
@@ -60,7 +61,7 @@ func SendEmail(ctx *context.PrivateContext) {
6061
}
6162
}
6263
} else {
63-
err := db.IterateObjects(ctx, func(user *user_model.User) error {
64+
err := db.Iterate(ctx, nil, func(ctx stdCtx.Context, user *user_model.User) error {
6465
if len(user.Email) > 0 && user.IsActive {
6566
emails = append(emails, user.Email)
6667
}

‎services/repository/avatar.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error {
9696

9797
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
9898
func RemoveRandomAvatars(ctx context.Context) error {
99-
return db.IterateObjects(ctx, func(repository *repo_model.Repository) error {
99+
return db.Iterate(ctx, nil, func(ctx context.Context, repository *repo_model.Repository) error {
100100
select {
101101
case <-ctx.Done():
102102
return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())

‎services/repository/check.go

+3-9
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,8 @@ func GitFsck(ctx context.Context, timeout time.Duration, args []git.CmdArg) erro
2929

3030
if err := db.Iterate(
3131
ctx,
32-
new(repo_model.Repository),
3332
builder.Expr("id>0 AND is_fsck_enabled=?", true),
34-
func(idx int, bean interface{}) error {
35-
repo := bean.(*repo_model.Repository)
33+
func(ctx context.Context, repo *repo_model.Repository) error {
3634
select {
3735
case <-ctx.Done():
3836
return db.ErrCancelledf("before fsck of %s", repo.FullName())
@@ -64,10 +62,8 @@ func GitGcRepos(ctx context.Context, timeout time.Duration, args ...git.CmdArg)
6462

6563
if err := db.Iterate(
6664
ctx,
67-
new(repo_model.Repository),
6865
builder.Gt{"id": 0},
69-
func(idx int, bean interface{}) error {
70-
repo := bean.(*repo_model.Repository)
66+
func(ctx context.Context, repo *repo_model.Repository) error {
7167
select {
7268
case <-ctx.Done():
7369
return db.ErrCancelledf("before GC of %s", repo.FullName())
@@ -113,10 +109,8 @@ func gatherMissingRepoRecords(ctx context.Context) ([]*repo_model.Repository, er
113109
repos := make([]*repo_model.Repository, 0, 10)
114110
if err := db.Iterate(
115111
ctx,
116-
new(repo_model.Repository),
117112
builder.Gt{"id": 0},
118-
func(idx int, bean interface{}) error {
119-
repo := bean.(*repo_model.Repository)
113+
func(ctx context.Context, repo *repo_model.Repository) error {
120114
select {
121115
case <-ctx.Done():
122116
return db.ErrCancelledf("during gathering missing repo records before checking %s", repo.FullName())

‎services/repository/hooks.go

+1-3
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@ func SyncRepositoryHooks(ctx context.Context) error {
2525

2626
if err := db.Iterate(
2727
ctx,
28-
new(repo_model.Repository),
2928
builder.Gt{"id": 0},
30-
func(idx int, bean interface{}) error {
31-
repo := bean.(*repo_model.Repository)
29+
func(ctx context.Context, repo *repo_model.Repository) error {
3230
select {
3331
case <-ctx.Done():
3432
return db.ErrCancelledf("before sync repository hooks for %s", repo.FullName())

0 commit comments

Comments
 (0)