Skip to content

Commit d9db28a

Browse files
6543guillep2klunny
authored
Doctor check & fix db consistency (#11111) (#11676)
needed to fix issue as described in #10280 * rename check-db to check-db-version * add check-db-consistency: * find issues without existing repository * find pulls without existing issues * find tracked times without existing issues/pulls * find labels without repository or org reference Co-authored-by: guillep2k <[email protected]> Co-authored-by: Lunny Xiao <[email protected]> Co-authored-by: guillep2k <[email protected]> Co-authored-by: Lunny Xiao <[email protected]>
1 parent 5911e12 commit d9db28a

File tree

5 files changed

+275
-64
lines changed

5 files changed

+275
-64
lines changed

‎cmd/doctor.go

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,16 @@ var checklist = []check{
8585
},
8686
{
8787
title: "Check Database Version",
88-
name: "check-db",
88+
name: "check-db-version",
8989
isDefault: true,
9090
f: runDoctorCheckDBVersion,
91-
abortIfFailed: true,
91+
abortIfFailed: false,
92+
},
93+
{
94+
title: "Check consistency of database",
95+
name: "check-db-consistency",
96+
isDefault: false,
97+
f: runDoctorCheckDBConsistency,
9298
},
9399
{
94100
title: "Check if OpenSSH authorized_keys file is up-to-date",
@@ -495,3 +501,80 @@ func runDoctorScriptType(ctx *cli.Context) ([]string, error) {
495501
}
496502
return []string{fmt.Sprintf("ScriptType %s is on the current PATH at %s", setting.ScriptType, path)}, nil
497503
}
504+
505+
func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
506+
var results []string
507+
508+
// make sure DB version is uptodate
509+
if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
510+
return nil, fmt.Errorf("model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded")
511+
}
512+
513+
//find labels without existing repo or org
514+
count, err := models.CountOrphanedLabels()
515+
if err != nil {
516+
return nil, err
517+
}
518+
if count > 0 {
519+
if ctx.Bool("fix") {
520+
if err = models.DeleteOrphanedLabels(); err != nil {
521+
return nil, err
522+
}
523+
results = append(results, fmt.Sprintf("%d labels without existing repository/organisation deleted", count))
524+
} else {
525+
results = append(results, fmt.Sprintf("%d labels without existing repository/organisation", count))
526+
}
527+
}
528+
529+
//find issues without existing repository
530+
count, err = models.CountOrphanedIssues()
531+
if err != nil {
532+
return nil, err
533+
}
534+
if count > 0 {
535+
if ctx.Bool("fix") {
536+
if err = models.DeleteOrphanedIssues(); err != nil {
537+
return nil, err
538+
}
539+
results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count))
540+
} else {
541+
results = append(results, fmt.Sprintf("%d issues without existing repository", count))
542+
}
543+
}
544+
545+
//find pulls without existing issues
546+
count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id")
547+
if err != nil {
548+
return nil, err
549+
}
550+
if count > 0 {
551+
if ctx.Bool("fix") {
552+
if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil {
553+
return nil, err
554+
}
555+
results = append(results, fmt.Sprintf("%d pull requests without existing issue deleted", count))
556+
} else {
557+
results = append(results, fmt.Sprintf("%d pull requests without existing issue", count))
558+
}
559+
}
560+
561+
//find tracked times without existing issues/pulls
562+
count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id")
563+
if err != nil {
564+
return nil, err
565+
}
566+
if count > 0 {
567+
if ctx.Bool("fix") {
568+
if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil {
569+
return nil, err
570+
}
571+
results = append(results, fmt.Sprintf("%d tracked times without existing issue deleted", count))
572+
} else {
573+
results = append(results, fmt.Sprintf("%d tracked times without existing issue", count))
574+
}
575+
}
576+
577+
//ToDo: function to recalc all counters
578+
579+
return results, nil
580+
}

‎models/consistency.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"testing"
1111

1212
"github.com/stretchr/testify/assert"
13+
"xorm.io/builder"
1314
)
1415

1516
// consistencyCheckable a type that can be tested for database consistency
@@ -167,3 +168,118 @@ func (action *Action) checkForConsistency(t *testing.T) {
167168
repo := AssertExistsAndLoadBean(t, &Repository{ID: action.RepoID}).(*Repository)
168169
assert.Equal(t, repo.IsPrivate, action.IsPrivate, "action: %+v", action)
169170
}
171+
172+
// CountOrphanedLabels return count of labels witch are broken and not accessible via ui anymore
173+
func CountOrphanedLabels() (int64, error) {
174+
noref, err := x.Table("label").Where("repo_id=? AND org_id=?", 0, 0).Count("label.id")
175+
if err != nil {
176+
return 0, err
177+
}
178+
179+
norepo, err := x.Table("label").
180+
Join("LEFT", "repository", "label.repo_id=repository.id").
181+
Where(builder.IsNull{"repository.id"}).And(builder.Gt{"label.repo_id": 0}).
182+
Count("id")
183+
if err != nil {
184+
return 0, err
185+
}
186+
187+
noorg, err := x.Table("label").
188+
Join("LEFT", "`user`", "label.org_id=`user`.id").
189+
Where(builder.IsNull{"`user`.id"}).And(builder.Gt{"label.org_id": 0}).
190+
Count("id")
191+
if err != nil {
192+
return 0, err
193+
}
194+
195+
return noref + norepo + noorg, nil
196+
}
197+
198+
// DeleteOrphanedLabels delete labels witch are broken and not accessible via ui anymore
199+
func DeleteOrphanedLabels() error {
200+
// delete labels with no reference
201+
if _, err := x.Table("label").Where("repo_id=? AND org_id=?", 0, 0).Delete(new(Label)); err != nil {
202+
return err
203+
}
204+
205+
// delete labels with none existing repos
206+
if _, err := x.In("id", builder.Select("label.id").From("label").
207+
Join("LEFT", "repository", "label.repo_id=repository.id").
208+
Where(builder.IsNull{"repository.id"}).And(builder.Gt{"label.repo_id": 0})).
209+
Delete(Label{}); err != nil {
210+
return err
211+
}
212+
213+
// delete labels with none existing orgs
214+
if _, err := x.In("id", builder.Select("label.id").From("label").
215+
Join("LEFT", "`user`", "label.org_id=`user`.id").
216+
Where(builder.IsNull{"`user`.id"}).And(builder.Gt{"label.org_id": 0})).
217+
Delete(Label{}); err != nil {
218+
return err
219+
}
220+
221+
return nil
222+
}
223+
224+
// CountOrphanedIssues count issues without a repo
225+
func CountOrphanedIssues() (int64, error) {
226+
return x.Table("issue").
227+
Join("LEFT", "repository", "issue.repo_id=repository.id").
228+
Where(builder.IsNull{"repository.id"}).
229+
Count("id")
230+
}
231+
232+
// DeleteOrphanedIssues delete issues without a repo
233+
func DeleteOrphanedIssues() error {
234+
sess := x.NewSession()
235+
defer sess.Close()
236+
if err := sess.Begin(); err != nil {
237+
return err
238+
}
239+
240+
var ids []int64
241+
242+
if err := sess.Table("issue").Distinct("issue.repo_id").
243+
Join("LEFT", "repository", "issue.repo_id=repository.id").
244+
Where(builder.IsNull{"repository.id"}).GroupBy("issue.repo_id").
245+
Find(&ids); err != nil {
246+
return err
247+
}
248+
249+
var attachmentPaths []string
250+
for i := range ids {
251+
paths, err := deleteIssuesByRepoID(sess, ids[i])
252+
if err != nil {
253+
return err
254+
}
255+
attachmentPaths = append(attachmentPaths, paths...)
256+
}
257+
258+
if err := sess.Commit(); err != nil {
259+
return err
260+
}
261+
262+
// Remove issue attachment files.
263+
for i := range attachmentPaths {
264+
removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i])
265+
}
266+
return nil
267+
}
268+
269+
// CountOrphanedObjects count subjects with have no existing refobject anymore
270+
func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) {
271+
return x.Table("`"+subject+"`").
272+
Join("LEFT", refobject, joinCond).
273+
Where(builder.IsNull{"`" + refobject + "`.id"}).
274+
Count("id")
275+
}
276+
277+
// DeleteOrphanedObjects delete subjects with have no existing refobject anymore
278+
func DeleteOrphanedObjects(subject, refobject, joinCond string) error {
279+
_, err := x.In("id", builder.Select("`"+subject+"`.id").
280+
From("`"+subject+"`").
281+
Join("LEFT", "`"+refobject+"`", joinCond).
282+
Where(builder.IsNull{"`" + refobject + "`.id"})).
283+
Delete("`" + subject + "`")
284+
return err
285+
}

‎models/issue.go

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1916,3 +1916,70 @@ func UpdateReactionsMigrationsByType(gitServiceType structs.GitServiceType, orig
19161916
})
19171917
return err
19181918
}
1919+
1920+
func deleteIssuesByRepoID(sess Engine, repoID int64) (attachmentPaths []string, err error) {
1921+
deleteCond := builder.Select("id").From("issue").Where(builder.Eq{"issue.repo_id": repoID})
1922+
1923+
// Delete comments and attachments
1924+
if _, err = sess.In("issue_id", deleteCond).
1925+
Delete(&Comment{}); err != nil {
1926+
return
1927+
}
1928+
1929+
// Dependencies for issues in this repository
1930+
if _, err = sess.In("issue_id", deleteCond).
1931+
Delete(&IssueDependency{}); err != nil {
1932+
return
1933+
}
1934+
1935+
// Delete dependencies for issues in other repositories
1936+
if _, err = sess.In("dependency_id", deleteCond).
1937+
Delete(&IssueDependency{}); err != nil {
1938+
return
1939+
}
1940+
1941+
if _, err = sess.In("issue_id", deleteCond).
1942+
Delete(&IssueUser{}); err != nil {
1943+
return
1944+
}
1945+
1946+
if _, err = sess.In("issue_id", deleteCond).
1947+
Delete(&Reaction{}); err != nil {
1948+
return
1949+
}
1950+
1951+
if _, err = sess.In("issue_id", deleteCond).
1952+
Delete(&IssueWatch{}); err != nil {
1953+
return
1954+
}
1955+
1956+
if _, err = sess.In("issue_id", deleteCond).
1957+
Delete(&Stopwatch{}); err != nil {
1958+
return
1959+
}
1960+
1961+
if _, err = sess.In("issue_id", deleteCond).
1962+
Delete(&TrackedTime{}); err != nil {
1963+
return
1964+
}
1965+
1966+
var attachments []*Attachment
1967+
if err = sess.In("issue_id", deleteCond).
1968+
Find(&attachments); err != nil {
1969+
return
1970+
}
1971+
for j := range attachments {
1972+
attachmentPaths = append(attachmentPaths, attachments[j].LocalPath())
1973+
}
1974+
1975+
if _, err = sess.In("issue_id", deleteCond).
1976+
Delete(&Attachment{}); err != nil {
1977+
return
1978+
}
1979+
1980+
if _, err = sess.Delete(&Issue{RepoID: repoID}); err != nil {
1981+
return
1982+
}
1983+
1984+
return
1985+
}

‎models/models.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,10 @@ func SetEngine() (err error) {
182182
}
183183

184184
// NewEngine initializes a new xorm.Engine
185+
// This function must never call .Sync2() if the provided migration function fails.
186+
// When called from the "doctor" command, the migration function is a version check
187+
// that prevents the doctor from fixing anything in the database if the migration level
188+
// is different from the expected value.
185189
func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
186190
if err = SetEngine(); err != nil {
187191
return err

‎models/repo.go

Lines changed: 3 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ import (
3535
"code.gitea.io/gitea/modules/util"
3636

3737
"github.com/unknwon/com"
38-
"xorm.io/builder"
3938
)
4039

4140
var (
@@ -1590,67 +1589,9 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
15901589
return fmt.Errorf("deleteBeans: %v", err)
15911590
}
15921591

1593-
deleteCond := builder.Select("id").From("issue").Where(builder.Eq{"repo_id": repoID})
1594-
// Delete comments and attachments
1595-
if _, err = sess.In("issue_id", deleteCond).
1596-
Delete(&Comment{}); err != nil {
1597-
return err
1598-
}
1599-
1600-
// Dependencies for issues in this repository
1601-
if _, err = sess.In("issue_id", deleteCond).
1602-
Delete(&IssueDependency{}); err != nil {
1603-
return err
1604-
}
1605-
1606-
// Delete dependencies for issues in other repositories
1607-
if _, err = sess.In("dependency_id", deleteCond).
1608-
Delete(&IssueDependency{}); err != nil {
1609-
return err
1610-
}
1611-
1612-
if _, err = sess.In("issue_id", deleteCond).
1613-
Delete(&IssueUser{}); err != nil {
1614-
return err
1615-
}
1616-
1617-
if _, err = sess.In("issue_id", deleteCond).
1618-
Delete(&Reaction{}); err != nil {
1619-
return err
1620-
}
1621-
1622-
if _, err = sess.In("issue_id", deleteCond).
1623-
Delete(&IssueWatch{}); err != nil {
1624-
return err
1625-
}
1626-
1627-
if _, err = sess.In("issue_id", deleteCond).
1628-
Delete(&Stopwatch{}); err != nil {
1629-
return err
1630-
}
1631-
1632-
if _, err = sess.In("issue_id", deleteCond).
1633-
Delete(&TrackedTime{}); err != nil {
1634-
return err
1635-
}
1636-
1637-
attachments = attachments[:0]
1638-
if err = sess.Join("INNER", "issue", "issue.id = attachment.issue_id").
1639-
Where("issue.repo_id = ?", repoID).
1640-
Find(&attachments); err != nil {
1641-
return err
1642-
}
1643-
attachmentPaths := make([]string, 0, len(attachments))
1644-
for j := range attachments {
1645-
attachmentPaths = append(attachmentPaths, attachments[j].LocalPath())
1646-
}
1647-
1648-
if _, err = sess.In("issue_id", deleteCond).
1649-
Delete(&Attachment{}); err != nil {
1650-
return err
1651-
}
1652-
1653-
if _, err = sess.Delete(&Issue{RepoID: repoID}); err != nil {
1592+
// Delete Issues and related objects
1593+
var attachmentPaths []string
1594+
if attachmentPaths, err = deleteIssuesByRepoID(sess, repoID); err != nil {
16541595
return err
16551596
}
16561597

0 commit comments

Comments
 (0)