From eb26de4260789abdc45315788232a8a34ed1adc0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 30 Mar 2024 22:15:00 +0800 Subject: [PATCH 1/4] Add unique index for project_issue to prevent duplicate data --- .../project_issue.yml | 9 ++++ models/migrations/migrations.go | 5 ++ models/migrations/v1_23/v294.go | 38 ++++++++++++++ models/migrations/v1_23/v294_test.go | 52 +++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 models/migrations/fixtures/Test_AddUniqueIndexForProjectIssue/project_issue.yml create mode 100644 models/migrations/v1_23/v294.go create mode 100644 models/migrations/v1_23/v294_test.go diff --git a/models/migrations/fixtures/Test_AddUniqueIndexForProjectIssue/project_issue.yml b/models/migrations/fixtures/Test_AddUniqueIndexForProjectIssue/project_issue.yml new file mode 100644 index 0000000000000..6feaeb39f0598 --- /dev/null +++ b/models/migrations/fixtures/Test_AddUniqueIndexForProjectIssue/project_issue.yml @@ -0,0 +1,9 @@ +- + id: 1 + project_id: 1 + issue_id: 1 + +- + id: 2 + project_id: 1 + issue_id: 1 diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 0daa799ff6754..387cd96a53471 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/models/migrations/v1_20" "code.gitea.io/gitea/models/migrations/v1_21" "code.gitea.io/gitea/models/migrations/v1_22" + "code.gitea.io/gitea/models/migrations/v1_23" "code.gitea.io/gitea/models/migrations/v1_6" "code.gitea.io/gitea/models/migrations/v1_7" "code.gitea.io/gitea/models/migrations/v1_8" @@ -572,6 +573,10 @@ var migrations = []Migration{ NewMigration("Ensure every project has exactly one default column - No Op", noopMigration), // v293 -> v294 NewMigration("Ensure every project has exactly one default column", v1_22.CheckProjectColumnsConsistency), + + // Gitea 1.22.0 ends at 294 + + NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_23/v294.go b/models/migrations/v1_23/v294.go new file mode 100644 index 0000000000000..9c18da185dd74 --- /dev/null +++ b/models/migrations/v1_23/v294.go @@ -0,0 +1,38 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_23 //nolint + +import "xorm.io/xorm" + +// AddUniqueIndexForProjectIssue adds unique indexes for project issue table +func AddUniqueIndexForProjectIssue(x *xorm.Engine) error { + // remove possible duplicated records in table project_issue + type result struct { + IssueID int64 + ProjectID int64 + Cnt int + } + var results []result + if err := x.Select("issue_id, project_id, count(*) as cnt"). + Table("project_issue"). + GroupBy("issue_id, project_id"). + Where("cnt > 1"). + Find(&results); err != nil { + return err + } + for _, r := range results { + if _, err := x.Exec("DELETE FROM project_issue WHERE issue_id = ? AND project_id = ? ORDER BY id DESC LIMIT ?", r.IssueID, r.ProjectID, r.Cnt-1); err != nil { + return err + } + } + + // add unique index for project_issue table + type ProjectIssue struct { //revive:disable-line:exported + ID int64 `xorm:"pk autoincr"` + IssueID int64 `xorm:"INDEX unique(s)"` + ProjectID int64 `xorm:"INDEX unique(s)"` + } + + return x.Sync(new(ProjectIssue)) +} diff --git a/models/migrations/v1_23/v294_test.go b/models/migrations/v1_23/v294_test.go new file mode 100644 index 0000000000000..fdbdcd38e0ed1 --- /dev/null +++ b/models/migrations/v1_23/v294_test.go @@ -0,0 +1,52 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_23 //nolint + +import ( + "slices" + "testing" + + "code.gitea.io/gitea/models/migrations/base" + "xorm.io/xorm/schemas" + + "github.com/stretchr/testify/assert" +) + +func Test_AddUniqueIndexForProjectIssue(t *testing.T) { + type ProjectIssue struct { //revive:disable-line:exported + ID int64 `xorm:"pk autoincr"` + IssueID int64 `xorm:"INDEX"` + ProjectID int64 `xorm:"INDEX"` + } + + // Prepare and load the testing database + x, deferable := base.PrepareTestEnv(t, 0, new(ProjectIssue)) + defer deferable() + if x == nil || t.Failed() { + return + } + + cnt, err := x.Table("project_issue").Where("project_id=1 AND issue_id=1").Count() + assert.NoError(t, err) + assert.EqualValues(t, 2, cnt) + + assert.NoError(t, AddUniqueIndexForProjectIssue(x)) + + cnt, err = x.Table("project_issue").Where("project_id=1 AND issue_id=1").Count() + assert.NoError(t, err) + assert.EqualValues(t, 1, cnt) + + tables, err := x.DBMetas() + assert.NoError(t, err) + assert.EqualValues(t, 1, len(tables)) + found := false + for _, index := range tables[0].Indexes { + if index.Type == schemas.UniqueType { + found = true + slices.Equal(index.Cols, []string{"project_id", "issue_id"}) + break + } + } + assert.True(t, found) +} From 4161e1068650913cf2be7f8664aa2883bbf17d45 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 1 Apr 2024 16:18:23 +0800 Subject: [PATCH 2/4] Fix test --- models/migrations/v1_23/v294.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v1_23/v294.go b/models/migrations/v1_23/v294.go index 9c18da185dd74..45aaf18499870 100644 --- a/models/migrations/v1_23/v294.go +++ b/models/migrations/v1_23/v294.go @@ -17,7 +17,7 @@ func AddUniqueIndexForProjectIssue(x *xorm.Engine) error { if err := x.Select("issue_id, project_id, count(*) as cnt"). Table("project_issue"). GroupBy("issue_id, project_id"). - Where("cnt > 1"). + Having("count(*) > 1"). Find(&results); err != nil { return err } From a7a44680f3bfb63cdb91cad41cfb755a1a6c0991 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 1 Apr 2024 21:02:53 +0800 Subject: [PATCH 3/4] Fix test --- models/migrations/v1_23/main_test.go | 14 ++++++++++++++ models/migrations/v1_23/v294.go | 21 ++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 models/migrations/v1_23/main_test.go diff --git a/models/migrations/v1_23/main_test.go b/models/migrations/v1_23/main_test.go new file mode 100644 index 0000000000000..b7948bd4dd248 --- /dev/null +++ b/models/migrations/v1_23/main_test.go @@ -0,0 +1,14 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_23 //nolint + +import ( + "testing" + + "code.gitea.io/gitea/models/migrations/base" +) + +func TestMain(m *testing.M) { + base.MainTest(m) +} diff --git a/models/migrations/v1_23/v294.go b/models/migrations/v1_23/v294.go index 45aaf18499870..f2a54f6d23ecc 100644 --- a/models/migrations/v1_23/v294.go +++ b/models/migrations/v1_23/v294.go @@ -3,7 +3,12 @@ package v1_23 //nolint -import "xorm.io/xorm" +import ( + "fmt" + + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) // AddUniqueIndexForProjectIssue adds unique indexes for project issue table func AddUniqueIndexForProjectIssue(x *xorm.Engine) error { @@ -22,8 +27,18 @@ func AddUniqueIndexForProjectIssue(x *xorm.Engine) error { return err } for _, r := range results { - if _, err := x.Exec("DELETE FROM project_issue WHERE issue_id = ? AND project_id = ? ORDER BY id DESC LIMIT ?", r.IssueID, r.ProjectID, r.Cnt-1); err != nil { - return err + if x.Dialect().URI().DBType == schemas.MSSQL { + if _, err := x.Exec(fmt.Sprintf("delete from project_issue where id in (SELECT top %d id FROM project_issue WHERE issue_id = ? and project_id = ?)", r.Cnt-1), r.IssueID, r.ProjectID); err != nil { + return err + } + } else { + var ids []int64 + if err := x.SQL("SELECT id FROM project_issue WHERE issue_id = ? and project_id = ? limit ?", r.IssueID, r.ProjectID, r.Cnt-1).Find(&ids); err != nil { + return err + } + if _, err := x.Table("project_issue").In("id", ids).Delete(); err != nil { + return err + } } } From 6ed21862580437af02ae95510ab94aef1ad92994 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 1 Apr 2024 21:36:50 +0800 Subject: [PATCH 4/4] Fix check --- models/migrations/v1_23/v294_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v1_23/v294_test.go b/models/migrations/v1_23/v294_test.go index fdbdcd38e0ed1..d9a44ad866bb1 100644 --- a/models/migrations/v1_23/v294_test.go +++ b/models/migrations/v1_23/v294_test.go @@ -8,9 +8,9 @@ import ( "testing" "code.gitea.io/gitea/models/migrations/base" - "xorm.io/xorm/schemas" "github.com/stretchr/testify/assert" + "xorm.io/xorm/schemas" ) func Test_AddUniqueIndexForProjectIssue(t *testing.T) {