Skip to content

Commit dbff926

Browse files
dk-lockdowndk-lockdown
authored
feat: support limit without orderby (#170)
1 parent 005ea15 commit dbff926

9 files changed

Lines changed: 340 additions & 21 deletions

File tree

‎cmd/cmd.go‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ import (
4646
"github.com/cectc/dbpack/pkg/proto"
4747
"github.com/cectc/dbpack/pkg/resource"
4848
"github.com/cectc/dbpack/pkg/server"
49-
"github.com/cectc/dbpack/pkg/tracing"
5049
"github.com/cectc/dbpack/third_party/pools"
5150
_ "github.com/cectc/dbpack/third_party/types/parser_driver"
5251
)
@@ -155,13 +154,12 @@ var (
155154
}
156155
}
157156

158-
tracingMgr, err := tracing.NewTracer(Version, "console")
159-
if err != nil {
160-
log.Fatalf("could not setup tracing manager: %s", err.Error())
161-
}
162-
if err != nil {
163-
log.Fatalf("could not setup tracing exporter: %s", err.Error())
164-
}
157+
// temporarily turn off tracer output
158+
//tracingMgr, err := tracing.NewTracer(Version, "console")
159+
//if err != nil {
160+
// log.Fatalf("could not setup tracing manager: %s", err.Error())
161+
//}
162+
165163
ctx, cancel := context.WithCancel(context.Background())
166164
c := make(chan os.Signal, 2)
167165
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
@@ -174,7 +172,7 @@ var (
174172
cancel()
175173
}()
176174
<-c
177-
_ = tracingMgr.Shutdown(ctx)
175+
//_ = tracingMgr.Shutdown(ctx)
178176
os.Exit(1) // second signal. Exit directly.
179177
}()
180178

‎pkg/optimize/optimize_select.go‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,22 @@ import (
2323
"github.com/pkg/errors"
2424

2525
"github.com/cectc/dbpack/pkg/cond"
26+
"github.com/cectc/dbpack/pkg/dt/schema"
27+
"github.com/cectc/dbpack/pkg/meta"
2628
"github.com/cectc/dbpack/pkg/plan"
2729
"github.com/cectc/dbpack/pkg/proto"
30+
"github.com/cectc/dbpack/pkg/resource"
2831
"github.com/cectc/dbpack/pkg/topo"
2932
"github.com/cectc/dbpack/third_party/parser/ast"
3033
)
3134

3235
func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, args []interface{}) (proto.Plan, error) {
3336
var (
34-
alg cond.ShardingAlgorithm
35-
topology *topo.Topology
36-
exists bool
37+
alg cond.ShardingAlgorithm
38+
topology *topo.Topology
39+
tableMeta schema.TableMeta
40+
exists bool
41+
err error
3742
)
3843
tableName := stmt.From.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName).Name.String()
3944

@@ -44,6 +49,17 @@ func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, arg
4449
return nil, errors.New("topology should not be nil")
4550
}
4651

52+
for db, tables := range topology.DBs {
53+
sqlDB := resource.GetDBManager().GetDB(db)
54+
tableMeta, err = meta.GetTableMetaCache().GetTableMeta(ctx, sqlDB, tables[0])
55+
if err != nil {
56+
continue
57+
} else {
58+
break
59+
}
60+
}
61+
pk := tableMeta.GetPKName()
62+
4763
condition, err := cond.ParseCondition(stmt.Where, args...)
4864
if err != nil {
4965
return nil, errors.Wrap(err, "parse condition failed")
@@ -69,6 +85,7 @@ func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, arg
6985
return &plan.QueryOnSingleDBPlan{
7086
Database: k,
7187
Tables: v,
88+
PK: pk,
7289
Stmt: stmt,
7390
Args: args,
7491
Executor: executor,
@@ -92,6 +109,7 @@ func (o Optimizer) optimizeSelect(ctx context.Context, stmt *ast.SelectStmt, arg
92109
plans = append(plans, &plan.QueryOnSingleDBPlan{
93110
Database: k,
94111
Tables: shardMap[k],
112+
PK: pk,
95113
Stmt: stmt,
96114
Args: args,
97115
Executor: executor,

‎pkg/optimize/optimizer_test.go‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,95 @@ func TestOptimizeQueryOnSingleDB(t *testing.T) {
4646
return
4747
}
4848
stmt.Accept(&visitor.ParamVisitor{})
49+
50+
resource.SetDBManager(&resource.DBManager{})
51+
var cache *meta.MysqlTableMetaCache
52+
patches := gomonkey.ApplyMethodFunc(cache, "GetTableMeta", func(ctx context.Context, db proto.DB, tableName string) (schema.TableMeta, error) {
53+
return schema.TableMeta{
54+
SchemaName: "school",
55+
TableName: "student",
56+
Columns: []string{"id", "age"},
57+
AllColumns: map[string]schema.ColumnMeta{
58+
"id": {
59+
TableCat: "def",
60+
TableSchemeName: "school",
61+
TableName: "student",
62+
ColumnName: "id",
63+
DataType: -5,
64+
DataTypeName: "bigint",
65+
ColumnSize: 0,
66+
DecimalDigits: 19,
67+
NumPrecRadix: 0,
68+
Nullable: 0,
69+
Remarks: "",
70+
ColumnDef: "",
71+
SqlDataType: 0,
72+
SqlDatetimeSub: 0,
73+
CharOctetLength: 0,
74+
OrdinalPosition: 1,
75+
IsNullable: "NO",
76+
IsAutoIncrement: "auto_increment",
77+
},
78+
"age": {
79+
TableCat: "def",
80+
TableSchemeName: "school",
81+
TableName: "student",
82+
ColumnName: "age",
83+
DataType: 0,
84+
DataTypeName: "int",
85+
ColumnSize: 0,
86+
DecimalDigits: 10,
87+
NumPrecRadix: 0,
88+
Nullable: 0,
89+
Remarks: "",
90+
ColumnDef: "",
91+
SqlDataType: 0,
92+
SqlDatetimeSub: 0,
93+
CharOctetLength: 0,
94+
OrdinalPosition: 2,
95+
IsNullable: "NO",
96+
IsAutoIncrement: "",
97+
},
98+
},
99+
AllIndexes: map[string]schema.IndexMeta{
100+
"id": {
101+
Values: []schema.ColumnMeta{
102+
{
103+
TableCat: "def",
104+
TableSchemeName: "school",
105+
TableName: "student",
106+
ColumnName: "id",
107+
DataType: -5,
108+
DataTypeName: "bigint",
109+
ColumnSize: 0,
110+
DecimalDigits: 19,
111+
NumPrecRadix: 0,
112+
Nullable: 0,
113+
Remarks: "",
114+
ColumnDef: "",
115+
SqlDataType: 0,
116+
SqlDatetimeSub: 0,
117+
CharOctetLength: 0,
118+
OrdinalPosition: 1,
119+
IsNullable: "NO",
120+
IsAutoIncrement: "auto_increment",
121+
},
122+
},
123+
NonUnique: false,
124+
IndexQualifier: "",
125+
IndexName: "PRIMARY",
126+
ColumnName: "id",
127+
Type: 0,
128+
IndexType: schema.IndexTypePrimary,
129+
AscOrDesc: "A",
130+
Cardinality: 1,
131+
OrdinalPosition: 1,
132+
},
133+
},
134+
}, nil
135+
})
136+
defer patches.Reset()
137+
49138
pl, err := o.Optimize(context.Background(), stmt, args...)
50139
assert.Equal(t, nil, err)
51140
queryPlan, ok := pl.(*plan.QueryOnSingleDBPlan)
@@ -66,6 +155,95 @@ func TestOptimizeQueryOnMultiDB(t *testing.T) {
66155
return
67156
}
68157
stmt.Accept(&visitor.ParamVisitor{})
158+
159+
resource.SetDBManager(&resource.DBManager{})
160+
var cache *meta.MysqlTableMetaCache
161+
patches := gomonkey.ApplyMethodFunc(cache, "GetTableMeta", func(ctx context.Context, db proto.DB, tableName string) (schema.TableMeta, error) {
162+
return schema.TableMeta{
163+
SchemaName: "school",
164+
TableName: "student",
165+
Columns: []string{"id", "age"},
166+
AllColumns: map[string]schema.ColumnMeta{
167+
"id": {
168+
TableCat: "def",
169+
TableSchemeName: "school",
170+
TableName: "student",
171+
ColumnName: "id",
172+
DataType: -5,
173+
DataTypeName: "bigint",
174+
ColumnSize: 0,
175+
DecimalDigits: 19,
176+
NumPrecRadix: 0,
177+
Nullable: 0,
178+
Remarks: "",
179+
ColumnDef: "",
180+
SqlDataType: 0,
181+
SqlDatetimeSub: 0,
182+
CharOctetLength: 0,
183+
OrdinalPosition: 1,
184+
IsNullable: "NO",
185+
IsAutoIncrement: "auto_increment",
186+
},
187+
"age": {
188+
TableCat: "def",
189+
TableSchemeName: "school",
190+
TableName: "student",
191+
ColumnName: "age",
192+
DataType: 0,
193+
DataTypeName: "int",
194+
ColumnSize: 0,
195+
DecimalDigits: 10,
196+
NumPrecRadix: 0,
197+
Nullable: 0,
198+
Remarks: "",
199+
ColumnDef: "",
200+
SqlDataType: 0,
201+
SqlDatetimeSub: 0,
202+
CharOctetLength: 0,
203+
OrdinalPosition: 2,
204+
IsNullable: "NO",
205+
IsAutoIncrement: "",
206+
},
207+
},
208+
AllIndexes: map[string]schema.IndexMeta{
209+
"id": {
210+
Values: []schema.ColumnMeta{
211+
{
212+
TableCat: "def",
213+
TableSchemeName: "school",
214+
TableName: "student",
215+
ColumnName: "id",
216+
DataType: -5,
217+
DataTypeName: "bigint",
218+
ColumnSize: 0,
219+
DecimalDigits: 19,
220+
NumPrecRadix: 0,
221+
Nullable: 0,
222+
Remarks: "",
223+
ColumnDef: "",
224+
SqlDataType: 0,
225+
SqlDatetimeSub: 0,
226+
CharOctetLength: 0,
227+
OrdinalPosition: 1,
228+
IsNullable: "NO",
229+
IsAutoIncrement: "auto_increment",
230+
},
231+
},
232+
NonUnique: false,
233+
IndexQualifier: "",
234+
IndexName: "PRIMARY",
235+
ColumnName: "id",
236+
Type: 0,
237+
IndexType: schema.IndexTypePrimary,
238+
AscOrDesc: "A",
239+
Cardinality: 1,
240+
OrdinalPosition: 1,
241+
},
242+
},
243+
}, nil
244+
})
245+
defer patches.Reset()
246+
69247
pl, err := o.Optimize(context.Background(), stmt, args...)
70248
assert.Equal(t, nil, err)
71249
queryPlan, ok := pl.(*plan.QueryOnMultiDBPlan)

‎pkg/plan/query.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
type QueryOnSingleDBPlan struct {
3939
Database string
4040
Tables []string
41+
PK string
4142
Stmt *ast.SelectStmt
4243
Limit *Limit
4344
Args []interface{}
@@ -83,9 +84,8 @@ func (p *QueryOnSingleDBPlan) generate(sb *strings.Builder, args *[]interface{})
8384
err = generateSelect(p.Tables[0], p.Stmt, sb, p.Limit)
8485
p.appendArgs(args)
8586
default:
86-
if p.Stmt.OrderBy != nil {
87-
sb.WriteString("SELECT * FROM (")
88-
}
87+
sb.WriteString("SELECT * FROM (")
88+
8989
sb.WriteByte('(')
9090
if err = generateSelect(p.Tables[0], p.Stmt, sb, p.Limit); err != nil {
9191
return
@@ -103,12 +103,14 @@ func (p *QueryOnSingleDBPlan) generate(sb *strings.Builder, args *[]interface{})
103103
sb.WriteByte(')')
104104
p.appendArgs(args)
105105
}
106+
sb.WriteString(") t ")
106107
if p.Stmt.OrderBy != nil {
107-
sb.WriteString(") t ")
108108
restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, sb)
109109
if err := p.Stmt.OrderBy.Restore(restoreCtx); err != nil {
110110
return errors.WithStack(err)
111111
}
112+
} else {
113+
sb.WriteString(fmt.Sprintf("ORDER BY `%s` ASC", p.PK))
112114
}
113115
}
114116
return
@@ -258,7 +260,7 @@ func generateSelect(table string, stmt *ast.SelectStmt, sb *strings.Builder, lim
258260
if limit != nil {
259261
sb.WriteByte(' ')
260262
limitCount := limit.Offset + limit.Count
261-
sb.WriteString(fmt.Sprintf("limit %d", limitCount))
263+
sb.WriteString(fmt.Sprintf("LIMIT %d", limitCount))
262264
}
263265

264266
return nil

‎pkg/plan/query_test.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,26 +31,30 @@ func TestQueryOnSingleDBPlan(t *testing.T) {
3131
testCases := []struct {
3232
selectSql string
3333
tables []string
34+
pk string
3435
args []interface{}
3536
expectedGenerateSql string
3637
}{
3738
{
3839
selectSql: "select * from student where id in (?,?)",
3940
tables: []string{"student_1", "student_5"},
41+
pk: "id",
4042
args: []interface{}{1, 5},
41-
expectedGenerateSql: "(SELECT * FROM `student_1` WHERE `id` IN (?,?)) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?))",
43+
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?)) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?))) t ORDER BY `id` ASC",
4244
},
4345
{
4446
selectSql: "select * from student where id in (?,?) order by id desc",
4547
tables: []string{"student_1", "student_5"},
48+
pk: "id",
4649
args: []interface{}{1, 5},
4750
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?) ORDER BY `id` DESC) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?) ORDER BY `id` DESC)) t ORDER BY `id` DESC",
4851
},
4952
{
5053
selectSql: "select * from student where id in (?,?) order by id desc limit ?, ?",
5154
tables: []string{"student_1", "student_5"},
55+
pk: "id",
5256
args: []interface{}{1, 5, 1000, 20},
53-
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?) ORDER BY `id` DESC limit 1020) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?) ORDER BY `id` DESC limit 1020)) t ORDER BY `id` DESC",
57+
expectedGenerateSql: "SELECT * FROM ((SELECT * FROM `student_1` WHERE `id` IN (?,?) ORDER BY `id` DESC LIMIT 1020) UNION ALL (SELECT * FROM `student_5` WHERE `id` IN (?,?) ORDER BY `id` DESC LIMIT 1020)) t ORDER BY `id` DESC",
5458
},
5559
}
5660

@@ -67,6 +71,7 @@ func TestQueryOnSingleDBPlan(t *testing.T) {
6771
plan := &QueryOnSingleDBPlan{
6872
Database: "school_0",
6973
Tables: c.tables,
74+
PK: c.pk,
7075
Stmt: selectStmt,
7176
Args: c.args,
7277
Executor: nil,

0 commit comments

Comments
 (0)