Skip to content

Commit c0ea3e6

Browse files
dtunikovdtunikov
committed
support more maria types
1 parent de3eff2 commit c0ea3e6

2 files changed

Lines changed: 78 additions & 14 deletions

File tree

‎flow/connectors/mysql/cdc_test.go‎

Lines changed: 71 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ func TestParseSQLParsesTrailingNull(t *testing.T) {
163163
}
164164
}
165165

166-
func TestParseSQLAlterTableAddColumnTypes(t *testing.T) {
166+
func TestAlterTableTypes(t *testing.T) {
167167
// addColumnType parses `ALTER TABLE t ADD COLUMN c <colDef>` and returns the
168168
// column's InfoSchemaStr, the same value processAlterTableQuery feeds to
169169
// QkindFromMysqlColumnType.
@@ -201,6 +201,7 @@ func TestParseSQLAlterTableAddColumnTypes(t *testing.T) {
201201
{"bigint", "BIGINT", types.QValueKindInt64},
202202
{"bigint unsigned", "BIGINT UNSIGNED", types.QValueKindUInt64},
203203
{"bit", "BIT(8)", types.QValueKindUInt64},
204+
{"bit no args", "BIT", types.QValueKindUInt64},
204205

205206
// floating point and exact numeric
206207
{"float", "FLOAT", types.QValueKindFloat32},
@@ -209,7 +210,10 @@ func TestParseSQLAlterTableAddColumnTypes(t *testing.T) {
209210
{"numeric", "NUMERIC(10,2)", types.QValueKindNumeric},
210211

211212
// strings
212-
{"char", "CHAR(10)", types.QValueKindString},
213+
{"char(10)", "CHAR(10)", types.QValueKindString},
214+
{"char", "CHAR", types.QValueKindString},
215+
{"character", "CHARACTER", types.QValueKindString},
216+
{"character(10)", "CHARACTER(10)", types.QValueKindString},
213217
{"varchar", "VARCHAR(255)", types.QValueKindString},
214218
{"tinytext", "TINYTEXT", types.QValueKindString},
215219
{"text", "TEXT", types.QValueKindString},
@@ -266,24 +270,47 @@ func TestParseSQLAlterTableAddColumnTypes(t *testing.T) {
266270
// exact numeric spellings normalize to decimal
267271
{"dec", "DEC(10,2)", types.QValueKindNumeric},
268272
{"fixed", "FIXED(10,2)", types.QValueKindNumeric},
273+
// Bracketless decimal family defaults to decimal(10,0) (the (M,D) is cut away anyway).
274+
{"decimal no args", "DECIMAL", types.QValueKindNumeric},
275+
{"numeric no args", "NUMERIC", types.QValueKindNumeric},
276+
{"dec no args", "DEC", types.QValueKindNumeric},
277+
{"fixed no args", "FIXED", types.QValueKindNumeric},
269278

270279
// approximate numeric spellings normalize to double
271280
{"double precision", "DOUBLE PRECISION", types.QValueKindFloat64},
272281
{"real", "REAL", types.QValueKindFloat64},
273282

274283
// char spellings normalize to char
275-
{"character", "CHARACTER(10)", types.QValueKindString},
284+
{"character(10)", "CHARACTER(10)", types.QValueKindString},
285+
{"character", "CHARACTER", types.QValueKindString},
276286
{"nchar", "NCHAR(10)", types.QValueKindString},
277287
{"national char", "NATIONAL CHAR(10)", types.QValueKindString},
278288

279289
// varchar spellings normalize to varchar
280290
{"character varying", "CHARACTER VARYING(255)", types.QValueKindString},
281291
{"nvarchar", "NVARCHAR(255)", types.QValueKindString},
282292
{"national varchar", "NATIONAL VARCHAR(255)", types.QValueKindString},
283-
284-
// MariaDB LONG / LONG VARCHAR normalize to mediumtext
293+
// more MariaDB varchar spellings (all normalize to varchar)
294+
{"char varying", "CHAR VARYING(255)", types.QValueKindString},
295+
{"varcharacter", "VARCHARACTER(255)", types.QValueKindString},
296+
{"national char varying", "NATIONAL CHAR VARYING(255)", types.QValueKindString},
297+
{"national character varying", "NATIONAL CHARACTER VARYING(255)", types.QValueKindString},
298+
{"national varcharacter", "NATIONAL VARCHARACTER(255)", types.QValueKindString},
299+
{"nchar varchar", "NCHAR VARCHAR(255)", types.QValueKindString},
300+
{"nchar varying", "NCHAR VARYING(255)", types.QValueKindString},
301+
{"nchar varcharacter", "NCHAR VARCHARACTER(255)", types.QValueKindString},
302+
303+
// MariaDB NATIONAL CHARACTER normalizes to char
304+
{"national character", "NATIONAL CHARACTER(10)", types.QValueKindString},
305+
306+
// MariaDB LONG / LONG VARCHAR and friends normalize to mediumtext
285307
{"long", "LONG", types.QValueKindString},
286308
{"long varchar", "LONG VARCHAR", types.QValueKindString},
309+
{"long char varying", "LONG CHAR VARYING", types.QValueKindString},
310+
{"long character varying", "LONG CHARACTER VARYING", types.QValueKindString},
311+
{"long varcharacter", "LONG VARCHARACTER", types.QValueKindString},
312+
// LONG VARBINARY normalizes to mediumblob
313+
{"long varbinary", "LONG VARBINARY", types.QValueKindBytes},
287314

288315
// Display widths: the (M) on integers is parsed away (we cut at "(").
289316
{"tinyint width", "TINYINT(4)", types.QValueKindInt8},
@@ -308,8 +335,12 @@ func TestParseSQLAlterTableAddColumnTypes(t *testing.T) {
308335
{"double unsigned", "DOUBLE UNSIGNED", types.QValueKindFloat64},
309336
{"double precision(m,d)", "DOUBLE PRECISION(10,2)", types.QValueKindFloat64},
310337

338+
// FLOAT(p) single-precision form: p<=24 stays float, p>=25 normalizes to double.
339+
{"float(p) p<=24 is float", "FLOAT(24)", types.QValueKindFloat32},
340+
{"float(p) p>=25 is double", "FLOAT(25)", types.QValueKindFloat64},
341+
311342
// Character types without an explicit length normalize to length 1.
312-
{"char without length", "CHAR", types.QValueKindString},
343+
{"char without length", "CHAR CHARACTER SET utf8mb4 COLLATE utf8mb4_bin", types.QValueKindString},
313344
{"nchar without length", "NCHAR", types.QValueKindString},
314345
{"national char without length", "NATIONAL CHAR", types.QValueKindString},
315346

@@ -323,8 +354,14 @@ func TestParseSQLAlterTableAddColumnTypes(t *testing.T) {
323354
{"varchar collate", "VARCHAR(10) COLLATE utf8mb4_bin", types.QValueKindString},
324355
{"varchar charset collate", "VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin", types.QValueKindString},
325356
{"text charset", "TEXT CHARACTER SET latin1", types.QValueKindString},
326-
{"enum charset", "ENUM('a','b') CHARACTER SET utf8mb4", types.QValueKindEnum},
357+
{"enum charset", "ENUM('a','b') CHARACTER SET utf8mb4 COLLATE utf8mb4_bin", types.QValueKindEnum},
327358
{"set collate", "SET('a','b') COLLATE utf8mb4_bin", types.QValueKindString},
359+
// The deprecated BINARY modifier only forces binary collation; the parser strips it,
360+
// so the qkind is unaffected (it is NOT the BINARY column type).
361+
{"varchar binary modifier", "VARCHAR(10) BINARY", types.QValueKindString},
362+
{"text binary modifier", "TEXT BINARY", types.QValueKindString},
363+
{"enum binary modifier", "ENUM('a','b') BINARY", types.QValueKindEnum},
364+
{"set binary modifier", "SET('a','b') BINARY", types.QValueKindString},
328365
} {
329366
t.Run(tc.name, func(t *testing.T) {
330367
qkind, err := QkindFromMysqlColumnType(addColumnType(t, tc.colDef), true, shared.InternalVersion_Latest)
@@ -345,15 +382,17 @@ func TestParseSQLAlterTableAddColumnEnumWithoutRowMetadata(t *testing.T) {
345382
require.Equal(t, types.QValueKindUint16Enum, qkind)
346383
}
347384

348-
// TestParseSQLAlterTableAddColumnUnsupportedTypes covers types the TiDB parser rejects
349-
// in ADD COLUMN: spatial types, and MariaDB's network/UUID types.
350-
func TestParseSQLAlterTableAddColumnUnsupportedTypes(t *testing.T) {
385+
func TestUnsupportedDDLTypes(t *testing.T) {
351386
for _, colType := range []string{
352387
// spatial types
353388
"GEOMETRY", "POINT", "POLYGON", "LINESTRING", "MULTIPOINT",
354389
"MULTILINESTRING", "MULTIPOLYGON", "GEOMETRYCOLLECTION", "GEOMCOLLECTION",
355390
// MariaDB-only network/UUID types
356391
"INET4", "INET6", "UUID",
392+
// MariaDB/Oracle-mode aliases the TiDB parser doesn't accept. The ones with a scalar
393+
// mapping (CHAR BYTE/RAW->bytes, CLOB->text, VARCHAR2->varchar, XMLTYPE->text) are still
394+
// handled directly by QkindFromMysqlColumnType, see TestQkindParserRejectedAliases.
395+
"CHAR BYTE", "RAW", "CLOB", "VARCHAR2", "XMLTYPE",
357396
} {
358397
t.Run(colType, func(t *testing.T) {
359398
_, _, err := parseSQL(parser.New(), []byte("ALTER TABLE t ADD COLUMN c "+colType))
@@ -362,6 +401,28 @@ func TestParseSQLAlterTableAddColumnUnsupportedTypes(t *testing.T) {
362401
}
363402
}
364403

404+
func TestQkindFromMysSQLTypeForUnsupportedDDLTypes(t *testing.T) {
405+
for _, tc := range []struct {
406+
ct string
407+
want types.QValueKind
408+
}{
409+
{"char byte", types.QValueKindBytes}, // BINARY
410+
{"raw", types.QValueKindBytes}, // VARBINARY (Oracle mode)
411+
{"clob", types.QValueKindString}, // LONGTEXT (Oracle mode)
412+
{"varchar2", types.QValueKindString}, // VARCHAR (Oracle mode)
413+
{"xmltype", types.QValueKindString}, // XML stored as text (MariaDB 12.3+)
414+
{"inet(4)", types.QValueKindINET},
415+
{"inet(6)", types.QValueKindINET},
416+
{"uuid", types.QValueKindUUID},
417+
} {
418+
t.Run(tc.ct, func(t *testing.T) {
419+
qkind, err := QkindFromMysqlColumnType(tc.ct, true, shared.InternalVersion_Latest)
420+
require.NoError(t, err)
421+
require.Equal(t, tc.want, qkind)
422+
})
423+
}
424+
}
425+
365426
func TestShouldReportColumnTypeChange(t *testing.T) {
366427
for _, tc := range []struct {
367428
name string

‎flow/connectors/mysql/type_conversion.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,17 @@ func QkindFromMysqlColumnType(ct string, binlogRowMetadataSupported bool, versio
1818
switch strings.ToLower(ct) {
1919
case "json":
2020
return types.QValueKindJSON, nil
21-
case "char", "varchar", "text", "set", "tinytext", "mediumtext", "longtext":
21+
case "char", "varchar", "text", "set", "tinytext", "mediumtext", "longtext",
22+
"clob", "varchar2", // maria
23+
"xmltype": // maria
2224
return types.QValueKindString, nil
2325
case "enum":
2426
if !binlogRowMetadataSupported && version >= shared.InternalVersion_MySQL5ConvertEnumsToInts {
2527
return types.QValueKindUint16Enum, nil
2628
}
2729
return types.QValueKindEnum, nil
28-
case "binary", "varbinary", "blob", "tinyblob", "mediumblob", "longblob":
30+
case "binary", "varbinary", "blob", "tinyblob", "mediumblob", "longblob",
31+
"char byte", "raw": // maria
2932
return types.QValueKindBytes, nil
3033
case "date":
3134
return types.QValueKindDate, nil
@@ -69,9 +72,9 @@ func QkindFromMysqlColumnType(ct string, binlogRowMetadataSupported bool, versio
6972
}
7073
case "vector":
7174
return types.QValueKindArrayFloat32, nil
72-
case "uuid": // MariaDB 10.7+
75+
case "uuid": // maria
7376
return types.QValueKindUUID, nil
74-
case "inet4", "inet6": // MariaDB 10.10+ / 10.5+; both rendered as text
77+
case "inet": // maria
7578
return types.QValueKindINET, nil
7679
case "geometry", "point", "polygon", "linestring", "multipoint", "multilinestring", "multipolygon", "geomcollection", "geometrycollection":
7780
return types.QValueKindGeometry, nil

0 commit comments

Comments
 (0)