Skip to content

Performance on insert many #548

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mkolumb wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Performance on insert many #548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6f19ea8
Add insert many overload
mkolumb Jan 19, 2022
c861e95
Merge remote-tracking branch 'sqlkata/master' into feature/compiler-p…
mkolumb May 25, 2022
24f3679
Merge pull request #1 from sqlkata/master
mkolumb Aug 6, 2022
7716c1b
Merge remote-tracking branch 'master/master' into feature/compiler-pe…
mkolumb Mar 28, 2023
9424ec4
Fixing build after merge
mkolumb Mar 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 10 additions & 13 deletions QueryBuilder.Tests/Infrastructure/TestCompilersContainer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,14 @@ private static class Messages
public const string ERR_INVALID_ENGINECODES = "Invalid engine codes supplied '{0}'";
}

protected readonly IDictionary<string, Compiler> Compilers = new Dictionary<string, Compiler>
protected readonly IDictionary<string, Func<Compiler>> Compilers = new Dictionary<string, Func<Compiler>>
{
[EngineCodes.Firebird] = new FirebirdCompiler(),
[EngineCodes.MySql] = new MySqlCompiler(),
[EngineCodes.Oracle] = new OracleCompiler(),
[EngineCodes.PostgreSql] = new PostgresCompiler(),
[EngineCodes.Sqlite] = new SqliteCompiler(),
[EngineCodes.SqlServer] = new SqlServerCompiler()
{
UseLegacyPagination = true
}
[EngineCodes.Firebird] = () => new FirebirdCompiler(),
[EngineCodes.MySql] = () => new MySqlCompiler(),
[EngineCodes.Oracle] = () => new OracleCompiler(),
[EngineCodes.PostgreSql] = () => new PostgresCompiler(),
[EngineCodes.Sqlite] = () => new SqliteCompiler(),
[EngineCodes.SqlServer] = () => new SqlServerCompiler { UseLegacyPagination = true }
};

public IEnumerable<string> KnownEngineCodes
Expand All @@ -43,7 +40,7 @@ public Compiler Get(string engineCode)
throw new InvalidOperationException(string.Format(Messages.ERR_INVALID_ENGINECODE, engineCode));
}

return Compilers[engineCode];
return Compilers[engineCode]();
}

/// <summary>
Expand Down Expand Up @@ -82,7 +79,7 @@ public TestSqlResultContainer Compile(IEnumerable<string> engineCodes, Query que

var results = Compilers
.Where(w => codes.Contains(w.Key))
.ToDictionary(k => k.Key, v => v.Value.Compile(query.Clone()));
.ToDictionary(k => k.Key, v => v.Value().Compile(query.Clone()));

if (results.Count != codes.Count)
{
Expand All @@ -102,7 +99,7 @@ public TestSqlResultContainer Compile(IEnumerable<string> engineCodes, Query que
public TestSqlResultContainer Compile(Query query)
{
var resultKeyValues = Compilers
.ToDictionary(k => k.Key, v => v.Value.Compile(query.Clone()));
.ToDictionary(k => k.Key, v => v.Value().Compile(query.Clone()));
return new TestSqlResultContainer(resultKeyValues);
}
}
Expand Down
24 changes: 24 additions & 0 deletions QueryBuilder.Tests/InsertTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,30 @@ public void InsertMultiRecords()
c[EngineCodes.Firebird]);
}

[Fact]
public void InsertMultiRecordsByDictionary()
{
var data = new List<Dictionary<string, object>>
{
new() { { "name", "Chiron" }, { "brand", "Bugatti" }, { "year", null } },
new() { { "name", "Huayra" }, { "brand", "Pagani" }, { "year", 2012 } },
new() { { "name", "Reventon roadster" }, { "brand", "Lamborghini" }, { "year", 2009 } }
};

var query = new Query("expensive_cars")
.AsInsert(data);

var c = Compile(query);

Assert.Equal(
"INSERT INTO [expensive_cars] ([name], [brand], [year]) VALUES ('Chiron', 'Bugatti', NULL), ('Huayra', 'Pagani', 2012), ('Reventon roadster', 'Lamborghini', 2009)",
c[EngineCodes.SqlServer]);

Assert.Equal(
"INSERT INTO \"EXPENSIVE_CARS\" (\"NAME\", \"BRAND\", \"YEAR\") SELECT 'Chiron', 'Bugatti', NULL FROM RDB$DATABASE UNION ALL SELECT 'Huayra', 'Pagani', 2012 FROM RDB$DATABASE UNION ALL SELECT 'Reventon roadster', 'Lamborghini', 2009 FROM RDB$DATABASE",
c[EngineCodes.Firebird]);
}

[Fact]
public void InsertWithNullValues()
{
Expand Down
6 changes: 2 additions & 4 deletions QueryBuilder/Clauses/InsertClause.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ public abstract class AbstractInsertClause : AbstractClause

public class InsertClause : AbstractInsertClause
{
public List<string> Columns { get; set; }
public List<object> Values { get; set; }
public Dictionary<string, object> Data { get; set; }
public bool ReturnId { get; set; } = false;

public override AbstractClause Clone()
Expand All @@ -19,8 +18,7 @@ public override AbstractClause Clone()
{
Engine = Engine,
Component = Component,
Columns = Columns,
Values = Values,
Data = Data,
ReturnId = ReturnId,
};
}
Expand Down
7 changes: 4 additions & 3 deletions QueryBuilder/Compilers/Compiler.Conditions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;

namespace SqlKata.Compilers
{
Expand Down Expand Up @@ -43,7 +44,7 @@ protected virtual string CompileCondition(SqlResult ctx, AbstractCondition claus

protected virtual string CompileConditions(SqlResult ctx, List<AbstractCondition> conditions)
{
var result = new List<string>();
var result = new StringBuilder();

for (var i = 0; i < conditions.Count; i++)
{
Expand All @@ -56,10 +57,10 @@ protected virtual string CompileConditions(SqlResult ctx, List<AbstractCondition

var boolOperator = i == 0 ? "" : (conditions[i].IsOr ? "OR " : "AND ");

result.Add(boolOperator + compiled);
result.Append(boolOperator + compiled + " ");
}

return string.Join(" ", result);
return result.ToString().Trim();
}

protected virtual string CompileRawCondition(SqlResult ctx, RawCondition x)
Expand Down
60 changes: 32 additions & 28 deletions QueryBuilder/Compilers/Compiler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,23 +367,25 @@ protected virtual SqlResult CompileUpdateQuery(Query query)


var toUpdate = ctx.Query.GetOneComponent<InsertClause>("update", EngineCode);
var parts = new List<string>();
var parts = new StringBuilder(toUpdate.Data.Count);

for (var i = 0; i < toUpdate.Columns.Count; i++)
var separator = ", ";

foreach (var item in toUpdate.Data)
{
parts.Add(Wrap(toUpdate.Columns[i]) + " = " + Parameter(ctx, toUpdate.Values[i]));
parts.Append(Wrap(item.Key) + " = " + Parameter(ctx, item.Value) + separator);
}

var sets = string.Join(", ", parts);

parts.Length -= separator.Length;
wheres = CompileWheres(ctx);

if (!string.IsNullOrEmpty(wheres))
{
wheres = " " + wheres;
}

ctx.RawSql = $"UPDATE {table} SET {sets}{wheres}";
ctx.RawSql = $"UPDATE {table} SET {parts}{wheres}".Trim();

return ctx;
}
Expand Down Expand Up @@ -418,7 +420,7 @@ protected virtual SqlResult CompileInsertQuery(Query query)
if (inserts[0] is InsertQueryClause insertQueryClause)
return CompileInsertQueryClause(ctx, table, insertQueryClause);
else
return CompileValueInsertClauses(ctx, table, inserts.Cast<InsertClause>());
return CompileValueInsertClauses(ctx, table, inserts.Cast<InsertClause>().ToArray());
}

protected virtual SqlResult CompileInsertQueryClause(
Expand All @@ -435,15 +437,15 @@ protected virtual SqlResult CompileInsertQueryClause(
}

protected virtual SqlResult CompileValueInsertClauses(
SqlResult ctx, string table, IEnumerable<InsertClause> insertClauses)
SqlResult ctx, string table, IReadOnlyCollection<InsertClause> insertClauses)
{
bool isMultiValueInsert = insertClauses.Skip(1).Any();
bool isMultiValueInsert = insertClauses.Count > 1;

var insertInto = (isMultiValueInsert) ? MultiInsertStartClause : SingleInsertStartClause;

var firstInsert = insertClauses.First();
string columns = GetInsertColumnsList(firstInsert.Columns);
var values = string.Join(", ", Parameterize(ctx, firstInsert.Values));
string columns = GetInsertColumnsList(firstInsert.Data.Keys);
var values = string.Join(", ", Parameterize(ctx, firstInsert.Data.Values));

ctx.RawSql = $"{insertInto} {table}{columns} VALUES ({values})";

Expand All @@ -456,17 +458,21 @@ protected virtual SqlResult CompileValueInsertClauses(
return ctx;
}

protected virtual SqlResult CompileRemainingInsertClauses(SqlResult ctx, string table, IEnumerable<InsertClause> inserts)
protected virtual SqlResult CompileRemainingInsertClauses(SqlResult ctx, string table, IReadOnlyCollection<InsertClause> inserts)
{
var sql = new StringBuilder(ctx.RawSql, inserts.Count - 1);
Copy link

Copilot AI Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

StringBuilder constructor parameters are (string, int capacity), but you're passing (string, int length). The second parameter should be the initial capacity, not the count calculation. Use new StringBuilder(ctx.RawSql) or provide a proper capacity estimate.

Suggested change
var sql = new StringBuilder(ctx.RawSql, inserts.Count - 1);
var sql = new StringBuilder(ctx.RawSql);

Copilot uses AI. Check for mistakes.

foreach (var insert in inserts.Skip(1))
{
string values = string.Join(", ", Parameterize(ctx, insert.Values));
ctx.RawSql += $", ({values})";
sql.Append($", ({string.Join(", ", Parameterize(ctx, insert.Data.Values))})");
}

ctx.RawSql = sql.ToString();

return ctx;
}

protected string GetInsertColumnsList(List<string> columnList)
protected string GetInsertColumnsList(IReadOnlyCollection<string> columnList)
{
var columns = "";
if (columnList.Any())
Expand All @@ -480,7 +486,7 @@ protected virtual SqlResult CompileCteQuery(SqlResult ctx, Query query)
var cteFinder = new CteFinder(query, EngineCode);
var cteSearchResult = cteFinder.Find();

var rawSql = new StringBuilder("WITH ");
var rawSql = new StringBuilder("WITH ", cteSearchResult.Count * 2 + 3);
var cteBindings = new List<object>();

foreach (var cte in cteSearchResult)
Expand All @@ -497,7 +503,7 @@ protected virtual SqlResult CompileCteQuery(SqlResult ctx, Query query)
rawSql.Append(ctx.RawSql);

ctx.Bindings.InsertRange(0, cteBindings);
ctx.RawSql = rawSql.ToString();
ctx.RawSql = rawSql.ToString().Trim();

return ctx;
}
Expand Down Expand Up @@ -657,7 +663,7 @@ public virtual string CompileUnion(SqlResult ctx)
return null;
}

var combinedQueries = new List<string>();
var combinedQueries = new StringBuilder();

var clauses = ctx.Query.GetComponents<AbstractCombine>("combine", EngineCode);

Expand All @@ -671,21 +677,19 @@ public virtual string CompileUnion(SqlResult ctx)

ctx.Bindings.AddRange(subCtx.Bindings);

combinedQueries.Add($"{combineOperator}{subCtx.RawSql}");
combinedQueries.Append($"{combineOperator}{subCtx.RawSql} ");
}
else
{
var combineRawClause = clause as RawCombine;

ctx.Bindings.AddRange(combineRawClause.Bindings);

combinedQueries.Add(WrapIdentifiers(combineRawClause.Expression));

combinedQueries.Append(WrapIdentifiers(combineRawClause.Expression) + " ");
}
}

return string.Join(" ", combinedQueries);

return combinedQueries.ToString().Trim();
}

public virtual string CompileTableExpression(SqlResult ctx, AbstractFrom from)
Expand Down Expand Up @@ -817,7 +821,7 @@ public virtual string CompileHaving(SqlResult ctx)
return null;
}

var sql = new List<string>();
var sql = new StringBuilder();
string boolOperator;

var having = ctx.Query.GetComponents("having", EngineCode)
Expand All @@ -832,11 +836,11 @@ public virtual string CompileHaving(SqlResult ctx)
{
boolOperator = i > 0 ? having[i].IsOr ? "OR " : "AND " : "";

sql.Add(boolOperator + compiled);
sql.Append(boolOperator + compiled + " ");
}
}

return $"HAVING {string.Join(" ", sql)}";
return $"HAVING {sql}".Trim();
}

public virtual string CompileLimit(SqlResult ctx)
Expand Down Expand Up @@ -1042,9 +1046,9 @@ public virtual string Parameterize<T>(SqlResult ctx, IEnumerable<T> values)
/// </summary>
/// <param name="values"></param>
/// <returns></returns>
public virtual List<string> WrapArray(List<string> values)
public virtual IEnumerable<string> WrapArray(IEnumerable<string> values)
{
return values.Select(x => Wrap(x)).ToList();
return values.Select(Wrap);
}

public virtual string WrapIdentifiers(string input)
Expand Down
19 changes: 12 additions & 7 deletions QueryBuilder/Compilers/OracleCompiler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace SqlKata.Compilers
Expand Down Expand Up @@ -156,20 +157,24 @@ protected override string CompileBasicDateCondition(SqlResult ctx, BasicDateCond
}

protected override SqlResult CompileRemainingInsertClauses(
SqlResult ctx, string table, IEnumerable<InsertClause> inserts)
SqlResult ctx, string table, IReadOnlyCollection<InsertClause> inserts)
{
var sql = new StringBuilder(ctx.RawSql, inserts.Count - 1);
Copy link

Copilot AI Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same StringBuilder constructor issue as in Compiler.cs. The second parameter should be capacity, not a count calculation.

Suggested change
var sql = new StringBuilder(ctx.RawSql, inserts.Count - 1);
var sql = new StringBuilder(ctx.RawSql);

Copilot uses AI. Check for mistakes.

foreach (var insert in inserts.Skip(1))
{
string columns = GetInsertColumnsList(insert.Columns);
string values = string.Join(", ", Parameterize(ctx, insert.Values));
string columns = GetInsertColumnsList(insert.Data.Keys);
string values = string.Join(", ", Parameterize(ctx, insert.Data.Values));

string intoFormat = " INTO {0}{1} VALUES ({2})";
var nextInsert = string.Format(intoFormat, table, columns, values);
const string intoFormat = " INTO {0}{1} VALUES ({2})";

ctx.RawSql += nextInsert;
sql.Append(string.Format(intoFormat, table, columns, values));
}

ctx.RawSql += " SELECT 1 FROM DUAL";
sql.Append(" SELECT 1 FROM DUAL");

ctx.RawSql = sql.ToString();

return ctx;
}
}
Expand Down
30 changes: 30 additions & 0 deletions QueryBuilder/Extensions/CollectionExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
using System.Collections.Generic;
using System.Linq;

namespace SqlKata.Extensions
{
public static class CollectionExtensions
{
public static Dictionary<string, object> MergeKeysAndValues(this List<string> keys, List<object> values)
{
var data = new Dictionary<string, object>();

for (var i = 0; i < keys.Count; i++)
{
data.Add(keys[i], values[i]);
Copy link

Copilot AI Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Potential IndexOutOfRangeException if values list is shorter than keys list. The validation should happen in this method or the caller should guarantee equal lengths.

Copilot uses AI. Check for mistakes.
}

return data;
}

public static Dictionary<string, object> CreateDictionary(this IEnumerable<KeyValuePair<string, object>> values)
{
if (values is Dictionary<string, object> dictionary)
{
return dictionary;
}

return values.ToDictionary(x => x.Key, x => x.Value);
}
}
}
8 changes: 5 additions & 3 deletions QueryBuilder/Helper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,14 +96,16 @@ public static string ReplaceAll(string subject, string match, Func<int, string>

public static string JoinArray(string glue, IEnumerable array)
{
var result = new List<string>();
var result = new StringBuilder();

foreach (var item in array)
{
result.Add(item.ToString());
result.Append(item + glue);
Copy link

Copilot AI Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

String concatenation using the + operator within a loop creates unnecessary string objects. Use StringBuilder.Append() with separate calls for better performance.

Suggested change
result.Append(item + glue);
result.Append(item);
result.Append(glue);

Copilot uses AI. Check for mistakes.
}

return string.Join(glue, result);
result.Length -= glue.Length;

return result.ToString().Trim();
Comment on lines +106 to +108
Copy link

Copilot AI Sep 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This approach will fail when the array is empty, causing result.Length to become negative. Add a check for empty arrays before attempting to modify the length.

Copilot uses AI. Check for mistakes.
}

public static string ExpandParameters(string sql, string placeholder, object[] bindings)
Expand Down
Loading