Skip to content

Commit be043e3

Browse files
committed
- 增加 IInsertOrUpdate 高性能方法 ExecuteMySqlBulkCopy;
1 parent dc65941 commit be043e3

File tree

5 files changed

+89
-10
lines changed

5 files changed

+89
-10
lines changed

Examples/base_entity/Program.cs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66
using FreeSql.Internal.CommonProvider;
77
using FreeSql.Internal.Model;
88
using FreeSql.Odbc.Default;
9-
using K4os.Hash.xxHash;
10-
using MySqlX.XDevAPI;
119
using NetTopologySuite.Geometries;
1210
using Newtonsoft.Json;
1311
using Newtonsoft.Json.Linq;
@@ -541,13 +539,13 @@ static void Main(string[] args)
541539
.UseConnectionString(FreeSql.DataType.Firebird, @"database=localhost:D:\fbdata\EXAMPLES.fdb;user=sysdba;password=123456;max pool size=5")
542540
//.UseQuoteSqlName(false)
543541

544-
//.UseConnectionString(FreeSql.DataType.MySql, "Data Source=127.0.0.1;Port=3306;User ID=root;Password=root;Initial Catalog=cccddd;Charset=utf8;SslMode=none;min pool size=1;Max pool size=2")
542+
.UseConnectionString(FreeSql.DataType.MySql, "Data Source=127.0.0.1;Port=3306;User ID=root;Password=root;Initial Catalog=cccddd;Charset=utf8;SslMode=none;min pool size=1;Max pool size=2;AllowLoadLocalInfile=true")
545543

546-
.UseConnectionString(FreeSql.DataType.SqlServer, "Data Source=.;Integrated Security=True;Initial Catalog=freesqlTest;Pooling=true;Max Pool Size=3;TrustServerCertificate=true")
544+
//.UseConnectionString(FreeSql.DataType.SqlServer, "Data Source=.;Integrated Security=True;Initial Catalog=freesqlTest;Pooling=true;Max Pool Size=3;TrustServerCertificate=true")
547545

548-
.UseConnectionString(FreeSql.DataType.PostgreSQL, "Host=192.168.164.10;Port=5432;Username=postgres;Password=123456;Database=tedb;Pooling=true;Maximum Pool Size=2")
546+
//.UseConnectionString(FreeSql.DataType.PostgreSQL, "Host=192.168.164.10;Port=5432;Username=postgres;Password=123456;Database=tedb;Pooling=true;Maximum Pool Size=2")
549547
//.UseConnectionString(FreeSql.DataType.PostgreSQL, "Host=192.168.164.10;Port=5432;Username=postgres;Password=123456;Database=toc;Pooling=true;Maximum Pool Size=2")
550-
.UseNameConvert(FreeSql.Internal.NameConvertType.ToLower)
548+
//.UseNameConvert(FreeSql.Internal.NameConvertType.ToLower)
551549

552550
//.UseConnectionString(FreeSql.DataType.Oracle, "user id=user1;password=123456;data source=//127.0.0.1:1521/XE;Pooling=true;Max Pool Size=2")
553551
//.UseNameConvert(FreeSql.Internal.NameConvertType.ToUpper)
@@ -586,7 +584,7 @@ static void Main(string[] args)
586584

587585
fsql.InsertOrUpdate<User1>()
588586
.SetSource(fsql.Select<User1>().ToList())
589-
.ExecutePgCopy();
587+
.ExecuteMySqlBulkCopy();
590588

591589
var updatejoin01 = fsql.Update<User1>()
592590
.Join(fsql.Select<UserGroup>(), (a, b) => a.GroupId == b.Id)

Examples/base_entity/base_entity.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
<ProjectReference Include="..\..\FreeSql\FreeSql.csproj" />
2828
<ProjectReference Include="..\..\Providers\FreeSql.Provider.Dameng\FreeSql.Provider.Dameng.csproj" />
2929
<ProjectReference Include="..\..\Providers\FreeSql.Provider.Firebird\FreeSql.Provider.Firebird.csproj" />
30-
<ProjectReference Include="..\..\Providers\FreeSql.Provider.MySql\FreeSql.Provider.MySql.csproj" />
30+
<ProjectReference Include="..\..\Providers\FreeSql.Provider.MySqlConnector\FreeSql.Provider.MySqlConnector.csproj" />
3131
<ProjectReference Include="..\..\Providers\FreeSql.Provider.Odbc\FreeSql.Provider.Odbc.csproj" />
3232
<ProjectReference Include="..\..\Providers\FreeSql.Provider.Oracle\FreeSql.Provider.Oracle.csproj" />
3333
<ProjectReference Include="..\..\Providers\FreeSql.Provider.PostgreSQL\FreeSql.Provider.PostgreSQL.csproj" />

FreeSql.DbContext/FreeSql.DbContext.xml

Lines changed: 18 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Providers/FreeSql.Provider.MySqlConnector/FreeSqlMySqlConnectorGlobalExtensions.cs

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,57 @@
1818
public static class FreeSqlMySqlConnectorGlobalExtensions
1919
{
2020
#region ExecuteMySqlBulkCopy
21-
21+
/// <summary>
22+
/// 批量插入或更新(操作的字段数量超过 2000 时收益大)<para></para>
23+
/// 实现原理:使用 MySqlBulkCopy 插入临时表,再执行 INSERT INTO t1 select * from #temp ON DUPLICATE KEY UPDATE ...
24+
/// </summary>
25+
/// <typeparam name="T"></typeparam>
26+
/// <param name="that"></param>
27+
/// <param name="bulkCopyTimeout"></param>
28+
/// <returns></returns>
29+
public static int ExecuteMySqlBulkCopy<T>(this IInsertOrUpdate<T> that, int? bulkCopyTimeout = null) where T : class
30+
{
31+
var upsert = that as InsertOrUpdateProvider<T>;
32+
if (upsert._source.Any() != true || upsert._tempPrimarys.Any() == false) return 0;
33+
var state = ExecuteMySqlBulkCopyState(upsert);
34+
return UpdateProvider.ExecuteBulkUpsert(upsert, state, insert => insert.ExecuteMySqlBulkCopy(bulkCopyTimeout));
35+
}
36+
static NativeTuple<string, string, string, string, string[]> ExecuteMySqlBulkCopyState<T>(InsertOrUpdateProvider<T> upsert) where T : class
37+
{
38+
if (upsert._source.Any() != true) return null;
39+
var _table = upsert._table;
40+
var _commonUtils = upsert._commonUtils;
41+
var updateTableName = upsert._tableRule?.Invoke(_table.DbName) ?? _table.DbName;
42+
var tempTableName = $"Temp_{Guid.NewGuid().ToString("N")}";
43+
if (upsert._orm.CodeFirst.IsSyncStructureToLower) tempTableName = tempTableName.ToLower();
44+
if (upsert._orm.CodeFirst.IsSyncStructureToUpper) tempTableName = tempTableName.ToUpper();
45+
if (upsert._connection == null && upsert._orm.Ado.TransactionCurrentThread != null)
46+
upsert.WithTransaction(upsert._orm.Ado.TransactionCurrentThread);
47+
var sb = new StringBuilder().Append("CREATE TEMPORARY TABLE ").Append(_commonUtils.QuoteSqlName(tempTableName)).Append(" ( ");
48+
foreach (var col in _table.Columns.Values)
49+
{
50+
sb.Append(" \r\n ").Append(_commonUtils.QuoteSqlName(col.Attribute.Name)).Append(" ").Append(col.Attribute.DbType.Replace("NOT NULL", ""));
51+
sb.Append(",");
52+
}
53+
var sql1 = sb.Remove(sb.Length - 1, 1).Append(" \r\n) Engine=InnoDB;").ToString();
54+
try
55+
{
56+
upsert._sourceSql = $"select __**__ from {tempTableName}";
57+
var sql2 = upsert.ToSql();
58+
if (string.IsNullOrWhiteSpace(sql2) == false)
59+
{
60+
var field = sql2.Substring(sql2.IndexOf("`(") + 2);
61+
field = field.Remove(field.IndexOf(upsert._sourceSql)).TrimEnd().TrimEnd(')');
62+
sql2 = sql2.Replace(upsert._sourceSql, $"select {field} from {tempTableName}");
63+
}
64+
var sql3 = $"DROP TABLE {_commonUtils.QuoteSqlName(tempTableName)}";
65+
return NativeTuple.Create(sql1, sql2, sql3, tempTableName, _table.Columns.Values.Select(a => a.Attribute.Name).ToArray());
66+
}
67+
finally
68+
{
69+
upsert._sourceSql = null;
70+
}
71+
}
2272
/// <summary>
2373
/// 批量更新(更新字段数量超过 2000 时收益大)<para></para>
2474
/// 实现原理:使用 MySqlBulkCopy 插入临时表,再使用 UPDATE INNER JOIN 联表更新
@@ -147,6 +197,13 @@ public static void ExecuteMySqlBulkCopy<T>(this IInsert<T> that, int? bulkCopyTi
147197
}
148198
#if net40
149199
#else
200+
public static Task<int> ExecuteMySqlBulkCopyAsync<T>(this IInsertOrUpdate<T> that, int? bulkCopyTimeout = null, CancellationToken cancellationToken = default) where T : class
201+
{
202+
var upsert = that as UpdateProvider<T>;
203+
if (upsert._source.Any() != true || upsert._tempPrimarys.Any() == false) return Task.FromResult(0);
204+
var state = ExecuteMySqlBulkCopyState(upsert);
205+
return UpdateProvider.ExecuteBulkUpdateAsync(upsert, state, insert => insert.ExecuteMySqlBulkCopyAsync(bulkCopyTimeout, cancellationToken));
206+
}
150207
public static Task<int> ExecuteMySqlBulkCopyAsync<T>(this IUpdate<T> that, int? bulkCopyTimeout = null, CancellationToken cancellationToken = default) where T : class
151208
{
152209
var update = that as UpdateProvider<T>;

Providers/FreeSql.Provider.PostgreSQL/PostgreSQLExtensions.cs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,14 @@ static NativeTuple<string, string, string, string, string[]> ExecutePgCopyState<
7171
sb.Clear();
7272
try
7373
{
74-
upsert._sourceSql = $"select * from {tempTableName}";
74+
upsert._sourceSql = $"select __**__ from {tempTableName}";
7575
var sql2 = upsert.ToSql();
76+
if (string.IsNullOrWhiteSpace(sql2) == false)
77+
{
78+
var field = sql2.Substring(sql2.IndexOf("\"(") + 2);
79+
field = field.Remove(field.IndexOf(upsert._sourceSql)).TrimEnd().TrimEnd(')');
80+
sql2 = sql2.Replace(upsert._sourceSql, $"select {field} from {tempTableName}");
81+
}
7682
var sql3 = $"DROP TABLE {_commonUtils.QuoteSqlName(tempTableName)}";
7783
return NativeTuple.Create(sql1, sql2, sql3, tempTableName, _table.Columns.Values.Select(a => a.Attribute.Name).ToArray());
7884
}

0 commit comments

Comments
 (0)