Skip to content

Commit f2d4996

Browse files
sfc-gh-pcyreksfc-gh-pcyrek
authored
SNOW-2021012-first-round-cicd-tests-optimzation (#1171)
1 parent 63ace70 commit f2d4996

File tree

8 files changed

+91
-67
lines changed

8 files changed

+91
-67
lines changed

‎Snowflake.Data.Tests/IcebergTests/TestIcebergTable.cs‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ tx varchar(16777216),
3232
ntz timestamp_ntz(6),
3333
ltz timestamp_ltz(6),
3434
bi binary(5),
35-
ar array(number(10,0)),
36-
ob object(a number(10,0), b varchar),
35+
ar array(number(10,0)),
36+
ob object(a number(10,0), b varchar),
3737
ma map(varchar, varchar)";
3838
private const string SqlCreateHybridTableColumns = @"id number(10,0) not null primary key,
3939
nu number(10,0),
@@ -73,7 +73,7 @@ public TestIcebergTable(ResultFormat resultFormat)
7373
}
7474

7575
[Test]
76-
[Ignore("TODO: Enable when features available on the automated tests environment")]
76+
[Ignore("Not a scope for CICD")]
7777
public void TestInsertPlainText()
7878
{
7979
// Arrange
@@ -83,12 +83,12 @@ public void TestInsertPlainText()
8383
SetResultFormat(conn);
8484

8585
// Act
86-
conn.ExecuteNonQuery(@$"insert into {TableNameIceberg} ({SqlColumnsSimpleTypes})
87-
values ({I32}, {I64}, {Dec}, {Dbl}, {Flt}, '{Txt}', {B1}, {B0},
88-
'{_dt.ToString(FormatYmd)}',
89-
'{_tm.ToString(FormatHms)}',
90-
'{_ntz.ToString(FormatYmdHms)}',
91-
'{_ltz.ToString(FormatYmdHmsfZ)}',
86+
conn.ExecuteNonQuery(@$"insert into {TableNameIceberg} ({SqlColumnsSimpleTypes})
87+
values ({I32}, {I64}, {Dec}, {Dbl}, {Flt}, '{Txt}', {B1}, {B0},
88+
'{_dt.ToString(FormatYmd)}',
89+
'{_tm.ToString(FormatHms)}',
90+
'{_ntz.ToString(FormatYmdHms)}',
91+
'{_ltz.ToString(FormatYmdHmsfZ)}',
9292
'{ByteArrayToHexString(_bi)}')");
9393

9494
// Assert
@@ -105,7 +105,7 @@ public void TestInsertPlainText()
105105

106106

107107
[Test]
108-
[Ignore("TODO: Enable when features available on the automated tests environment")]
108+
[Ignore("Not a scope for CICD tests")]
109109
public void TestInsertWithValueBinding()
110110
{
111111
// Arrange
@@ -130,7 +130,7 @@ public void TestInsertWithValueBinding()
130130
}
131131

132132
[Test]
133-
[Ignore("TODO: Enable when features available on the automated tests environment")]
133+
[Ignore("Not a scope for CICD")]
134134
public void TestUpdateWithValueBinding()
135135
{
136136
// Arrange
@@ -188,7 +188,7 @@ public void TestUpdateWithValueBinding()
188188
}
189189

190190
[Test]
191-
[Ignore("TODO: Enable when features available on the automated tests environment")]
191+
[Ignore("Not a scope for CICD")]
192192
public void TestJoin()
193193
{
194194
using (var conn = OpenConnection())
@@ -201,9 +201,9 @@ public void TestJoin()
201201
SetResultFormat(conn);
202202

203203
// Act
204-
var sql = @$"select i.nu1,i.nu2,i.nu3,i.nu4,i.f,i.tx,i.bt,i.bf,i.dt,i.tm,i.ntz,i.ltz,i.bi, h.id,h.nu,h.tx2
205-
from {TableNameIceberg} i
206-
join {TableNameHybrid} h
204+
var sql = @$"select i.nu1,i.nu2,i.nu3,i.nu4,i.f,i.tx,i.bt,i.bf,i.dt,i.tm,i.ntz,i.ltz,i.bi, h.id,h.nu,h.tx2
205+
from {TableNameIceberg} i
206+
join {TableNameHybrid} h
207207
on i.nu1 = h.nu order by i.nu1";
208208

209209
// Assert
@@ -233,7 +233,7 @@ nu number(10,0),
233233
}
234234

235235
[Test]
236-
[Ignore("TODO: Enable when features available on the automated tests environment")]
236+
[Ignore("Not a scope for CICD")]
237237
public void TestDelete()
238238
{
239239
using (var conn = OpenConnection())
@@ -257,7 +257,7 @@ public void TestDelete()
257257
}
258258

259259
[Test]
260-
[Ignore("TODO: Enable when features available on the automated tests environment")]
260+
[Ignore("Not a scope for CICD")]
261261
public void TestDeleteAll()
262262
{
263263
using (var conn = OpenConnection())
@@ -280,7 +280,7 @@ public void TestDeleteAll()
280280
}
281281

282282
[Test]
283-
[Ignore("TODO: Enable when features available on the automated tests environment")]
283+
[Ignore("Not a scope for CICD")]
284284
public void TestMultiStatement()
285285
{
286286
using (var conn = OpenConnection())
@@ -307,7 +307,7 @@ public void TestMultiStatement()
307307
}
308308

309309
[Test]
310-
[Ignore("TODO: Enable when features available on the automated tests environment")]
310+
[Ignore("Not a scope for CICD")]
311311
public void TestBatchInsertForLargeData()
312312
{
313313
using (var conn = OpenConnection())
@@ -336,16 +336,16 @@ public void TestBatchInsertForLargeData()
336336
}
337337

338338
[Test]
339-
[Ignore("TODO: Enable when features available on the automated tests environment")]
339+
[Ignore("Not a scope for CICD")]
340340
public void TestStructuredTypesAsJsonString()
341341
{
342342
using (var conn = OpenConnection())
343343
{
344344
SetResultFormat(conn);
345345
CreateIcebergTable(conn);
346-
var sql = @$"insert into {TableNameIceberg} ({SqlColumnsStructureTypes})
347-
select
348-
[1,2,3]::ARRAY(number),
346+
var sql = @$"insert into {TableNameIceberg} ({SqlColumnsStructureTypes})
347+
select
348+
[1,2,3]::ARRAY(number),
349349
{{'a' : 1, 'b': 'two'}}::OBJECT(a number, b varchar),
350350
{{'4':'one', '5': 'two', '6': 'three'}}::MAP(varchar, varchar)
351351
";

‎Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsAsyncIT.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1+
using System;
12
using System.Data.Common;
23
using System.Threading;
34
using System.Threading.Tasks;
45
using Moq;
56
using NUnit.Framework;
67
using Snowflake.Data.Client;
7-
using Snowflake.Data.Core;
88
using Snowflake.Data.Core.Session;
99
using Snowflake.Data.Log;
1010
using Snowflake.Data.Tests.Mock;
@@ -125,10 +125,10 @@ public async Task TestMinPoolSizeAsync()
125125

126126
// act
127127
await connection.OpenAsync().ConfigureAwait(false);
128-
Thread.Sleep(3000);
129128

130129
// assert
131130
var pool = SnowflakeDbConnectionPool.GetPool(connection.ConnectionString);
131+
Awaiter.WaitUntilConditionOrTimeout(() => pool.GetCurrentPoolSize() == 3, TimeSpan.FromMilliseconds(1000));
132132
Assert.AreEqual(3, pool.GetCurrentPoolSize());
133133

134134
// cleanup
@@ -182,7 +182,7 @@ public void TestConcurrentConnectionPoolingAsync()
182182
// add test case name in connection string to make in unique for each test case
183183
// set short expiration timeout to cover the case that connection expired
184184
string connStr = ConnectionString + ";application=TestConcurrentConnectionPoolingAsync2;ExpirationTimeout=3";
185-
ConnectionSinglePoolCacheAsyncIT.ConcurrentPoolingAsyncHelper(connStr, true, 7, 100, 2);
185+
ConnectionSinglePoolCacheAsyncIT.ConcurrentPoolingAsyncHelper(connStr, true, 5, 5, 3);
186186
}
187187

188188
[Test(Description = "test connection pooling with concurrent connection and using async calls no close call for connection. Connection is closed when Dispose() is called by framework.")]
@@ -191,7 +191,7 @@ public void TestConcurrentConnectionPoolingDisposeAsync()
191191
// add test case name in connection string to make in unique for each test case
192192
// set short expiration timeout to cover the case that connection expired
193193
string connStr = ConnectionString + ";application=TestConcurrentConnectionPoolingDisposeAsync2;ExpirationTimeout=3";
194-
ConnectionSinglePoolCacheAsyncIT.ConcurrentPoolingAsyncHelper(connStr, false, 7, 100, 2);
194+
ConnectionSinglePoolCacheAsyncIT.ConcurrentPoolingAsyncHelper(connStr, false, 5, 5, 3);
195195
}
196196
}
197197
}

‎Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
173173
}
174174

175175
[Test]
176-
[Retry(2)]
176+
[Retry(3)]
177177
public void TestWaitInAQueueForAnIdleSession()
178178
{
179179
// arrange
@@ -321,7 +321,7 @@ public void TestNewConnectionPoolClean()
321321
public void TestConnectionPoolExpirationWorks()
322322
{
323323
// arrange
324-
const int ExpirationTimeoutInSeconds = 10;
324+
const int ExpirationTimeoutInSeconds = 1;
325325
var connectionString = ConnectionString + $"expirationTimeout={ExpirationTimeoutInSeconds};maxPoolSize=4;minPoolSize=2";
326326
var pool = SnowflakeDbConnectionPool.GetPoolInternal(connectionString);
327327
Assert.AreEqual(0, pool.GetCurrentPoolSize());
@@ -369,10 +369,10 @@ public void TestMinPoolSize()
369369

370370
// act
371371
connection.Open();
372-
Thread.Sleep(3000);
373372

374373
// assert
375374
var pool = SnowflakeDbConnectionPool.GetPool(connection.ConnectionString);
375+
Awaiter.WaitUntilConditionOrTimeout(() => pool.GetCurrentPoolSize() == 3, TimeSpan.FromMilliseconds(1000));
376376
Assert.AreEqual(3, pool.GetCurrentPoolSize());
377377

378378
// cleanup
@@ -467,7 +467,11 @@ public void TestReturningCancelledSessionsToThePool([Values] bool cancelAsync)
467467
}
468468
}, CancellationToken.None);
469469

470-
Thread.Sleep(2000);
470+
Awaiter.WaitUntilConditionOrTimeout(() =>
471+
{
472+
var state = pool.GetCurrentState();
473+
return state.IdleSessionsCount == 0 && state.BusySessionsCount == 1;
474+
}, TimeSpan.FromMilliseconds(1000));
471475

472476
// one busy session
473477
Assert.AreEqual(0, pool.GetCurrentState().IdleSessionsCount);

‎Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,8 @@ void ThreadProcess1(string connstr)
6868
var conn1 = new SnowflakeDbConnection();
6969
conn1.ConnectionString = connstr;
7070
conn1.Open();
71-
Thread.Sleep(1000);
7271
conn1.Close();
73-
Thread.Sleep(4000);
72+
Thread.Sleep(1000);
7473
Assert.AreEqual(ConnectionState.Closed, conn1.State);
7574
}
7675

@@ -80,7 +79,7 @@ void ThreadProcess2(string connstr)
8079
conn1.ConnectionString = connstr;
8180
conn1.Open();
8281

83-
Thread.Sleep(5000);
82+
Thread.Sleep(1000);
8483
SFStatement statement = new SFStatement(conn1.SfSession);
8584
SFBaseResultSet resultSet = statement.Execute(0, "select 1", null, false, false);
8685
Assert.AreEqual(true, resultSet.Next());

‎Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheAsyncIT.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,9 @@ public void TestConcurrentConnectionPoolingAsync()
128128
{
129129
// add test case name in connection string to make in unique for each test case
130130
string connStr = ConnectionString + ";application=TestConcurrentConnectionPoolingAsync";
131-
SnowflakeDbConnectionPool.SetMaxPoolSize(10);
132-
SnowflakeDbConnectionPool.SetTimeout(3); // set short pooling timeout to cover the case that connection expired
133-
ConcurrentPoolingAsyncHelper(connStr, true, 12, 100, 100);
131+
SnowflakeDbConnectionPool.SetMaxPoolSize(3);
132+
SnowflakeDbConnectionPool.SetTimeout(1); // set short pooling timeout to cover the case that connection expired
133+
ConcurrentPoolingAsyncHelper(connStr, true, 5, 5, 3);
134134
SnowflakeDbConnectionPool.SetTimeout(3600);
135135
}
136136

@@ -139,9 +139,9 @@ public void TestConcurrentConnectionPoolingDisposeAsync()
139139
{
140140
// add test case name in connection string to make in unique for each test case
141141
string connStr = ConnectionString + ";application=TestConcurrentConnectionPoolingDisposeAsync";
142-
SnowflakeDbConnectionPool.SetMaxPoolSize(10);
143-
SnowflakeDbConnectionPool.SetTimeout(3); // set short pooling timeout to cover the case that connection expired
144-
ConcurrentPoolingAsyncHelper(connStr, false, 12, 100, 100);
142+
SnowflakeDbConnectionPool.SetMaxPoolSize(3);
143+
SnowflakeDbConnectionPool.SetTimeout(1); // set short pooling timeout to cover the case that connection expired
144+
ConcurrentPoolingAsyncHelper(connStr, false, 5, 5, 3);
145145
SnowflakeDbConnectionPool.SetTimeout(3600);
146146
}
147147

‎Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ public void TestBasicConnectionPool()
5656
public void TestConcurrentConnectionPooling()
5757
{
5858
// add test case name in connection string to make in unique for each test case
59-
string connStr = ConnectionString + ";application=TestConcurrentConnectionPooling";
59+
string connStr = ConnectionString + ";application=TestConcurrentConnectionPooling;";
60+
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
6061
ConcurrentPoolingHelper(connStr, true);
6162
}
6263

@@ -67,21 +68,22 @@ public void TestConcurrentConnectionPooling()
6768
public void TestConcurrentConnectionPoolingDispose()
6869
{
6970
// add test case name in connection string to make in unique for each test case
70-
string connStr = ConnectionString + ";application=TestConcurrentConnectionPoolingNoClose";
71+
string connStr = ConnectionString + ";application=TestConcurrentConnectionPoolingNoClose;";
72+
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
7173
ConcurrentPoolingHelper(connStr, false);
7274
}
7375

7476
static void ConcurrentPoolingHelper(string connectionString, bool closeConnection)
7577
{
7678
// thread number a bit larger than pool size so some connections
7779
// would fail on pooling while some connections could success
78-
const int ThreadNum = 12;
80+
const int ThreadNum = 3;
7981
// set short pooling timeout to cover the case that connection expired
80-
const int PoolTimeout = 3;
82+
const int PoolTimeout = 1;
8183

8284
// reset to default settings in case it changed by other test cases
8385
Assert.AreEqual(true, SnowflakeDbConnectionPool.GetPool(connectionString).GetPooling()); // to instantiate pool
84-
SnowflakeDbConnectionPool.SetMaxPoolSize(10);
86+
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
8587
SnowflakeDbConnectionPool.SetTimeout(PoolTimeout);
8688

8789
var threads = new Task[ThreadNum];
@@ -98,7 +100,7 @@ static void ConcurrentPoolingHelper(string connectionString, bool closeConnectio
98100
// thead to execute query with new connection in a loop
99101
static void QueryExecutionThread(string connectionString, bool closeConnection)
100102
{
101-
for (int i = 0; i < 100; i++)
103+
for (int i = 0; i < 10; i++)
102104
{
103105
using (DbConnection conn = new SnowflakeDbConnection(connectionString))
104106
{
@@ -347,8 +349,8 @@ public void TestReleaseConnectionWhenRollbackFails()
347349
public void TestCloseSessionAfterTimeout()
348350
{
349351
// arrange
350-
const int SessionTimeoutSeconds = 2;
351-
const int TimeForBackgroundSessionCloseMillis = 2000;
352+
const int SessionTimeoutSeconds = 1;
353+
const int TimeForBackgroundSessionCloseMillis = 1000;
352354
SnowflakeDbConnectionPool.SetTimeout(SessionTimeoutSeconds);
353355
var conn1 = new SnowflakeDbConnection(ConnectionString);
354356
conn1.Open();
@@ -360,7 +362,8 @@ public void TestCloseSessionAfterTimeout()
360362

361363
// act
362364
conn2.Open(); // it gets a session from the caching pool firstly closing session of conn1 in background
363-
Thread.Sleep(TimeForBackgroundSessionCloseMillis); // wait for closing expired session
365+
366+
Awaiter.WaitUntilConditionOrTimeout(() => !session.IsEstablished(), TimeSpan.FromMilliseconds(TimeForBackgroundSessionCloseMillis));
364367

365368
// assert
366369
Assert.IsFalse(session.IsEstablished());

0 commit comments

Comments
 (0)