Skip to content
This repository was archived by the owner on Dec 24, 2022. It is now read-only.

Commit 0e63be8

Browse files
LayoricLayoric
committed
Use RedisConfig for new static config option.
Add tests for deletes bigger than batch size. Change to use while loop rather than recursion.
1 parent a7fb93e commit 0e63be8

File tree

7 files changed

+137
-46
lines changed

7 files changed

+137
-46
lines changed

‎src/ServiceStack.Redis/Generic/RedisTypedClient.Async.cs

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -131,24 +131,25 @@ async Task IEntityStoreAsync<T>.DeleteByIdsAsync(IEnumerable ids, CancellationTo
131131

132132
async Task IEntityStoreAsync<T>.DeleteAllAsync(CancellationToken token)
133133
{
134-
await DeleteAllAsync(0,1000, token).ConfigureAwait(false);
134+
await DeleteAllAsync(0,RedisConfig.DeleteAllBatchSize, token).ConfigureAwait(false);
135135
}
136136

137137
private async Task DeleteAllAsync(ulong cursor, int pageSize, CancellationToken token)
138138
{
139-
var scanResult = await AsyncNative.SScanAsync(this.TypeIdsSetKey, cursor, pageSize, token: token).ConfigureAwait(false);
140-
var lastCursor = scanResult.Cursor;
141-
var ids = scanResult.Results.Select(x => Encoding.UTF8.GetString(x)).ToList();
142-
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
143-
if (urnKeys.Count > 0)
139+
var callCount = 0;
140+
while (cursor != 0 || callCount == 0)
144141
{
145-
await AsyncClient.RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
142+
var scanResult = await AsyncNative.SScanAsync(this.TypeIdsSetKey, cursor, pageSize, token: token).ConfigureAwait(false);
143+
callCount++;
144+
cursor = scanResult.Cursor;
145+
var ids = scanResult.Results.Select(x => Encoding.UTF8.GetString(x)).ToList();
146+
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
147+
if (urnKeys.Count > 0)
148+
{
149+
await AsyncClient.RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
150+
}
146151
}
147-
148-
if (lastCursor != 0)
149-
await DeleteAllAsync(lastCursor, pageSize, token).ConfigureAwait(false);
150-
else
151-
await AsyncClient.RemoveEntryAsync(new[] { this.TypeIdsSetKey }, token).ConfigureAwait(false);
152+
await AsyncClient.RemoveEntryAsync(new[] { this.TypeIdsSetKey }, token).ConfigureAwait(false);
152153
}
153154

154155
async ValueTask<List<T>> IRedisTypedClientAsync<T>.GetValuesAsync(List<string> keys, CancellationToken token)

‎src/ServiceStack.Redis/Generic/RedisTypedClient.cs

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -468,23 +468,26 @@ public void DeleteByIds(IEnumerable ids)
468468

469469
private void DeleteAll(ulong cursor, int pageSize)
470470
{
471-
var scanResult = client.SScan(this.TypeIdsSetKey, cursor, pageSize);
472-
var resultCursor = scanResult.Cursor;
473-
var ids = scanResult.Results.Select(x => Encoding.UTF8.GetString(x)).ToList();
474-
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
475-
if (urnKeys.Count > 0)
471+
var callCount = 0;
472+
while (cursor != 0 || callCount == 0)
476473
{
477-
this.RemoveEntry(urnKeys.ToArray());
474+
var scanResult = client.SScan(this.TypeIdsSetKey, cursor, pageSize);
475+
callCount++;
476+
cursor = scanResult.Cursor;
477+
var ids = scanResult.Results.Select(x => Encoding.UTF8.GetString(x)).ToList();
478+
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
479+
if (urnKeys.Count > 0)
480+
{
481+
this.RemoveEntry(urnKeys.ToArray());
482+
}
478483
}
479-
if(resultCursor != 0)
480-
DeleteAll(resultCursor,pageSize);
481-
else
482-
this.RemoveEntry(this.TypeIdsSetKey);
484+
485+
this.RemoveEntry(this.TypeIdsSetKey);
483486
}
484487

485488
public void DeleteAll()
486489
{
487-
DeleteAll(0,1000);
490+
DeleteAll(0,RedisConfig.DeleteAllBatchSize);
488491
}
489492

490493
#endregion

‎src/ServiceStack.Redis/RedisClient.Async.cs

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -659,25 +659,26 @@ async Task IEntityStoreAsync.DeleteByIdsAsync<T>(ICollection ids, CancellationTo
659659

660660
async Task IEntityStoreAsync.DeleteAllAsync<T>(CancellationToken token)
661661
{
662-
await DeleteAllAsync<T>(0, 1000, token).ConfigureAwait(false);
662+
await DeleteAllAsync<T>(0, RedisConfig.DeleteAllBatchSize, token).ConfigureAwait(false);
663663
}
664664

665665
private async Task DeleteAllAsync<T>(ulong cursor, int pageSize, CancellationToken token)
666666
{
667667
var typeIdsSetKey = this.GetTypeIdsSetKey<T>();
668-
var scanResult = await NativeAsync.SScanAsync(typeIdsSetKey, cursor, pageSize, token: token).ConfigureAwait(false);
669-
var lastCursor = scanResult.Cursor;
670-
var ids = scanResult.Results.Select(x => x.FromUtf8Bytes());
671-
var urnKeys = ids.Map(t => AsAsync().UrnKey<T>(t));
672-
if (urnKeys.Count > 0)
668+
var callCount = 0;
669+
while (cursor != 0 || callCount == 0)
673670
{
674-
await AsAsync().RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
671+
var scanResult = await NativeAsync.SScanAsync(typeIdsSetKey, cursor, pageSize, token: token).ConfigureAwait(false);
672+
callCount++;
673+
cursor = scanResult.Cursor;
674+
var ids = scanResult.Results.Select(x => x.FromUtf8Bytes());
675+
var urnKeys = ids.Map(t => AsAsync().UrnKey<T>(t));
676+
if (urnKeys.Count > 0)
677+
{
678+
await AsAsync().RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
679+
}
675680
}
676-
677-
if (lastCursor != 0)
678-
await DeleteAllAsync<T>(lastCursor, pageSize, token).ConfigureAwait(false);
679-
else
680-
await AsAsync().RemoveEntryAsync(new[] { typeIdsSetKey }, token).ConfigureAwait(false);
681+
await AsAsync().RemoveEntryAsync(new[] { typeIdsSetKey }, token).ConfigureAwait(false);
681682
}
682683

683684
ValueTask<List<string>> IRedisClientAsync.SearchSortedSetAsync(string setId, string start, string end, int? skip, int? take, CancellationToken token)

‎src/ServiceStack.Redis/RedisClient.cs

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -831,24 +831,27 @@ public void DeleteByIds<T>(ICollection ids)
831831

832832
public void DeleteAll<T>()
833833
{
834-
DeleteAll<T>(0,1000);
834+
DeleteAll<T>(0,RedisConfig.DeleteAllBatchSize);
835835
}
836836

837837
private void DeleteAll<T>(ulong cursor, int pageSize)
838838
{
839839
var typeIdsSetKey = this.GetTypeIdsSetKey<T>();
840-
var scanResult = this.SScan(typeIdsSetKey, cursor, pageSize);
841-
var resultCursor = scanResult.Cursor;
842-
var ids = scanResult.Results.Select(x => x.FromUtf8Bytes());
843-
var urnKeys = ids.Map(t => this.UrnKey(t));
844-
if (urnKeys.Count > 0)
840+
var callCount = 0;
841+
while (cursor != 0 || callCount == 0)
845842
{
846-
this.RemoveEntry(urnKeys.ToArray());
843+
var scanResult = this.SScan(typeIdsSetKey, cursor, pageSize);
844+
callCount++;
845+
cursor = scanResult.Cursor;
846+
var ids = scanResult.Results.Select(x => x.FromUtf8Bytes());
847+
var urnKeys = ids.Map(t => this.UrnKey(t));
848+
if (urnKeys.Count > 0)
849+
{
850+
this.RemoveEntry(urnKeys.ToArray());
851+
}
847852
}
848-
if(resultCursor != 0)
849-
DeleteAll<T>(resultCursor, pageSize);
850-
else
851-
this.RemoveEntry(typeIdsSetKey);
853+
854+
this.RemoveEntry(typeIdsSetKey);
852855
}
853856

854857
public RedisClient CloneClient()

‎src/ServiceStack.Redis/RedisConfig.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public class RedisConfig
7272
/// </summary>
7373
public static int BufferPoolMaxSize = 500000;
7474

75+
/// <summary>
76+
/// The DeleteAll Batch Size is the number of keys returned each SSCAN when using DeleteAll on the RedisTypedClient.
77+
/// </summary>
78+
public static int DeleteAllBatchSize = 1000;
79+
7580
/// <summary>
7681
/// Whether Connections to Master hosts should be verified they're still master instances (default true)
7782
/// </summary>

‎tests/ServiceStack.Redis.Tests/Generic/RedisTypedClientTests.Async.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,46 @@ public async Task Can_Delete_All_Items()
121121
Assert.That(await RedisTyped.GetByIdAsync("key"), Is.Null);
122122

123123
}
124+
125+
[Test]
126+
public async Task Can_Delete_All_Items_multiple_batches()
127+
{
128+
// Clear previous usage
129+
await RedisAsync.DeleteAsync(RedisRaw.GetTypeIdsSetKey<CacheRecord>());
130+
var cachedRecord = new CacheRecord
131+
{
132+
Id = "key",
133+
Children = {
134+
new CacheRecordChild { Id = "childKey", Data = "data" }
135+
}
136+
};
137+
138+
var exists = RedisRaw.Exists(RedisRaw.GetTypeIdsSetKey(typeof(CacheRecord)));
139+
Assert.That(exists, Is.EqualTo(0));
140+
141+
await RedisTyped.StoreAsync(cachedRecord);
142+
143+
exists = RedisRaw.Exists(RedisRaw.GetTypeIdsSetKey(typeof(CacheRecord)));
144+
Assert.That(exists, Is.EqualTo(1));
145+
146+
RedisConfig.DeleteAllBatchSize = 5;
147+
148+
for (int i = 0; i < 50; i++)
149+
{
150+
cachedRecord.Id = "key" + i;
151+
await RedisTyped.StoreAsync(cachedRecord);
152+
}
153+
154+
Assert.That(await RedisTyped.GetByIdAsync("key"), Is.Not.Null);
155+
156+
await RedisTyped.DeleteAllAsync();
157+
158+
Assert.That(await RedisTyped.GetByIdAsync("key"), Is.Null);
159+
160+
exists = RedisRaw.Exists(RedisRaw.GetTypeIdsSetKey(typeof(CacheRecord)));
161+
Assert.That(exists, Is.EqualTo(0));
162+
163+
}
124164
}
125165

126166
}

‎tests/ServiceStack.Redis.Tests/Generic/RedisTypedClientTests.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,45 @@ public void Can_Delete_All_Items()
117117
RedisTyped.DeleteAll();
118118

119119
Assert.That(RedisTyped.GetById("key"), Is.Null);
120+
}
121+
122+
[Test]
123+
public void Can_Delete_All_Items_multiple_batches()
124+
{
125+
// Clear previous usage
126+
Redis.Delete(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
127+
var cachedRecord = new CacheRecord
128+
{
129+
Id = "key",
130+
Children = {
131+
new CacheRecordChild { Id = "childKey", Data = "data" }
132+
}
133+
};
120134

135+
var exists = Redis.Exists(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
136+
Assert.That(exists, Is.EqualTo(0));
137+
138+
RedisTyped.Store(cachedRecord);
139+
140+
exists = Redis.Exists(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
141+
142+
Assert.That(exists, Is.EqualTo(1));
143+
144+
RedisConfig.DeleteAllBatchSize = 5;
145+
146+
for (int i = 0; i < 50; i++)
147+
{
148+
cachedRecord.Id = "key" + i;
149+
RedisTyped.Store(cachedRecord);
150+
}
151+
152+
Assert.That(RedisTyped.GetById("key"), Is.Not.Null);
153+
154+
RedisTyped.DeleteAll();
155+
156+
exists = Redis.Exists(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
157+
Assert.That(exists, Is.EqualTo(0));
158+
Assert.That(RedisTyped.GetById("key"), Is.Null);
121159
}
122160
}
123161

0 commit comments

Comments
 (0)