Skip to content
This repository was archived by the owner on Dec 24, 2022. It is now read-only.

Commit cc41b61

Browse files
mythzmythz
authored
Merge pull request #257 from ServiceStack/delete-all-sscan
Use `SSCAN` for `DeleteAll` by default to avoid pulling large sets into memory.
2 parents b96308f + 0e63be8 commit cc41b61

File tree

7 files changed

+154
-22
lines changed

7 files changed

+154
-22
lines changed

‎src/ServiceStack.Redis/Generic/RedisTypedClient.Async.cs

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -131,13 +131,25 @@ async Task IEntityStoreAsync<T>.DeleteByIdsAsync(IEnumerable ids, CancellationTo
131131

132132
async Task IEntityStoreAsync<T>.DeleteAllAsync(CancellationToken token)
133133
{
134-
var ids = await AsyncClient.GetAllItemsFromSetAsync(this.TypeIdsSetKey, token).ConfigureAwait(false);
135-
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
136-
if (urnKeys.Count > 0)
134+
await DeleteAllAsync(0,RedisConfig.DeleteAllBatchSize, token).ConfigureAwait(false);
135+
}
136+
137+
private async Task DeleteAllAsync(ulong cursor, int pageSize, CancellationToken token)
138+
{
139+
var callCount = 0;
140+
while (cursor != 0 || callCount == 0)
137141
{
138-
await AsyncClient.RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
139-
await AsyncClient.RemoveEntryAsync(new[] { this.TypeIdsSetKey }, token).ConfigureAwait(false);
142+
var scanResult = await AsyncNative.SScanAsync(this.TypeIdsSetKey, cursor, pageSize, token: token).ConfigureAwait(false);
143+
callCount++;
144+
cursor = scanResult.Cursor;
145+
var ids = scanResult.Results.Select(x => Encoding.UTF8.GetString(x)).ToList();
146+
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
147+
if (urnKeys.Count > 0)
148+
{
149+
await AsyncClient.RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
150+
}
140151
}
152+
await AsyncClient.RemoveEntryAsync(new[] { this.TypeIdsSetKey }, token).ConfigureAwait(false);
141153
}
142154

143155
async ValueTask<List<T>> IRedisTypedClientAsync<T>.GetValuesAsync(List<string> keys, CancellationToken token)

‎src/ServiceStack.Redis/Generic/RedisTypedClient.cs

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -466,16 +466,28 @@ public void DeleteByIds(IEnumerable ids)
466466
}
467467
}
468468

469-
public void DeleteAll()
469+
private void DeleteAll(ulong cursor, int pageSize)
470470
{
471-
var ids = client.GetAllItemsFromSet(this.TypeIdsSetKey);
472-
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
473-
if (urnKeys.Count > 0)
471+
var callCount = 0;
472+
while (cursor != 0 || callCount == 0)
474473
{
475-
476-
this.RemoveEntry(urnKeys.ToArray());
477-
this.RemoveEntry(this.TypeIdsSetKey);
474+
var scanResult = client.SScan(this.TypeIdsSetKey, cursor, pageSize);
475+
callCount++;
476+
cursor = scanResult.Cursor;
477+
var ids = scanResult.Results.Select(x => Encoding.UTF8.GetString(x)).ToList();
478+
var urnKeys = ids.Map(t => client.UrnKey<T>(t));
479+
if (urnKeys.Count > 0)
480+
{
481+
this.RemoveEntry(urnKeys.ToArray());
482+
}
478483
}
484+
485+
this.RemoveEntry(this.TypeIdsSetKey);
486+
}
487+
488+
public void DeleteAll()
489+
{
490+
DeleteAll(0,RedisConfig.DeleteAllBatchSize);
479491
}
480492

481493
#endregion

‎src/ServiceStack.Redis/RedisClient.Async.cs

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -658,15 +658,27 @@ async Task IEntityStoreAsync.DeleteByIdsAsync<T>(ICollection ids, CancellationTo
658658
}
659659

660660
async Task IEntityStoreAsync.DeleteAllAsync<T>(CancellationToken token)
661+
{
662+
await DeleteAllAsync<T>(0, RedisConfig.DeleteAllBatchSize, token).ConfigureAwait(false);
663+
}
664+
665+
private async Task DeleteAllAsync<T>(ulong cursor, int pageSize, CancellationToken token)
661666
{
662667
var typeIdsSetKey = this.GetTypeIdsSetKey<T>();
663-
var ids = await AsAsync().GetAllItemsFromSetAsync(typeIdsSetKey, token).ConfigureAwait(false);
664-
if (ids.Count > 0)
668+
var callCount = 0;
669+
while (cursor != 0 || callCount == 0)
665670
{
666-
var urnKeys = ids.ToList().ConvertAll(UrnKey<T>);
667-
await AsAsync().RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
668-
await AsAsync().RemoveAsync(typeIdsSetKey, token).ConfigureAwait(false);
671+
var scanResult = await NativeAsync.SScanAsync(typeIdsSetKey, cursor, pageSize, token: token).ConfigureAwait(false);
672+
callCount++;
673+
cursor = scanResult.Cursor;
674+
var ids = scanResult.Results.Select(x => x.FromUtf8Bytes());
675+
var urnKeys = ids.Map(t => AsAsync().UrnKey<T>(t));
676+
if (urnKeys.Count > 0)
677+
{
678+
await AsAsync().RemoveEntryAsync(urnKeys.ToArray(), token).ConfigureAwait(false);
679+
}
669680
}
681+
await AsAsync().RemoveEntryAsync(new[] { typeIdsSetKey }, token).ConfigureAwait(false);
670682
}
671683

672684
ValueTask<List<string>> IRedisClientAsync.SearchSortedSetAsync(string setId, string start, string end, int? skip, int? take, CancellationToken token)

‎src/ServiceStack.Redis/RedisClient.cs

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -830,15 +830,28 @@ public void DeleteByIds<T>(ICollection ids)
830830
}
831831

832832
public void DeleteAll<T>()
833+
{
834+
DeleteAll<T>(0,RedisConfig.DeleteAllBatchSize);
835+
}
836+
837+
private void DeleteAll<T>(ulong cursor, int pageSize)
833838
{
834839
var typeIdsSetKey = this.GetTypeIdsSetKey<T>();
835-
var ids = this.GetAllItemsFromSet(typeIdsSetKey);
836-
if (ids.Count > 0)
840+
var callCount = 0;
841+
while (cursor != 0 || callCount == 0)
837842
{
838-
var urnKeys = ids.ToList().ConvertAll(UrnKey<T>);
839-
this.RemoveEntry(urnKeys.ToArray());
840-
this.Remove(typeIdsSetKey);
843+
var scanResult = this.SScan(typeIdsSetKey, cursor, pageSize);
844+
callCount++;
845+
cursor = scanResult.Cursor;
846+
var ids = scanResult.Results.Select(x => x.FromUtf8Bytes());
847+
var urnKeys = ids.Map(t => this.UrnKey(t));
848+
if (urnKeys.Count > 0)
849+
{
850+
this.RemoveEntry(urnKeys.ToArray());
851+
}
841852
}
853+
854+
this.RemoveEntry(typeIdsSetKey);
842855
}
843856

844857
public RedisClient CloneClient()

‎src/ServiceStack.Redis/RedisConfig.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public class RedisConfig
7272
/// </summary>
7373
public static int BufferPoolMaxSize = 500000;
7474

75+
/// <summary>
76+
/// The DeleteAll Batch Size is the number of keys returned each SSCAN when using DeleteAll on the RedisTypedClient.
77+
/// </summary>
78+
public static int DeleteAllBatchSize = 1000;
79+
7580
/// <summary>
7681
/// Whether Connections to Master hosts should be verified they're still master instances (default true)
7782
/// </summary>

‎tests/ServiceStack.Redis.Tests/Generic/RedisTypedClientTests.Async.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,46 @@ public async Task Can_Delete_All_Items()
121121
Assert.That(await RedisTyped.GetByIdAsync("key"), Is.Null);
122122

123123
}
124+
125+
[Test]
126+
public async Task Can_Delete_All_Items_multiple_batches()
127+
{
128+
// Clear previous usage
129+
await RedisAsync.DeleteAsync(RedisRaw.GetTypeIdsSetKey<CacheRecord>());
130+
var cachedRecord = new CacheRecord
131+
{
132+
Id = "key",
133+
Children = {
134+
new CacheRecordChild { Id = "childKey", Data = "data" }
135+
}
136+
};
137+
138+
var exists = RedisRaw.Exists(RedisRaw.GetTypeIdsSetKey(typeof(CacheRecord)));
139+
Assert.That(exists, Is.EqualTo(0));
140+
141+
await RedisTyped.StoreAsync(cachedRecord);
142+
143+
exists = RedisRaw.Exists(RedisRaw.GetTypeIdsSetKey(typeof(CacheRecord)));
144+
Assert.That(exists, Is.EqualTo(1));
145+
146+
RedisConfig.DeleteAllBatchSize = 5;
147+
148+
for (int i = 0; i < 50; i++)
149+
{
150+
cachedRecord.Id = "key" + i;
151+
await RedisTyped.StoreAsync(cachedRecord);
152+
}
153+
154+
Assert.That(await RedisTyped.GetByIdAsync("key"), Is.Not.Null);
155+
156+
await RedisTyped.DeleteAllAsync();
157+
158+
Assert.That(await RedisTyped.GetByIdAsync("key"), Is.Null);
159+
160+
exists = RedisRaw.Exists(RedisRaw.GetTypeIdsSetKey(typeof(CacheRecord)));
161+
Assert.That(exists, Is.EqualTo(0));
162+
163+
}
124164
}
125165

126166
}

‎tests/ServiceStack.Redis.Tests/Generic/RedisTypedClientTests.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,45 @@ public void Can_Delete_All_Items()
117117
RedisTyped.DeleteAll();
118118

119119
Assert.That(RedisTyped.GetById("key"), Is.Null);
120+
}
121+
122+
[Test]
123+
public void Can_Delete_All_Items_multiple_batches()
124+
{
125+
// Clear previous usage
126+
Redis.Delete(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
127+
var cachedRecord = new CacheRecord
128+
{
129+
Id = "key",
130+
Children = {
131+
new CacheRecordChild { Id = "childKey", Data = "data" }
132+
}
133+
};
120134

135+
var exists = Redis.Exists(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
136+
Assert.That(exists, Is.EqualTo(0));
137+
138+
RedisTyped.Store(cachedRecord);
139+
140+
exists = Redis.Exists(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
141+
142+
Assert.That(exists, Is.EqualTo(1));
143+
144+
RedisConfig.DeleteAllBatchSize = 5;
145+
146+
for (int i = 0; i < 50; i++)
147+
{
148+
cachedRecord.Id = "key" + i;
149+
RedisTyped.Store(cachedRecord);
150+
}
151+
152+
Assert.That(RedisTyped.GetById("key"), Is.Not.Null);
153+
154+
RedisTyped.DeleteAll();
155+
156+
exists = Redis.Exists(Redis.GetTypeIdsSetKey(typeof(CacheRecord)));
157+
Assert.That(exists, Is.EqualTo(0));
158+
Assert.That(RedisTyped.GetById("key"), Is.Null);
121159
}
122160
}
123161

0 commit comments

Comments
 (0)