Skip to content
This repository was archived by the owner on Jun 21, 2023. It is now read-only.

Commit 768c80b

Browse files
shanashana
committed
Add a way of clearing old items...
... when refreshing the index.
1 parent ff8c52a commit 768c80b

File tree

5 files changed

+106
-10
lines changed

5 files changed

+106
-10
lines changed

‎src/GitHub.App/Caches/CacheIndex.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
using System.Reactive.Linq;
55
using Akavache;
66
using NullGuard;
7+
using System.Reactive;
8+
using System.Linq;
79

810
namespace GitHub.Caches
911
{
@@ -17,6 +19,7 @@ public static CacheIndex Create(string key)
1719
public CacheIndex()
1820
{
1921
Keys = new List<string>();
22+
OldKeys = new List<string>();
2023
}
2124

2225
public IObservable<CacheIndex> AddAndSave(IBlobCache cache, string indexKey, CacheItem item,
@@ -45,9 +48,21 @@ public static IObservable<CacheIndex> AddAndSaveToIndex(IBlobCache cache, string
4548
.Select(x => index));
4649
}
4750

51+
public IObservable<CacheIndex> Clear(IBlobCache cache, string indexKey, DateTimeOffset? absoluteExpiration = null)
52+
{
53+
OldKeys = Keys.ToList();
54+
Keys.Clear();
55+
UpdatedAt = DateTimeOffset.UtcNow;
56+
return cache
57+
.InvalidateObject<CacheIndex>(indexKey)
58+
.SelectMany(_ => cache.InsertObject(indexKey, this, absoluteExpiration))
59+
.Select(_ => this);
60+
}
61+
4862
[AllowNull]
4963
public string IndexKey {[return: AllowNull] get; set; }
5064
public List<string> Keys { get; set; }
5165
public DateTimeOffset UpdatedAt { get; set; }
66+
public List<string> OldKeys { get; set; }
5267
}
5368
}

‎src/GitHub.App/Extensions/AkavacheExtensions.cs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public static IObservable<byte[]> GetAndRefresh(
8484
{
8585
return Observable.Defer(() =>
8686
{
87-
var absoluteExpiration = blobCache.Scheduler.Now.ToUniversalTime() + maxCacheDuration;
87+
var absoluteExpiration = blobCache.Scheduler.Now + maxCacheDuration;
8888

8989
try
9090
{
@@ -142,6 +142,7 @@ public static IObservable<T> GetAndFetchLatestFromIndex<T>(
142142
this IBlobCache blobCache,
143143
string key,
144144
Func<IObservable<T>> fetchFunc,
145+
Action<T> removedItemsCallback,
145146
TimeSpan refreshInterval,
146147
TimeSpan maxCacheDuration)
147148
where T : CacheItem
@@ -155,6 +156,7 @@ public static IObservable<T> GetAndFetchLatestFromIndex<T>(
155156
return blobCache.GetAndFetchLatestFromIndex(
156157
key,
157158
fetchFunc,
159+
removedItemsCallback,
158160
createdAt => IsExpired(blobCache, createdAt, refreshInterval),
159161
absoluteExpiration);
160162
}
@@ -168,6 +170,7 @@ public static IObservable<T> GetAndFetchLatestFromIndex<T>(
168170
static IObservable<T> GetAndFetchLatestFromIndex<T>(this IBlobCache This,
169171
string key,
170172
Func<IObservable<T>> fetchFunc,
173+
Action<T> removedItemsCallback,
171174
Func<DateTimeOffset, bool> fetchPredicate = null,
172175
DateTimeOffset? absoluteExpiration = null,
173176
bool shouldInvalidateOnError = false)
@@ -177,6 +180,7 @@ static IObservable<T> GetAndFetchLatestFromIndex<T>(this IBlobCache This,
177180
.Select(x => Tuple.Create(x, fetchPredicate == null || !x.Keys.Any() || fetchPredicate(x.UpdatedAt)))
178181
.Where(predicateIsTrue => predicateIsTrue.Item2)
179182
.Select(x => x.Item1)
183+
.SelectMany(index => index.Clear(This, key, absoluteExpiration))
180184
.SelectMany(index =>
181185
{
182186
var fetchObs = fetchFunc().Catch<T, Exception>(ex =>
@@ -189,7 +193,14 @@ static IObservable<T> GetAndFetchLatestFromIndex<T>(this IBlobCache This,
189193

190194
return fetchObs
191195
.SelectMany(x => x.Save<T>(This, key, absoluteExpiration))
192-
.Do(x => index.AddAndSave(This, key, x, absoluteExpiration));
196+
.Do(x => index.AddAndSave(This, key, x, absoluteExpiration))
197+
.Finally(() =>
198+
{
199+
This.GetObjects<T>(index.OldKeys)
200+
.SelectMany(dict => dict.Values)
201+
.Do(item => removedItemsCallback(item))
202+
.Subscribe();
203+
});
193204
}));
194205

195206
var cache = Observable.Defer(() => This.GetOrCreateObject(key, () => CacheIndex.Create(key))

‎src/GitHub.App/Services/ModelService.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using NLog;
1616
using NullGuard;
1717
using Octokit;
18+
using GitHub.Collections;
1819

1920
namespace GitHub.Services
2021
{
@@ -134,7 +135,7 @@ public IObservable<AccountCacheItem> GetUserFromCache()
134135
return Observable.Defer(() => hostCache.GetObject<AccountCacheItem>("user"));
135136
}
136137

137-
public IObservable<IPullRequestModel> GetPullRequests(ISimpleRepositoryModel repo)
138+
public ITrackingCollection<IPullRequestModel> GetPullRequests(ISimpleRepositoryModel repo)
138139
{
139140
// Since the api to list pull requests returns all the data for each pr, cache each pr in its own entry
140141
// and also cache an index that contains all the keys for each pr. This way we can fetch prs in bulk
@@ -145,16 +146,22 @@ public IObservable<IPullRequestModel> GetPullRequests(ISimpleRepositoryModel rep
145146
var keyobs = GetUserFromCache()
146147
.Select(user => string.Format(CultureInfo.InvariantCulture, "{0}|{1}|pr", user.Login, repo.Name));
147148

148-
return Observable.Defer(() => keyobs
149+
TrackingCollection<IPullRequestModel> col = null;
150+
151+
var source = Observable.Defer(() => keyobs
149152
.SelectMany(key =>
150153
hostCache.GetAndFetchLatestFromIndex(key, () =>
151154
apiClient.GetPullRequestsForRepository(repo.CloneUrl.Owner, repo.CloneUrl.RepositoryName)
152-
.Select(PullRequestCacheItem.Create),
155+
.Select(PullRequestCacheItem.Create),
156+
item => col.RemoveItem(Create(item)),
153157
TimeSpan.FromMinutes(5),
154158
TimeSpan.FromDays(1))
155159
)
156160
.Select(Create)
157161
);
162+
163+
col = new TrackingCollection<IPullRequestModel>(source);
164+
return col;
158165
}
159166

160167
public IObservable<Unit> InvalidateAll()

‎src/GitHub.Exports.Reactive/Services/IModelService.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Reactive;
44
using GitHub.Models;
55
using GitHub.Caches;
6+
using GitHub.Collections;
67

78
namespace GitHub.Services
89
{
@@ -18,7 +19,7 @@ public interface IModelService : IDisposable
1819
IObservable<IReadOnlyList<IRepositoryModel>> GetRepositories();
1920
IObservable<IReadOnlyList<LicenseItem>> GetLicenses();
2021
IObservable<IReadOnlyList<GitIgnoreItem>> GetGitIgnoreTemplates();
21-
IObservable<IPullRequestModel> GetPullRequests(ISimpleRepositoryModel repo);
22+
ITrackingCollection<IPullRequestModel> GetPullRequests(ISimpleRepositoryModel repo);
2223
IObservable<Unit> InvalidateAll();
2324
}
2425
}

‎src/UnitTests/GitHub.App/Models/ModelServiceTests.cs‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -436,8 +436,7 @@ public async Task NonExpiredIndexReturnsCache()
436436
apiClient.GetPullRequestsForRepository(user.Login, repo.Name).Returns(prlive);
437437

438438
await modelService.InsertUser(new AccountCacheItem(user));
439-
var source = modelService.GetPullRequests(repo);
440-
var col = new TrackingCollection<IPullRequestModel>(source);
439+
var col = modelService.GetPullRequests(repo);
441440

442441
var count = 0;
443442
var evt = new ManualResetEvent(false);
@@ -498,8 +497,7 @@ public async Task ExpiredIndexReturnsLive()
498497
apiClient.GetPullRequestsForRepository(user.Login, repo.Name).Returns(prlive);
499498

500499
await modelService.InsertUser(new AccountCacheItem(user));
501-
var source = modelService.GetPullRequests(repo);
502-
var col = new TrackingCollection<IPullRequestModel>(source);
500+
var col = modelService.GetPullRequests(repo);
503501

504502
var count = 0;
505503
var evt = new ManualResetEvent(false);
@@ -515,6 +513,70 @@ public async Task ExpiredIndexReturnsLive()
515513

516514
Assert.Collection(col, col.Select(x => new Action<IPullRequestModel>(t => Assert.True(x.Title.StartsWith("Live")))).ToArray());
517515
}
516+
517+
[Fact]
518+
public async Task ExpiredIndexClearsItems()
519+
{
520+
var expected = 5;
521+
522+
var username = "octocat";
523+
var reponame = "repo";
524+
525+
var cache = new InMemoryBlobCache();
526+
var apiClient = Substitute.For<IApiClient>();
527+
var modelService = new ModelService(apiClient, cache, Substitute.For<IAvatarProvider>());
528+
var user = CreateOctokitUser(username);
529+
apiClient.GetUser().Returns(Observable.Return(user));
530+
apiClient.GetOrganizations().Returns(Observable.Empty<Organization>());
531+
var act = modelService.GetAccounts().ToEnumerable().First().First();
532+
533+
var repo = Substitute.For<ISimpleRepositoryModel>();
534+
repo.Name.Returns(reponame);
535+
repo.CloneUrl.Returns(new UriString("https://github.com/" + username + "/" + reponame));
536+
537+
var indexKey = string.Format(CultureInfo.InvariantCulture, "{0}|{1}|pr", user.Login, repo.Name);
538+
539+
var prcache = Enumerable.Range(1, expected)
540+
.Select(id => CreatePullRequest(user, id, ItemState.Open, "Cache " + id, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 0));
541+
542+
// seed the cache
543+
prcache
544+
.Select(item => new ModelService.PullRequestCacheItem(item))
545+
.Select(item => item.Save<ModelService.PullRequestCacheItem>(cache, indexKey).ToEnumerable().First())
546+
.SelectMany(item => CacheIndex.AddAndSaveToIndex(cache, indexKey, item).ToEnumerable())
547+
.ToList();
548+
549+
// expire the index
550+
var indexobj = await cache.GetObject<CacheIndex>(indexKey);
551+
indexobj.UpdatedAt = DateTimeOffset.UtcNow - TimeSpan.FromMinutes(6);
552+
await cache.InsertObject(indexKey, indexobj);
553+
554+
var prlive = Observable.Range(6, expected)
555+
.Select(id => CreatePullRequest(user, id, ItemState.Open, "Live " + id, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 0))
556+
.DelaySubscription(TimeSpan.FromMilliseconds(10));
557+
558+
apiClient.GetPullRequestsForRepository(user.Login, repo.Name).Returns(prlive);
559+
560+
await modelService.InsertUser(new AccountCacheItem(user));
561+
var col = modelService.GetPullRequests(repo);
562+
563+
var count = 0;
564+
var evt = new ManualResetEvent(false);
565+
col.Subscribe(t =>
566+
{
567+
// we get all the items from the cache, all the items from the live,
568+
// and all the deletions because the cache expires old items when it's refreshed
569+
if (++count == expected * 3)
570+
evt.Set();
571+
}, () => { });
572+
573+
574+
evt.WaitOne();
575+
evt.Reset();
576+
577+
Assert.Equal(5, col.Count);
578+
Assert.Collection(col, col.Select(x => new Action<IPullRequestModel>(t => Assert.True(x.Title.StartsWith("Live")))).ToArray());
579+
}
518580
}
519581

520582
static User CreateOctokitUser(string login)

0 commit comments

Comments
 (0)