Skip to content

Commit 8a0bd5d

Browse files
hahn-kevhahn-kev
authored
add remote resource support (#2)
* add remote resources support. add method to upload all pending local resources, and call from SyncWithResourceUpload helper method * allow multiple object adapters * add resource example in the sample project * expose api to list all resources * test AllResources api, change `AddLocalResource` to return a HarmonyResource * try uploading resources to the remote server, log an error on failure but don't cancel and mark the upload as pending
1 parent f400e26 commit 8a0bd5d

25 files changed

+824
-9
lines changed

‎Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<PackageVersion Include="System.IO.Hashing" Version="9.0.0" />
1010
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
1111
<PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
12-
<PackageVersion Include="FluentAssertions" Version="6.12.2" />
12+
<PackageVersion Include="FluentAssertions" Version="7.0.0-alpha.6" />
1313
<PackageVersion Include="GitHubActionsTestLogger" Version="2.4.1" />
1414
<PackageVersion Include="JetBrains.Profiler.SelfApi" Version="2.5.12" />
1515
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />

‎src/SIL.Harmony.Core/EntityNotFoundException.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
namespace SIL.Harmony.Core;
2+
3+
public class EntityNotFoundException(string message) : Exception(message);

‎src/SIL.Harmony.Core/IRemoteResourceService.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
namespace SIL.Harmony.Core;
2+
3+
/// <summary>
4+
/// interface to facilitate downloading of resources, typically implemented in application code
5+
/// the remote Id is opaque to the CRDT lib and could be a URL or some other identifier provided by the backend
6+
/// the local path returned for the application code to use as required, it could be a URL if needed also.
7+
/// </summary>
8+
public interface IRemoteResourceService
9+
{
10+
/// <summary>
11+
/// instructs application code to download a resource from the remote server
12+
/// the service is responsible for downloading the resource and returning the local path
13+
/// </summary>
14+
/// <param name="remoteId">ID used to identify the remote resource, could be a URL</param>
15+
/// <param name="localResourceCachePath">path defined by the CRDT config where the resource should be stored</param>
16+
/// <returns>download result containing the path to the downloaded file, this is stored in the local db and not synced</returns>
17+
Task<DownloadResult> DownloadResource(string remoteId, string localResourceCachePath);
18+
/// <summary>
19+
/// upload a resource to the remote server
20+
/// </summary>
21+
/// <param name="localPath">full path to the resource on the local machine</param>
22+
/// <returns>an upload result with the remote id, the id will be stored and transmitted to other clients so they can also download the resource</returns>
23+
Task<UploadResult> UploadResource(string localPath);
24+
}
25+
26+
public record DownloadResult(string LocalPath);
27+
public record UploadResult(string RemoteId);

‎src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using SIL.Harmony.Changes;
2+
using SIL.Harmony.Entities;
3+
using SIL.Harmony.Sample.Models;
4+
5+
namespace SIL.Harmony.Sample.Changes;
6+
7+
public class AddWordImageChange(Guid entityId, Guid imageId) : EditChange<Word>(entityId), ISelfNamedType<AddWordImageChange>
8+
{
9+
public Guid ImageId { get; } = imageId;
10+
11+
public override async ValueTask ApplyChange(Word entity, ChangeContext context)
12+
{
13+
if (!await context.IsObjectDeleted(ImageId)) entity.ImageResourceId = ImageId;
14+
}
15+
}

‎src/SIL.Harmony.Sample/CrdtSampleKernel.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi
3636
services.AddCrdtData<SampleDbContext>(config =>
3737
{
3838
config.EnableProjectedTables = true;
39+
config.AddRemoteResourceEntity();
3940
config.ChangeTypeListBuilder
4041
.Add<NewWordChange>()
4142
.Add<NewDefinitionChange>()
@@ -44,6 +45,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi
4445
.Add<SetWordTextChange>()
4546
.Add<SetWordNoteChange>()
4647
.Add<AddAntonymReferenceChange>()
48+
.Add<AddWordImageChange>()
4749
.Add<SetOrderChange<Definition>>()
4850
.Add<SetDefinitionPartOfSpeechChange>()
4951
.Add<DeleteChange<Word>>()

‎src/SIL.Harmony.Sample/Models/Word.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,17 @@ public class Word : IObjectBase<Word>
1010
public Guid Id { get; init; }
1111
public DateTimeOffset? DeletedAt { get; set; }
1212
public Guid? AntonymId { get; set; }
13+
public Guid? ImageResourceId { get; set; }
1314

1415
public Guid[] GetReferences()
1516
{
16-
return AntonymId is null ? [] : [AntonymId.Value];
17+
return Refs().ToArray();
18+
19+
IEnumerable<Guid> Refs()
20+
{
21+
if (AntonymId.HasValue) yield return AntonymId.Value;
22+
if (ImageResourceId.HasValue) yield return ImageResourceId.Value;
23+
}
1724
}
1825

1926
public void RemoveReference(Guid id, Commit commit)

‎src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,40 @@
8181
Relational:TableName: Snapshots
8282
Relational:ViewName:
8383
Relational:ViewSchema:
84+
EntityType: LocalResource
85+
Properties:
86+
Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd
87+
LocalPath (string) Required
88+
Keys:
89+
Id PK
90+
Annotations:
91+
DiscriminatorProperty:
92+
Relational:FunctionName:
93+
Relational:Schema:
94+
Relational:SqlQuery:
95+
Relational:TableName: LocalResource
96+
Relational:ViewName:
97+
Relational:ViewSchema:
98+
EntityType: RemoteResource
99+
Properties:
100+
Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd
101+
DeletedAt (DateTimeOffset?)
102+
RemoteId (string)
103+
SnapshotId (no field, Guid?) Shadow FK Index
104+
Keys:
105+
Id PK
106+
Foreign keys:
107+
RemoteResource {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull
108+
Indexes:
109+
SnapshotId Unique
110+
Annotations:
111+
DiscriminatorProperty:
112+
Relational:FunctionName:
113+
Relational:Schema:
114+
Relational:SqlQuery:
115+
Relational:TableName: RemoteResource
116+
Relational:ViewName:
117+
Relational:ViewSchema:
84118
EntityType: Definition
85119
Properties:
86120
Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd
@@ -133,6 +167,7 @@
133167
Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd
134168
AntonymId (Guid?)
135169
DeletedAt (DateTimeOffset?)
170+
ImageResourceId (Guid?)
136171
Note (string)
137172
SnapshotId (no field, Guid?) Shadow FK Index
138173
Text (string) Required

‎src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
using System.Runtime.CompilerServices;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using SIL.Harmony.Resource;
4+
5+
namespace SIL.Harmony.Tests.ResourceTests;
6+
7+
public class RemoteResourcesTests : DataModelTestBase
8+
{
9+
private RemoteServiceMock _remoteServiceMock = new();
10+
private ResourceService _resourceService => _services.GetRequiredService<ResourceService>();
11+
12+
public RemoteResourcesTests()
13+
{
14+
}
15+
16+
private string CreateFile(string contents, [CallerMemberName] string fileName = "")
17+
{
18+
var filePath = Path.GetFullPath(fileName + ".txt");
19+
File.WriteAllText(filePath, contents);
20+
return filePath;
21+
}
22+
23+
private async Task<(Guid resourceId, string remoteId)> SetupRemoteResource(string fileContents)
24+
{
25+
var remoteId = _remoteServiceMock.CreateRemoteResource(fileContents);
26+
var resourceId = Guid.NewGuid();
27+
await DataModel.AddChange(_localClientId, new CreateRemoteResourceChange(resourceId, remoteId));
28+
return (resourceId, remoteId);
29+
}
30+
31+
private async Task<(Guid resourceId, string localPath)> SetupLocalFile(string contents, [CallerMemberName] string fileName = "")
32+
{
33+
var file = CreateFile(contents, fileName);
34+
//because resource service is null the file is not uploaded
35+
var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null);
36+
return (crdtResource.Id, file);
37+
}
38+
39+
[Fact]
40+
public async Task CreatingAResourceResultsInPendingLocalResources()
41+
{
42+
var (_, file) = await SetupLocalFile("contents");
43+
44+
//act
45+
var pending = await _resourceService.ListResourcesPendingUpload();
46+
47+
48+
pending.Should().ContainSingle().Which.LocalPath.Should().Be(file);
49+
}
50+
51+
[Fact]
52+
public async Task ResourcesNotLocalShouldShowUpAsNotDownloaded()
53+
{
54+
var (resourceId, remoteId) = await SetupRemoteResource("test");
55+
56+
//act
57+
var pending = await _resourceService.ListResourcesPendingDownload();
58+
59+
60+
var remoteResource = pending.Should().ContainSingle().Subject;
61+
remoteResource.RemoteId.Should().Be(remoteId);
62+
remoteResource.Id.Should().Be(resourceId);
63+
}
64+
65+
[Fact]
66+
public async Task CanUploadFileToRemote()
67+
{
68+
var fileContents = "resource";
69+
var localFile = CreateFile(fileContents);
70+
71+
//act
72+
var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock);
73+
74+
75+
var resource = await DataModel.GetLatest<RemoteResource>(crdtResource.Id);
76+
ArgumentNullException.ThrowIfNull(resource);
77+
ArgumentNullException.ThrowIfNull(resource.RemoteId);
78+
_remoteServiceMock.ReadFile(resource.RemoteId).Should().Be(fileContents);
79+
var pendingUpload = await _resourceService.ListResourcesPendingUpload();
80+
pendingUpload.Should().BeEmpty();
81+
}
82+
83+
[Fact]
84+
public async Task IfUploadingFailsTheResourceIsStillAddedAsPendingUpload()
85+
{
86+
var fileContents = "resource";
87+
var localFile = CreateFile(fileContents);
88+
89+
//todo setup a mock that throws an exception when uploading
90+
_remoteServiceMock.ThrowOnUpload(localFile);
91+
92+
//act
93+
var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock);
94+
95+
var harmonyResource = await _resourceService.GetResource(crdtResource.Id);
96+
harmonyResource.Should().NotBeNull();
97+
harmonyResource.Id.Should().Be(crdtResource.Id);
98+
harmonyResource.RemoteId.Should().BeNull();
99+
harmonyResource.LocalPath.Should().Be(localFile);
100+
var pendingUpload = await _resourceService.ListResourcesPendingUpload();
101+
pendingUpload.Should().ContainSingle().Which.Id.Should().Be(crdtResource.Id);
102+
}
103+
104+
[Fact]
105+
public async Task WillUploadMultiplePendingLocalFilesAtOnce()
106+
{
107+
await SetupLocalFile("file1", "file1");
108+
await SetupLocalFile("file2", "file2");
109+
110+
//act
111+
await _resourceService.UploadPendingResources(_localClientId, _remoteServiceMock);
112+
113+
114+
_remoteServiceMock.ListRemoteFiles()
115+
.Select(Path.GetFileName)
116+
.Should()
117+
.Contain(["file1.txt", "file2.txt"]);
118+
}
119+
120+
[Fact]
121+
public async Task CanDownloadFileFromRemote()
122+
{
123+
var fileContents = "resource";
124+
var (resourceId, _) = await SetupRemoteResource(fileContents);
125+
126+
//act
127+
var localResource = await _resourceService.DownloadResource(resourceId, _remoteServiceMock);
128+
129+
130+
localResource.Id.Should().Be(resourceId);
131+
var actualFileContents = await File.ReadAllTextAsync(localResource.LocalPath);
132+
actualFileContents.Should().Be(fileContents);
133+
var pendingDownloads = await _resourceService.ListResourcesPendingDownload();
134+
pendingDownloads.Should().BeEmpty();
135+
}
136+
137+
[Fact]
138+
public async Task CanGetALocalResourceGivenAnId()
139+
{
140+
var file = CreateFile("resource");
141+
//because resource service is null the file is not uploaded
142+
var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null);
143+
144+
//act
145+
var localResource = await _resourceService.GetLocalResource(crdtResource.Id);
146+
147+
148+
localResource.Should().NotBeNull();
149+
localResource!.LocalPath.Should().Be(file);
150+
}
151+
152+
[Fact]
153+
public async Task LocalResourceIsNullIfNotDownloaded()
154+
{
155+
var (resourceId, _) = await SetupRemoteResource("test");
156+
var localResource = await _resourceService.GetLocalResource(resourceId);
157+
localResource.Should().BeNull();
158+
}
159+
160+
[Fact]
161+
public async Task CanListAllResources()
162+
{
163+
var (localResourceId, localResourcePath) = await SetupLocalFile("localOnly", "localOnly.txt");
164+
var (remoteResourceId, remoteId) = await SetupRemoteResource("remoteOnly");
165+
var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"), _localClientId, resourceService: _remoteServiceMock);
166+
167+
var crdtResources = await _resourceService.AllResources();
168+
crdtResources.Should().BeEquivalentTo(
169+
[
170+
new HarmonyResource
171+
{
172+
Id = localResourceId,
173+
LocalPath = localResourcePath,
174+
RemoteId = null
175+
},
176+
new HarmonyResource
177+
{
178+
Id = remoteResourceId,
179+
LocalPath = null,
180+
RemoteId = remoteId
181+
},
182+
localAndRemoteResource
183+
]);
184+
}
185+
186+
[Fact]
187+
public async Task CanGetAResourceGivenAnId()
188+
{
189+
var (localResourceId, localResourcePath) = await SetupLocalFile("localOnly", "localOnly.txt");
190+
var (remoteResourceId, remoteId) = await SetupRemoteResource("remoteOnly");
191+
var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"),
192+
_localClientId,
193+
resourceService: _remoteServiceMock);
194+
195+
(await _resourceService.GetResource(localResourceId)).Should().BeEquivalentTo(new HarmonyResource
196+
{
197+
Id = localResourceId,
198+
LocalPath = localResourcePath,
199+
RemoteId = null
200+
});
201+
(await _resourceService.GetResource(remoteResourceId)).Should().BeEquivalentTo(new HarmonyResource
202+
{
203+
Id = remoteResourceId,
204+
LocalPath = null,
205+
RemoteId = remoteId
206+
});
207+
(await _resourceService.GetResource(localAndRemoteResource.Id)).Should().BeEquivalentTo(localAndRemoteResource);
208+
(await _resourceService.GetResource(Guid.NewGuid())).Should().BeNull();
209+
}
210+
}

0 commit comments

Comments
 (0)