Skip to content

Commit a94d442

Browse files
Nfactor26Nfactor26
authored
Merge pull request #182 from Nfactor26/simplify-application-and-control-data-download
Simplify how application and controls are downloaded
2 parents 204197b + 1df05bf commit a94d442

28 files changed

+408
-440
lines changed

‎src/Pixel.Automation.Designer.ViewModels/AppBootStrapper.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ protected override void OnStartup(object sender, StartupEventArgs e)
5858
var projectDataManager = IoC.Get<IProjectDataManager>();
5959
var prefabDataManager = IoC.Get<IPrefabDataManager>();
6060
logger.Information("Downloading application data now");
61-
await applicationDataManger.DownloadApplicationsDataAsync();
61+
await applicationDataManger.UpdateApplicationRepository();
6262
logger.Information("Download of application data completed");
6363
logger.Information("Downloading project information now");
6464
await projectDataManager.DownloadProjectsAsync();

‎src/Pixel.Automation.Designer.ViewModels/Modules/PersistenceModules.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ public class PersistenceModules : NinjectModule
88
{
99
public override void Load()
1010
{
11-
Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();
12-
Kernel.Bind<IMetaDataClient>().To<MetaDataClient>();
11+
Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();
1312
Kernel.Bind<IApplicationRepositoryClient>().To<ApplicationRepositoryClient>();
1413
Kernel.Bind<IControlRepositoryClient>().To<ControlRepositoryClient>();
1514
Kernel.Bind<ITestSessionClient>().To<TestSessionClient>();

‎src/Pixel.Automation.Test.Runner/Modules/PersistenceModules.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ public class PersistenceModules : NinjectModule
88
{
99
public override void Load()
1010
{
11-
Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();
12-
Kernel.Bind<IMetaDataClient>().To<MetaDataClient>();
11+
Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();
1312
Kernel.Bind<IApplicationRepositoryClient>().To<ApplicationRepositoryClient>();
1413
Kernel.Bind<IControlRepositoryClient>().To<ControlRepositoryClient>();
1514
Kernel.Bind<IAutomationsRepositoryClient>().To<AutomationsRepositoryClient>();

‎src/Pixel.Automation.Test.Runner/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ static async Task<int> Main(string[] args)
5252
#endif
5353

5454
var applicationDataManager = kernel.Get<IApplicationDataManager>();
55-
await applicationDataManager.DownloadApplicationsDataAsync();
55+
await applicationDataManager.UpdateApplicationRepository();
5656
var projectDataManger = kernel.Get<IProjectDataManager>();
5757
await projectDataManger.DownloadProjectsAsync();
5858
var prefabDataManager = kernel.Get<IPrefabDataManager>();

‎src/Pixel.Persistence.Core/Models/ControlDataRequest.cs‎

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,28 @@
11
using System;
22
using System.Collections.Generic;
3-
using System.Text;
43

5-
namespace Pixel.Persistence.Core.Models
4+
namespace Pixel.Persistence.Core.Models;
5+
6+
/// <summary>
7+
/// Request data for retrieving specified controls belonging to an application
8+
/// </summary>
9+
public class GetControlsForApplicationRequest
610
{
711
/// <summary>
8-
/// Request data for retrieving specified controls belonging to an application
12+
/// Application Id of the owner application of control
913
/// </summary>
10-
public class GetControlDataForApplicationRequest
11-
{
12-
/// <summary>
13-
/// Application Id of the owner application of control
14-
/// </summary>
15-
public string ApplicationId { get; set; }
16-
17-
/// <summary>
18-
/// Collection of control id whose data needs to be retrieved
19-
/// </summary>
20-
public IEnumerable<string> ControlIdCollection { get; set; }
21-
}
14+
public string ApplicationId { get; set; }
2215

2316
/// <summary>
24-
/// Request data for retrieving controls belonging to multiple applications
17+
/// Filter criteria to get only those controls that have been modified after this time
2518
/// </summary>
26-
public class GetControlDataForMultipleApplicationRequest
27-
{
28-
public IEnumerable<GetControlDataForApplicationRequest> ControlDataRequestCollection { get; set; }
29-
}
19+
public DateTime laterThan { get; set; }
20+
}
21+
22+
/// <summary>
23+
/// Request data for retrieving controls belonging to multiple applications
24+
/// </summary>
25+
public class GetControlDataForMultipleApplicationRequest
26+
{
27+
public IEnumerable<GetControlsForApplicationRequest> ControlDataRequestCollection { get; set; }
3028
}

‎src/Pixel.Persistence.Core/Models/DataFile.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ public class DataFile
1313
public byte[] Bytes { get; set; }
1414
}
1515

16+
public class ControlImageDataFile : DataFile
17+
{
18+
public string ControlId { get; set; }
19+
}
20+
1621
public class ProjectDataFile : DataFile
1722
{
1823
public string ProjectId { get; set; }

‎src/Pixel.Persistence.Respository/ApplicationRepository.cs‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
using Dawn;
22
using MongoDB.Bson;
33
using MongoDB.Driver;
4-
using Pixel.Persistence.Core.Models;
54
using System;
65
using System.Collections.Generic;
76
using System.Threading.Tasks;
@@ -17,11 +16,10 @@ public ApplicationRepository(IMongoDbSettings dbSettings)
1716
var client = new MongoClient(dbSettings.ConnectionString);
1817
var database = client.GetDatabase(dbSettings.DatabaseName);
1918
applicationsCollection = database.GetCollection<BsonDocument>(dbSettings.ApplicationsCollectionName);
20-
2119
}
2220

2321
///<inheritdoc/>
24-
public async Task<object> GetApplicationData(string applicationId)
22+
public async Task<object> GetApplication(string applicationId)
2523
{
2624
Guard.Argument(applicationId).NotNull().NotEmpty();
2725

@@ -34,22 +32,18 @@ public async Task<object> GetApplicationData(string applicationId)
3432
}
3533

3634
///<inheritdoc/>
37-
public async IAsyncEnumerable<ApplicationMetaData> GetMetadataAsync()
35+
public async Task<IEnumerable<object>> GetAllApplications(DateTime laterThan)
3836
{
39-
var filter = Builders<BsonDocument>.Filter.Empty;
40-
var projection = Builders<BsonDocument>.Projection.Include("ApplicationId").Include("LastUpdated");
41-
var results = await applicationsCollection.Find<BsonDocument>(filter).Project(projection).ToListAsync();
42-
43-
foreach (var doc in results)
37+
var filter = Builders<BsonDocument>.Filter.Gt(x => x["LastUpdated"], laterThan);
38+
var results = await applicationsCollection.FindAsync<BsonDocument>(filter);
39+
List<object> applications = new List<object>();
40+
foreach(var application in (await results.ToListAsync()))
4441
{
45-
yield return new ApplicationMetaData()
46-
{
47-
ApplicationId = doc["ApplicationId"].AsString,
48-
LastUpdated = doc["LastUpdated"].ToUniversalTime()
49-
};
42+
applications.Add(BsonTypeMapper.MapToDotNetValue(application));
5043
}
44+
return applications;
5145
}
52-
46+
5347
///<inheritdoc/>
5448
public async Task AddOrUpdate(string applicationDescriptionJson)
5549
{

‎src/Pixel.Persistence.Respository/ControlRepository.cs‎

Lines changed: 84 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@
66
using System;
77
using System.Collections.Generic;
88
using System.IO;
9+
using System.Linq;
910
using System.Text.Json;
1011
using System.Threading.Tasks;
1112

1213
namespace Pixel.Persistence.Respository
13-
{
14+
{
1415
public class ControlRepository : IControlRepository
1516
{
1617
private readonly IMongoCollection<BsonDocument> controlsCollection;
@@ -32,6 +33,87 @@ public ControlRepository(IMongoDbSettings dbSettings)
3233

3334
}
3435

36+
///<inheritdoc/>
37+
public async IAsyncEnumerable<DataFile> GetControlFiles(string applicationId, string controlId)
38+
{
39+
Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
40+
Guard.Argument(controlId, nameof(controlId)).NotNull().NotEmpty();
41+
42+
//Get all the versoins of the control
43+
var projection = Builders<BsonDocument>.Projection.Exclude("_id").Exclude("LastUpdated");
44+
var result = controlsCollection.Find<BsonDocument>(CreateControlFilter(applicationId, controlId)).Project(projection);
45+
var documents = await result.ToListAsync();
46+
foreach (var document in documents)
47+
{
48+
using (MemoryStream ms = new MemoryStream())
49+
{
50+
using (StreamWriter sw = new StreamWriter(ms, System.Text.Encoding.UTF8))
51+
{
52+
var jsonData = JsonSerializer.Serialize(BsonTypeMapper.MapToDotNetValue(document), new JsonSerializerOptions()
53+
{
54+
WriteIndented = true
55+
});
56+
sw.Write(jsonData);
57+
sw.Flush();
58+
yield return new DataFile() { FileName = $"{controlId}.dat", Version = document["Version"].AsString, Bytes = ms.ToArray(), Type = "ControlFile" };
59+
60+
}
61+
}
62+
}
63+
64+
//Get all the versions of the images
65+
var filter = CreateImageFilter(applicationId, controlId);
66+
var sort = Builders<GridFSFileInfo>.Sort.Descending(x => x.UploadDateTime);
67+
var options = new GridFSFindOptions
68+
{
69+
Sort = sort
70+
};
71+
using (var cursor = await imageBucket.FindAsync(filter, new GridFSFindOptions()))
72+
{
73+
var imageFiles = await cursor.ToListAsync();
74+
foreach (var imageFile in imageFiles)
75+
{
76+
var imageBytes = await imageBucket.DownloadAsBytesAsync(imageFile.Id);
77+
yield return new DataFile() { FileName = imageFile.Filename, Version = imageFile.Metadata["version"].AsString, Bytes = imageBytes, Type = "ControlImage" };
78+
}
79+
}
80+
81+
}
82+
83+
///<inheritdoc/>
84+
public async Task<IEnumerable<object>> GetAllControlsForApplication(string applicationId, DateTime laterThan)
85+
{
86+
Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
87+
var controlFilter = Builders<BsonDocument>.Filter.Eq(x => x["ApplicationId"], applicationId) & Builders<BsonDocument>.Filter.Gt(x => x["LastUpdated"], laterThan);
88+
var controls = (await (await controlsCollection.FindAsync<BsonDocument>(controlFilter)).ToListAsync()).Select( s => BsonTypeMapper.MapToDotNetValue(s));
89+
return controls;
90+
}
91+
92+
///<inheritdoc/>
93+
public async Task<IEnumerable<ControlImageDataFile>> GetAllControlImagesForApplication(string applicationId, DateTime laterThan)
94+
{
95+
Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
96+
List<ControlImageDataFile> controlImages = new();
97+
var imageFilter = Builders<GridFSFileInfo>.Filter.Eq(x => x.Metadata["applicationId"], applicationId) & Builders<GridFSFileInfo>.Filter.Gt(x => x.UploadDateTime, laterThan);
98+
using (var cursor = await imageBucket.FindAsync(imageFilter, new GridFSFindOptions()))
99+
{
100+
var imageFiles = await cursor.ToListAsync();
101+
102+
foreach (var imageFile in imageFiles)
103+
{
104+
var imageBytes = await imageBucket.DownloadAsBytesAsync(imageFile.Id);
105+
controlImages.Add(new ControlImageDataFile()
106+
{
107+
FileName = imageFile.Filename,
108+
ControlId = imageFile.Metadata["controlId"].AsString,
109+
Version = imageFile.Metadata["version"].AsString,
110+
Bytes = imageBytes
111+
});
112+
}
113+
}
114+
return controlImages;
115+
}
116+
35117
///<inheritdoc/>
36118
public async Task AddOrUpdateControl(string controlDataJson)
37119
{
@@ -101,74 +183,7 @@ public async Task DeleteImageAsync(ControlImageMetaData imageMetaData)
101183
await imageBucket.DeleteAsync(imageFile.Id);
102184
}
103185
}
104-
}
105-
106-
///<inheritdoc/>
107-
public async IAsyncEnumerable<DataFile> GetControlFiles(string applicationId, string controlId)
108-
{
109-
Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
110-
Guard.Argument(controlId, nameof(controlId)).NotNull().NotEmpty();
111-
112-
//Get all the versoins of the control
113-
var projection = Builders<BsonDocument>.Projection.Exclude("_id").Exclude("LastUpdated");
114-
var result = controlsCollection.Find<BsonDocument>(CreateControlFilter(applicationId, controlId)).Project(projection);
115-
var documents = await result.ToListAsync();
116-
foreach(var document in documents)
117-
{
118-
using (MemoryStream ms = new MemoryStream())
119-
{
120-
using (StreamWriter sw = new StreamWriter(ms, System.Text.Encoding.UTF8))
121-
{
122-
var jsonData = JsonSerializer.Serialize(BsonTypeMapper.MapToDotNetValue(document), new JsonSerializerOptions()
123-
{
124-
WriteIndented = true
125-
});
126-
sw.Write(jsonData);
127-
sw.Flush();
128-
yield return new DataFile() { FileName = $"{controlId}.dat", Version = document["Version"].AsString, Bytes = ms.ToArray(), Type = "ControlFile" };
129-
130-
}
131-
}
132-
}
133-
134-
//Get all the versions of the images
135-
var filter = CreateImageFilter(applicationId, controlId);
136-
var sort = Builders<GridFSFileInfo>.Sort.Descending(x => x.UploadDateTime);
137-
var options = new GridFSFindOptions
138-
{
139-
Sort = sort
140-
};
141-
using (var cursor = await imageBucket.FindAsync(filter, new GridFSFindOptions()))
142-
{
143-
var imageFiles = await cursor.ToListAsync();
144-
foreach(var imageFile in imageFiles)
145-
{
146-
var imageBytes = await imageBucket.DownloadAsBytesAsync(imageFile.Id);
147-
yield return new DataFile() { FileName = imageFile.Filename, Version = imageFile.Metadata["version"].AsString, Bytes = imageBytes, Type = "ControlImage" };
148-
}
149-
}
150-
151-
}
152-
153-
///<inheritdoc/>
154-
public async IAsyncEnumerable<ControlMetaData> GetMetadataAsync(string applicationId)
155-
{
156-
Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
157-
158-
var filterBuilder = Builders<BsonDocument>.Filter;
159-
var filter = filterBuilder.Eq(x => x["ApplicationId"], applicationId);
160-
var projection = Builders<BsonDocument>.Projection.Include("ControlId").Include("Version").Include("LastUpdated");
161-
var results = await controlsCollection.Find<BsonDocument>(filter).Project(projection).ToListAsync();
162-
foreach (var doc in results)
163-
{
164-
yield return new ControlMetaData()
165-
{
166-
ControlId = doc["ControlId"].AsString,
167-
Version = doc["Version"].AsString,
168-
LastUpdated = doc["LastUpdated"].ToUniversalTime()
169-
};
170-
}
171-
}
186+
}
172187

173188
/// <summary>
174189
/// Create filter condition for control file with given applicationId and controlId

‎src/Pixel.Persistence.Respository/Interfaces/IApplicationRepository.cs‎

Lines changed: 24 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,33 @@
1-
using Pixel.Persistence.Core.Models;
1+
using System;
22
using System.Collections.Generic;
33
using System.Threading.Tasks;
44

5-
namespace Pixel.Persistence.Respository
5+
namespace Pixel.Persistence.Respository;
6+
7+
/// <summary>
8+
/// IApplicationRepository is used to manage the ApplicationDescription data stored in database
9+
/// </summary>
10+
public interface IApplicationRepository
611
{
712
/// <summary>
8-
/// IApplicationRepository is used to manage the ApplicationDescription data stored in database
13+
/// Get the application data document for a given applicationId
914
/// </summary>
10-
public interface IApplicationRepository
11-
{
12-
/// <summary>
13-
/// Get the application data document for a given applicationId
14-
/// </summary>
15-
/// <param name="applicationId"></param>
16-
/// <returns>Contents of application file</returns>
17-
Task<object> GetApplicationData(string applicationId);
18-
19-
/// <summary>
20-
/// Get the <see cref="ApplicationMetaData"/>. This is used on client side to compare if there are any
21-
/// newer files on server
22-
/// </summary>
23-
/// <returns>Collection of <see cref="ApplicationMetaData"/></returns>
24-
IAsyncEnumerable<ApplicationMetaData> GetMetadataAsync();
15+
/// <param name="applicationId"></param>
16+
/// <returns>Contents of application file</returns>
17+
Task<object> GetApplication(string applicationId);
2518

26-
/// <summary>
27-
/// Add or update the ApplicationDescription data
28-
/// </summary>
29-
/// <param name="applicationDescriptionJson">json serialized application description </param>
30-
/// <returns></returns>
31-
Task AddOrUpdate(string applicationDescriptionJson);
19+
/// <summary>
20+
/// Get all applications that were modified since specified
21+
/// </summary>
22+
/// <param name="laterThan"></param>
23+
/// <returns></returns>
24+
Task<IEnumerable<object>> GetAllApplications(DateTime laterThan);
25+
26+
/// <summary>
27+
/// Add or update the ApplicationDescription data
28+
/// </summary>
29+
/// <param name="applicationDescriptionJson">json serialized application description </param>
30+
/// <returns></returns>
31+
Task AddOrUpdate(string applicationDescriptionJson);
3232

33-
}
3433
}

0 commit comments

Comments
 (0)