Simplify how application and controls are downloaded

1. Instead of retrieving metadata on applications and controls that are not available locally or might have been updated  and then retrieving them, we now juts pass a datetime which represents when wa the last time application repository was updated. Any newer application or controls are retrieved. This makes it consistent with how fixtures , test-cases and test data source are retrieved.
2. Additionally, we are passing DateTime as a header now instead of FormBody. This is because while we can pass body in a get requet from .net , it is not a standard and also doesn't work with swagger.

Author: Nfactor26

src/Pixel.Automation.Designer.ViewModels/AppBootStrapper.cs

@@ -58,7 +58,7 @@ protected override void OnStartup(object sender, StartupEventArgs e)
                             var projectDataManager = IoC.Get<IProjectDataManager>();
                             var prefabDataManager = IoC.Get<IPrefabDataManager>();
                             logger.Information("Downloading application data now");
-                            await applicationDataManger.DownloadApplicationsDataAsync();
+                            await applicationDataManger.UpdateApplicationRepository();
                             logger.Information("Download of application data completed");
                             logger.Information("Downloading project information now");
                             await projectDataManager.DownloadProjectsAsync();

src/Pixel.Automation.Designer.ViewModels/Modules/PersistenceModules.cs

@@ -8,8 +8,7 @@ public class PersistenceModules : NinjectModule
     {
         public override void Load()
         {
-            Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();
-            Kernel.Bind<IMetaDataClient>().To<MetaDataClient>();
+            Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();          
             Kernel.Bind<IApplicationRepositoryClient>().To<ApplicationRepositoryClient>();
             Kernel.Bind<IControlRepositoryClient>().To<ControlRepositoryClient>();          
             Kernel.Bind<ITestSessionClient>().To<TestSessionClient>();

src/Pixel.Automation.Test.Runner/Modules/PersistenceModules.cs

@@ -8,8 +8,7 @@ public class PersistenceModules : NinjectModule
     {
         public override void Load()
         {
-            Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();
-            Kernel.Bind<IMetaDataClient>().To<MetaDataClient>();
+            Kernel.Bind<IRestClientFactory>().To<RestClientFactory>().InSingletonScope();           
             Kernel.Bind<IApplicationRepositoryClient>().To<ApplicationRepositoryClient>();
             Kernel.Bind<IControlRepositoryClient>().To<ControlRepositoryClient>();
             Kernel.Bind<IAutomationsRepositoryClient>().To<AutomationsRepositoryClient>();

src/Pixel.Automation.Test.Runner/Program.cs

@@ -52,7 +52,7 @@ static async Task<int> Main(string[] args)
                 #endif
 
                 var applicationDataManager = kernel.Get<IApplicationDataManager>();
-                await applicationDataManager.DownloadApplicationsDataAsync();
+                await applicationDataManager.UpdateApplicationRepository();
                 var projectDataManger = kernel.Get<IProjectDataManager>();
                 await projectDataManger.DownloadProjectsAsync();
                 var prefabDataManager = kernel.Get<IPrefabDataManager>();

src/Pixel.Persistence.Core/Models/ControlDataRequest.cs

@@ -1,30 +1,28 @@
 using System;
 using System.Collections.Generic;
-using System.Text;
 
-namespace Pixel.Persistence.Core.Models
+namespace Pixel.Persistence.Core.Models;
+
+/// <summary>
+/// Request data for retrieving specified controls belonging to an application
+/// </summary>
+public class GetControlsForApplicationRequest
 {
     /// <summary>
-    /// Request data for retrieving specified controls belonging to an application
+    /// Application Id of the owner application of control
     /// </summary>
-    public class GetControlDataForApplicationRequest
-    {
-        /// <summary>
-        /// Application Id of the owner application of control
-        /// </summary>
-        public string ApplicationId { get; set; }
-
-        /// <summary>
-        /// Collection of control id whose data needs to be retrieved
-        /// </summary>
-        public IEnumerable<string> ControlIdCollection { get; set; }
-    }
+    public string ApplicationId { get; set; }
 
     /// <summary>
-    /// Request data for retrieving controls belonging to multiple applications
+    /// Filter criteria to get only those controls that have been modified after this time
     /// </summary>
-    public class GetControlDataForMultipleApplicationRequest
-    {
-        public IEnumerable<GetControlDataForApplicationRequest> ControlDataRequestCollection { get; set; }
-    }
+    public DateTime laterThan { get; set; }
+}
+
+/// <summary>
+/// Request data for retrieving controls belonging to multiple applications
+/// </summary>
+public class GetControlDataForMultipleApplicationRequest
+{
+    public IEnumerable<GetControlsForApplicationRequest> ControlDataRequestCollection { get; set; }
 }

src/Pixel.Persistence.Core/Models/DataFile.cs

@@ -13,6 +13,11 @@ public class DataFile
         public byte[] Bytes { get; set; }
     } 
 
+    public class ControlImageDataFile : DataFile
+    {
+        public string ControlId { get; set; }
+    }
+
     public class ProjectDataFile : DataFile
     {
         public string ProjectId { get; set; }

src/Pixel.Persistence.Respository/ApplicationRepository.cs

@@ -1,7 +1,6 @@
 using Dawn;
 using MongoDB.Bson;
 using MongoDB.Driver;
-using Pixel.Persistence.Core.Models;
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
@@ -17,11 +16,10 @@ public ApplicationRepository(IMongoDbSettings dbSettings)
             var client = new MongoClient(dbSettings.ConnectionString);
             var database = client.GetDatabase(dbSettings.DatabaseName);
             applicationsCollection = database.GetCollection<BsonDocument>(dbSettings.ApplicationsCollectionName);
-
         }
 
         ///<inheritdoc/>
-        public async Task<object> GetApplicationData(string applicationId)
+        public async Task<object> GetApplication(string applicationId)
         {
             Guard.Argument(applicationId).NotNull().NotEmpty();
 
@@ -34,22 +32,18 @@ public async Task<object> GetApplicationData(string applicationId)
         }
 
         ///<inheritdoc/>
-        public async IAsyncEnumerable<ApplicationMetaData> GetMetadataAsync()
+        public async Task<IEnumerable<object>> GetAllApplications(DateTime laterThan)
         {
-            var filter = Builders<BsonDocument>.Filter.Empty;
-            var projection = Builders<BsonDocument>.Projection.Include("ApplicationId").Include("LastUpdated");
-            var results = await applicationsCollection.Find<BsonDocument>(filter).Project(projection).ToListAsync();
-
-            foreach (var doc in results)
+            var filter = Builders<BsonDocument>.Filter.Gt(x => x["LastUpdated"], laterThan);
+            var results = await applicationsCollection.FindAsync<BsonDocument>(filter);
+            List<object> applications = new List<object>();
+            foreach(var application in (await results.ToListAsync()))
             {
-                yield return new ApplicationMetaData()
-                {
-                    ApplicationId = doc["ApplicationId"].AsString,
-                    LastUpdated = doc["LastUpdated"].ToUniversalTime()
-                };
+                applications.Add(BsonTypeMapper.MapToDotNetValue(application));
             }
+            return applications;
         }
-
+       
         ///<inheritdoc/>
         public async Task AddOrUpdate(string applicationDescriptionJson)
         {

src/Pixel.Persistence.Respository/ControlRepository.cs

@@ -6,11 +6,12 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.Json;
 using System.Threading.Tasks;
 
 namespace Pixel.Persistence.Respository
-{    
+{
     public class ControlRepository : IControlRepository
     {
         private readonly IMongoCollection<BsonDocument> controlsCollection;    
@@ -32,6 +33,87 @@ public ControlRepository(IMongoDbSettings dbSettings)
 
         }
 
+        ///<inheritdoc/>
+        public async IAsyncEnumerable<DataFile> GetControlFiles(string applicationId, string controlId)
+        {
+            Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
+            Guard.Argument(controlId, nameof(controlId)).NotNull().NotEmpty();
+
+            //Get all the versoins of the control
+            var projection = Builders<BsonDocument>.Projection.Exclude("_id").Exclude("LastUpdated");
+            var result = controlsCollection.Find<BsonDocument>(CreateControlFilter(applicationId, controlId)).Project(projection);
+            var documents = await result.ToListAsync();
+            foreach (var document in documents)
+            {
+                using (MemoryStream ms = new MemoryStream())
+                {
+                    using (StreamWriter sw = new StreamWriter(ms, System.Text.Encoding.UTF8))
+                    {
+                        var jsonData = JsonSerializer.Serialize(BsonTypeMapper.MapToDotNetValue(document), new JsonSerializerOptions()
+                        {
+                            WriteIndented = true
+                        });
+                        sw.Write(jsonData);
+                        sw.Flush();
+                        yield return new DataFile() { FileName = $"{controlId}.dat", Version = document["Version"].AsString, Bytes = ms.ToArray(), Type = "ControlFile" };
+
+                    }
+                }
+            }
+
+            //Get all the versions of the images
+            var filter = CreateImageFilter(applicationId, controlId);
+            var sort = Builders<GridFSFileInfo>.Sort.Descending(x => x.UploadDateTime);
+            var options = new GridFSFindOptions
+            {
+                Sort = sort
+            };
+            using (var cursor = await imageBucket.FindAsync(filter, new GridFSFindOptions()))
+            {
+                var imageFiles = await cursor.ToListAsync();
+                foreach (var imageFile in imageFiles)
+                {
+                    var imageBytes = await imageBucket.DownloadAsBytesAsync(imageFile.Id);
+                    yield return new DataFile() { FileName = imageFile.Filename, Version = imageFile.Metadata["version"].AsString, Bytes = imageBytes, Type = "ControlImage" };
+                }
+            }
+
+        }
+
+        ///<inheritdoc/>
+        public async Task<IEnumerable<object>> GetAllControlsForApplication(string applicationId, DateTime laterThan)
+        {
+            Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
+            var controlFilter = Builders<BsonDocument>.Filter.Eq(x => x["ApplicationId"], applicationId) & Builders<BsonDocument>.Filter.Gt(x => x["LastUpdated"], laterThan);
+            var controls = (await (await controlsCollection.FindAsync<BsonDocument>(controlFilter)).ToListAsync()).Select( s => BsonTypeMapper.MapToDotNetValue(s));
+            return controls;
+        }
+
+        ///<inheritdoc/>
+        public async Task<IEnumerable<ControlImageDataFile>> GetAllControlImagesForApplication(string applicationId, DateTime laterThan)
+        {
+            Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
+            List<ControlImageDataFile> controlImages = new();
+            var imageFilter = Builders<GridFSFileInfo>.Filter.Eq(x => x.Metadata["applicationId"], applicationId) & Builders<GridFSFileInfo>.Filter.Gt(x => x.UploadDateTime, laterThan);
+            using (var cursor = await imageBucket.FindAsync(imageFilter, new GridFSFindOptions()))
+            {
+                var imageFiles = await cursor.ToListAsync();
+                
+                foreach (var imageFile in imageFiles)
+                {
+                    var imageBytes = await imageBucket.DownloadAsBytesAsync(imageFile.Id);
+                    controlImages.Add(new ControlImageDataFile() 
+                            { 
+                                FileName = imageFile.Filename, 
+                                ControlId = imageFile.Metadata["controlId"].AsString, 
+                                Version = imageFile.Metadata["version"].AsString,
+                                Bytes = imageBytes 
+                            });
+                }              
+            }
+            return controlImages;
+        }
+
         ///<inheritdoc/>
         public async Task AddOrUpdateControl(string controlDataJson)
         {
@@ -101,74 +183,7 @@ public async Task DeleteImageAsync(ControlImageMetaData imageMetaData)
                     await imageBucket.DeleteAsync(imageFile.Id);
                 }
             }
-        }
-
-        ///<inheritdoc/>
-        public async IAsyncEnumerable<DataFile> GetControlFiles(string applicationId, string controlId)
-        {
-            Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
-            Guard.Argument(controlId, nameof(controlId)).NotNull().NotEmpty();
-
-            //Get all the versoins of the control
-            var projection = Builders<BsonDocument>.Projection.Exclude("_id").Exclude("LastUpdated");
-            var result = controlsCollection.Find<BsonDocument>(CreateControlFilter(applicationId, controlId)).Project(projection);
-            var documents = await result.ToListAsync();   
-            foreach(var document in documents)
-            {
-                using (MemoryStream ms = new MemoryStream())
-                {
-                    using (StreamWriter sw = new StreamWriter(ms, System.Text.Encoding.UTF8))
-                    {
-                        var jsonData = JsonSerializer.Serialize(BsonTypeMapper.MapToDotNetValue(document), new JsonSerializerOptions()
-                        {
-                            WriteIndented = true
-                        });
-                        sw.Write(jsonData);
-                        sw.Flush();
-                        yield return new DataFile() { FileName = $"{controlId}.dat", Version = document["Version"].AsString, Bytes = ms.ToArray(), Type = "ControlFile" };
-
-                    }
-                }
-            }           
-
-            //Get all the versions of the images
-            var filter = CreateImageFilter(applicationId, controlId);
-            var sort = Builders<GridFSFileInfo>.Sort.Descending(x => x.UploadDateTime);
-            var options = new GridFSFindOptions
-            {                
-                Sort = sort
-            };          
-            using (var cursor = await imageBucket.FindAsync(filter, new GridFSFindOptions()))
-            {
-                var imageFiles = await cursor.ToListAsync();
-                foreach(var imageFile in imageFiles)
-                {
-                    var imageBytes = await imageBucket.DownloadAsBytesAsync(imageFile.Id);
-                    yield return new DataFile() { FileName = imageFile.Filename, Version = imageFile.Metadata["version"].AsString, Bytes = imageBytes, Type = "ControlImage" };
-                }
-            }
-           
-        }
-
-        ///<inheritdoc/>
-        public async IAsyncEnumerable<ControlMetaData> GetMetadataAsync(string applicationId)
-        {
-            Guard.Argument(applicationId, nameof(applicationId)).NotNull().NotEmpty();
-
-            var filterBuilder = Builders<BsonDocument>.Filter;
-            var filter = filterBuilder.Eq(x => x["ApplicationId"], applicationId);
-            var projection = Builders<BsonDocument>.Projection.Include("ControlId").Include("Version").Include("LastUpdated");
-            var results = await controlsCollection.Find<BsonDocument>(filter).Project(projection).ToListAsync();
-            foreach (var doc in results)
-            {
-                yield return new ControlMetaData()
-                {
-                    ControlId = doc["ControlId"].AsString,
-                    Version = doc["Version"].AsString,
-                    LastUpdated = doc["LastUpdated"].ToUniversalTime()
-                };
-            }
-        }
+        }   
 
         /// <summary>
         /// Create filter condition for control file with given applicationId and controlId

src/Pixel.Persistence.Respository/Interfaces/IApplicationRepository.cs

@@ -1,34 +1,33 @@
-using Pixel.Persistence.Core.Models;
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
-namespace Pixel.Persistence.Respository
+namespace Pixel.Persistence.Respository;
+
+/// <summary>
+/// IApplicationRepository is used to manage the ApplicationDescription data stored in database
+/// </summary>
+public interface IApplicationRepository
 {
     /// <summary>
-    /// IApplicationRepository is used to manage the ApplicationDescription data stored in database
+    /// Get the application data document for a given applicationId
     /// </summary>
-    public interface IApplicationRepository
-    {
-        /// <summary>
-        /// Get the application data document for a given applicationId
-        /// </summary>
-        /// <param name="applicationId"></param>
-        /// <returns>Contents of application file</returns>
-        Task<object> GetApplicationData(string applicationId);
-
-        /// <summary>
-        /// Get the <see cref="ApplicationMetaData"/>. This is used on client side to compare if there are any 
-        /// newer files on server
-        /// </summary>
-        /// <returns>Collection of <see cref="ApplicationMetaData"/></returns>
-        IAsyncEnumerable<ApplicationMetaData> GetMetadataAsync();
+    /// <param name="applicationId"></param>
+    /// <returns>Contents of application file</returns>
+    Task<object> GetApplication(string applicationId);
 
-        /// <summary>
-        /// Add or update the ApplicationDescription data 
-        /// </summary>
-        /// <param name="applicationDescriptionJson">json serialized application description </param>
-        /// <returns></returns>
-        Task AddOrUpdate(string applicationDescriptionJson);
+    /// <summary>
+    /// Get all applications that were modified since specified 
+    /// </summary>
+    /// <param name="laterThan"></param>
+    /// <returns></returns>
+    Task<IEnumerable<object>> GetAllApplications(DateTime laterThan);
+   
+    /// <summary>
+    /// Add or update the ApplicationDescription data 
+    /// </summary>
+    /// <param name="applicationDescriptionJson">json serialized application description </param>
+    /// <returns></returns>
+    Task AddOrUpdate(string applicationDescriptionJson);
 
-    }
 }