feat: adding discovery function

.github/workflows/stage-2-test.yaml

@@ -91,6 +91,7 @@ jobs:
       - name: "Checkout code"
         uses: actions/checkout@v4
         with:
+          submodules: 'true'
           fetch-depth: 0 # Full history is needed to improving relevancy of reporting
       - name: "Perform static analysis"
         uses: ./.github/actions/perform-static-analysis

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "src/dotnet-mesh-client"]
+	# path = src/Shared/dotnet-mesh-client
+	path = src/dotnet-mesh-client
+	url = https://github.com/NHSDigital/dotnet-mesh-client.git
+	branch = main

.vscode/extensions.json

@@ -1,5 +1,7 @@
 {
   "recommendations": [
+    "ms-azuretools.vscode-azurefunctions",
+    "ms-dotnettools.csharp",
     "alefragnani.bookmarks",
     "davidanson.vscode-markdownlint",
     "dbaeumer.vscode-eslint",

src/ServiceLayer.Mesh/Data/ServiceLayerDbConext.cs

@@ -0,0 +1,17 @@
+using Microsoft.EntityFrameworkCore;
+using ServiceLayer.Mesh.Models;
+
+namespace ServiceLayer.Mesh.Data;
+
+public class ServiceLayerDbContext(DbContextOptions<ServiceLayerDbContext> options) : DbContext(options)
+{
+    public DbSet<MeshFile> MeshFiles { get; set; }
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        // Configure relationships, keys, etc.
+        modelBuilder.Entity<MeshFile>().HasKey(p => p.FileId);
+        modelBuilder.Entity<MeshFile>().Property(e => e.Status).HasConversion<string>();
+        modelBuilder.Entity<MeshFile>().Property(e => e.FileType).HasConversion<string>();
+    }
+}

src/ServiceLayer.Mesh/Functions/DiscoveryFunction.cs

@@ -0,0 +1,70 @@
+using Azure.Storage.Queues;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using NHS.MESH.Client.Contracts.Services;
+using ServiceLayer.Mesh.Data;
+using ServiceLayer.Mesh.Models;
+
+namespace ServiceLayer.Mesh.Functions
+{
+    public class DiscoveryFunction
+    {
+        private readonly ILogger _logger;
+        private readonly IMeshInboxService _meshInboxService;
+        private readonly ServiceLayerDbContext _serviceLayerDbContext;
+        private readonly QueueClient _queueClient;
+
+        public DiscoveryFunction(ILogger<DiscoveryFunction> logger, IMeshInboxService meshInboxService, ServiceLayerDbContext serviceLayerDbContext, QueueClient queueClient)
+        {
+            _logger = logger;
+            _meshInboxService = meshInboxService;
+            _serviceLayerDbContext = serviceLayerDbContext;
+            _queueClient = queueClient;
+        }
+
+        [Function("DiscoveryFunction")]
+        public async Task Run([TimerTrigger("%DiscoveryTimerExpression%")] TimerInfo myTimer)
+        {
+            _logger.LogInformation($"DiscoveryFunction started at: {DateTime.Now}");
+
+            var mailboxId = Environment.GetEnvironmentVariable("MailboxId")
+                ?? throw new InvalidOperationException($"Environment variable 'MailboxId' is not set or is empty.");
+
+            var response = await _meshInboxService.GetMessagesAsync(mailboxId);
+
+            _queueClient.CreateIfNotExists();
+
+            foreach (var messageId in response.Response.Messages)
+            {
+                using var transaction = await _serviceLayerDbContext.Database.BeginTransactionAsync();
+
+                var existing = await _serviceLayerDbContext.MeshFiles
+                    .AsNoTracking()
+                    .FirstOrDefaultAsync(f => f.FileId == messageId);
+
+                if (existing == null)
+                {
+                    _serviceLayerDbContext.MeshFiles.Add(new MeshFile
+                    {
+                        FileId = messageId,
+                        FileType = MeshFileType.NbssAppointmentEvents,
+                        MailboxId = mailboxId,
+                        Status = MeshFileStatus.Discovered,
+                        FirstSeenUtc = DateTime.UtcNow,
+                        LastUpdatedUtc = DateTime.UtcNow
+                    });
+
+                    await _serviceLayerDbContext.SaveChangesAsync();
+                    await transaction.CommitAsync();
+
+                    _queueClient.SendMessage(messageId);
+                }
+                else
+                {
+                    await transaction.RollbackAsync();
+                }
+            }
+        }
+    }
+}

src/ServiceLayer.Mesh/Models/MeshFile.cs

@@ -0,0 +1,14 @@
+using Microsoft.EntityFrameworkCore.Metadata.Internal;
+
+namespace ServiceLayer.Mesh.Models;
+
+public class MeshFile
+{
+    public required string FileId { get; set; }
+    public required MeshFileType FileType { get; set; }
+    public required string MailboxId { get; set; }
+    public required MeshFileStatus Status { get; set; }
+    public string? BlobPath { get; set; }
+    public DateTime FirstSeenUtc { get; set; }
+    public DateTime LastUpdatedUtc { get; set; }
+}

src/ServiceLayer.Mesh/Models/MeshFileStatus.cs

@@ -0,0 +1,12 @@
+namespace ServiceLayer.Mesh.Models;
+
+public enum MeshFileStatus
+{
+    Discovered,
+    Extracting,
+    Extracted,
+    Transforming,
+    Transformed,
+    FailedExtract,
+    FailedTransform
+}

src/ServiceLayer.Mesh/Models/MeshFileType.cs

@@ -0,0 +1,6 @@
+namespace ServiceLayer.Mesh.Models;
+
+public enum MeshFileType
+{
+    NbssAppointmentEvents
+}

src/ServiceLayer.Mesh/Program.cs

@@ -0,0 +1,61 @@
+using Microsoft.Azure.Functions.Worker.Builder;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Azure.Storage.Queues;
+using Azure.Identity;
+using Microsoft.EntityFrameworkCore;
+using NHS.MESH.Client;
+using ServiceLayer.Mesh.Data;
+
+var host = new HostBuilder()
+    .ConfigureFunctionsWebApplication()
+    .ConfigureServices(services =>
+    {
+        // MESH Client config
+        services
+            .AddMeshClient(_ => _.MeshApiBaseUrl = Environment.GetEnvironmentVariable("MeshApiBaseUrl"))
+            .AddMailbox(Environment.GetEnvironmentVariable("BSSMailBox"), new NHS.MESH.Client.Configuration.MailboxConfiguration
+            {
+                Password = Environment.GetEnvironmentVariable("MeshPassword"),
+                SharedKey = Environment.GetEnvironmentVariable("MeshSharedKey"),
+            }).Build();
+
+        // EF Core DbContext
+        services.AddDbContext<ServiceLayerDbContext>(options =>
+        {
+            var connectionString = Environment.GetEnvironmentVariable("DatabaseConnectionString");
+            if (string.IsNullOrEmpty(connectionString))
+                throw new InvalidOperationException("The connection string has not been initialized.");
+
+            options.UseSqlServer(connectionString);
+        });
+
+        // Register QueueClient as singleton
+        services.AddSingleton(provider =>
+        {
+            var environment = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
+            var queueUrl = Environment.GetEnvironmentVariable("QueueUrl");
+
+            if (string.IsNullOrWhiteSpace(queueUrl))
+                throw new InvalidOperationException("QueueUrl environment variable is not set.");
+
+            if (environment == "Development")
+            {
+                return new QueueClient("UseDevelopmentStorage=true", "my-local-queue");
+            }
+            else
+            {
+                var credential = new ManagedIdentityCredential();
+                return new QueueClient(new Uri(queueUrl), credential);
+            }
+        });
+    });
+
+
+// Application Insights isn't enabled by default. See https://aka.ms/AAt8mw4.
+// builder.Services
+//     .AddApplicationInsightsTelemetryWorkerService()
+//     .ConfigureFunctionsApplicationInsights();
+
+var app = host.Build();
+await app.RunAsync();

src/ServiceLayer.Mesh/Properties/launchSettings.json

@@ -0,0 +1,9 @@
+{
+  "profiles": {
+    "ServiceLayer.Mesh": {
+      "commandName": "Project",
+      "commandLineArgs": "--port 7142",
+      "launchBrowser": false
+    }
+  }
+}

src/ServiceLayer.Mesh/ServiceLayer.Mesh.csproj

@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <AzureFunctionsVersion>v4</AzureFunctionsVersion>
+    <OutputType>Exe</OutputType>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    <PackageReference Include="Azure.Identity" Version="1.13.2" />
+    <PackageReference Include="Azure.Storage.Queues" Version="12.22.0" />
+    <!-- Application Insights isn't enabled by default. See https://aka.ms/AAt8mw4. -->
+    <!-- <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.22.0" /> -->
+    <!-- <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" /> -->
+    <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" Version="4.3.1" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="2.0.0" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.1" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.1" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.1" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Update="host.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="local.settings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToPublishDirectory>Never</CopyToPublishDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <Using Include="System.Threading.ExecutionContext" Alias="ExecutionContext" />
+    <ProjectReference Include="..\dotnet-mesh-client\application\DotNetMeshClient\NHS.Mesh.Client\NHS.Mesh.Client.csproj" />
+  </ItemGroup>
+</Project>

src/ServiceLayer.Mesh/host.json

@@ -0,0 +1,12 @@
+{
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      },
+      "enableLiveMetricsFilters": true
+    }
+  }
+}

src/ServiceLayer.sln

@@ -9,6 +9,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{0AB3BF05
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.API.Tests", "..\tests\ServiceLayer.API.Tests\ServiceLayer.API.Tests.csproj", "{BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ServiceLayer.Mesh", "ServiceLayer.Mesh\ServiceLayer.Mesh.csproj", "{803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}"
+EndProject
 Global
   GlobalSection(SolutionConfigurationPlatforms) = preSolution
     Debug|Any CPU = Debug|Any CPU
@@ -43,6 +45,18 @@ Global
     {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x64.Build.0 = Release|Any CPU
     {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x86.ActiveCfg = Release|Any CPU
     {BA052DAE-6FD1-483A-A0AF-DCBCF9E38C72}.Release|x86.Build.0 = Release|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x64.ActiveCfg = Debug|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x64.Build.0 = Debug|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x86.ActiveCfg = Debug|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Debug|x86.Build.0 = Debug|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|Any CPU.Build.0 = Release|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x64.ActiveCfg = Release|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x64.Build.0 = Release|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x86.ActiveCfg = Release|Any CPU
+    {803E8A5E-A180-4799-8BE2-DD5BD3C34ED2}.Release|x86.Build.0 = Release|Any CPU
   EndGlobalSection
   GlobalSection(SolutionProperties) = preSolution
     HideSolutionNode = FALSE