Refactor and update Cosmos DB integration

Removed Certification.cs and SkillsMeasured.cs files.
Updated ProcessFile.cs:
  - Removed EmbeddingClient initialization.
  - Modified Run method to return CertServiceDocument.
  - Changed GenerateEmbeddings to return List<float>.
  - Reformatted ValidateJsonContent method.
Added CertServiceDocument class for Cosmos DB structure.
Updated azure-project-generator.csproj with CosmosDB package.

Author: madebygps

azure-project-generator/CertServiceDocument.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Threading.Tasks;
+
+namespace azure_project_generator
+{
+    public class CertServiceDocument
+    {
+        [JsonPropertyName("id")]
+        public string id { get; set; }  // Unique identifier for the document
+
+        [JsonPropertyName("certificationServiceKey")]
+        public string CertificationServiceKey { get; set; }  // Composite key
+        [JsonPropertyName("certificationCode")]
+        public string CertificationCode { get; set; }  // The certification code
+        [JsonPropertyName("certificationName")]
+        public string CertificationName { get; set; }  // The certification name
+        [JsonPropertyName("skillName")]
+        public string SkillName { get; set; }  // The skill associated with this certification
+        [JsonPropertyName("topicName")]
+        public string TopicName { get; set; }  // The topic within the skill
+        [JsonPropertyName("serviceName")]
+        public string ServiceName { get; set; }  // The service relevant to this certification and skill
+        [JsonPropertyName("contextSentence")]
+        public string ContextSentence { get; set; }  // The combined sentence
+        [JsonPropertyName("contextVector")]
+        public float[] ContextVector { get; set; }  // Example vector embedding generated from the sentence
+
+    }
+}

azure-project-generator/Certification.cs

@@ -13,10 +13,7 @@ namespace azure_project_generator
     public class ProcessFile
     {
         private readonly ILogger<ProcessFile> _logger;
-
         private readonly EmbeddingClient _embeddingClient;
-
-
         public ProcessFile(ILogger<ProcessFile> logger)
         {
             _logger = logger;
@@ -40,9 +37,10 @@ public ProcessFile(ILogger<ProcessFile> logger)
 
         }
 
-
         [Function(nameof(ProcessFile))]
-        public async Task Run([BlobTrigger("certdata/{name}", Connection = "AzureWebJobsStorage")] Stream stream, string name)
+        [CosmosDBOutput("%CosmosDb%", "%CosmosContainerOut%", Connection = "CosmosDBConnection")]
+        public async Task<CertServiceDocument> Run(
+            [BlobTrigger("certdata/{name}", Connection = "AzureWebJobsStorage")] Stream stream, string name)
         {
             string content;
             try
@@ -53,21 +51,20 @@ public async Task Run([BlobTrigger("certdata/{name}", Connection = "AzureWebJobs
             catch (IOException ex)
             {
                 _logger.LogError($"Error reading blob content: {ex.Message}");
-                return;
+                return null;
             }
 
             _logger.LogInformation($"C# Blob trigger function Processed blob\n Name: {name}");
 
             if (string.IsNullOrWhiteSpace(content))
             {
                 _logger.LogError("Blob content is empty or whitespace.");
-                return;
+                return null;
             }
 
             try
             {
                 ValidateJsonContent(content);
-
             }
             catch (JsonReaderException ex)
             {
@@ -79,37 +76,49 @@ public async Task Run([BlobTrigger("certdata/{name}", Connection = "AzureWebJobs
             }
 
             var mappedServiceData = JsonConvert.DeserializeObject<MappedService>(content);
-           
-            string contextSentence = 
+
+            string contextSentence =
                 $"The {mappedServiceData.CertificationCode} {mappedServiceData.CertificationName} certification includes the skill of {mappedServiceData.SkillName}. Within this skill, there is a focus on the topic of {mappedServiceData.TopicName}, particularly through the use of the service {mappedServiceData.ServiceName}.";
 
-            await GenerateEmbeddings(contextSentence);
-           
+            List<float> contentVector = await GenerateEmbeddings(contextSentence);
+            CertServiceDocument certServiceDocument = new CertServiceDocument();
+            certServiceDocument.id = Guid.NewGuid().ToString();
+            certServiceDocument.CertificationServiceKey = $"{mappedServiceData.CertificationCode}-{mappedServiceData.ServiceName}";
+            certServiceDocument.CertificationCode = mappedServiceData.CertificationCode;
+            certServiceDocument.CertificationName = mappedServiceData.CertificationName;
+            certServiceDocument.SkillName = mappedServiceData.SkillName;
+            certServiceDocument.TopicName = mappedServiceData.TopicName;
+            certServiceDocument.ServiceName = mappedServiceData.ServiceName;
+            certServiceDocument.ContextSentence = contextSentence;
+            certServiceDocument.ContextVector = contentVector.ToArray();
 
-        }
+            _logger.LogInformation("Document created successfully.");
+
+            return certServiceDocument;
 
-        private async Task GenerateEmbeddings(string content)
+        }
+        private async Task<List<float>> GenerateEmbeddings(string content)
         {
-            
             try
             {
                 _logger.LogInformation("Generating embedding...");
-                Embedding embedding = await _embeddingClient.GenerateEmbeddingAsync(content).ConfigureAwait(false);
+                var embeddingResult = await _embeddingClient.GenerateEmbeddingAsync(content).ConfigureAwait(false);
+                List<float> embeddingVector = embeddingResult.Value.Vector.ToArray().ToList();
                 _logger.LogInformation("Embedding created successfully.");
-                // save to cosmos
-
-
+                return embeddingVector;
             }
             catch (RequestFailedException ex)
             {
                 _logger.LogError($"Azure OpenAI API request failed: {ex.Message}");
+                throw; // Re-throw the exception to ensure the caller is aware of the failure
             }
             catch (Exception ex)
             {
                 _logger.LogError($"Error generating embedding: {ex.Message}");
+                throw; // Re-throw the exception to ensure the caller is aware of the failure
             }
         }
-                private void ValidateJsonContent(string content)
+        private void ValidateJsonContent(string content)
         {
             var generator = new JSchemaGenerator();
             JSchema schema = generator.Generate(typeof(MappedService));

azure-project-generator/ProcessFile.cs

@@ -17,6 +17,7 @@
     <PackageReference Include="Azure.Storage.Files.Shares" Version="12.19.1" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.19.1" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.23.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.CosmosDB" Version="4.10.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.2.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs" Version="6.6.0" />
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.17.4" />