feat: AI Chat and embeddings

.github/workflows/Build-Test-And-Deploy.yml

@@ -147,14 +147,20 @@ jobs:
               emailsender-secret=keyvaultref:$KEYVAULTURI/secrets/authmessagesender-secretkey,identityref:$MANAGEDIDENTITYID emailsender-name=keyvaultref:$KEYVAULTURI/secrets/authmessagesender-sendfromname,identityref:$MANAGEDIDENTITYID \
               emailsender-email=keyvaultref:$KEYVAULTURI/secrets/authmessagesender-sendfromemail,identityref:$MANAGEDIDENTITYID connectionstring=keyvaultref:$KEYVAULTURI/secrets/connectionstrings-essentialcsharpwebcontextconnection,identityref:$MANAGEDIDENTITYID \
               captcha-sitekey=keyvaultref:$KEYVAULTURI/secrets/captcha-sitekey,identityref:$MANAGEDIDENTITYID captcha-secretkey=keyvaultref:$KEYVAULTURI/secrets/captcha-secretkey,identityref:$MANAGEDIDENTITYID \
-              appinsights-connectionstring=keyvaultref:$KEYVAULTURI/secrets/applicationinsights-connectionstring,identityref:$MANAGEDIDENTITYID
+              appinsights-connectionstring=keyvaultref:$KEYVAULTURI/secrets/applicationinsights-connectionstring,identityref:$MANAGEDIDENTITYID \
+              ai-endpoint=keyvaultref:$KEYVAULTURI/secrets/AIOptions--Endpoint,identityref:$MANAGEDIDENTITYID ai-apikey=keyvaultref:$KEYVAULTURI/secrets/AIOptions--ApiKey,identityref:$MANAGEDIDENTITYID \
+              ai-vectordeployment=keyvaultref:$KEYVAULTURI/secrets/AIOptions--VectorGenerationDeploymentName,identityref:$MANAGEDIDENTITYID ai-chatdeployment=keyvaultref:$KEYVAULTURI/secrets/AIOptions--ChatDeploymentName,identityref:$MANAGEDIDENTITYID \
+              ai-systemprompt=keyvaultref:$KEYVAULTURI/secrets/AIOptions--SystemPrompt,identityref:$MANAGEDIDENTITYID \
+              postgres-vectorstore-connectionstring=keyvaultref:$KEYVAULTURI/secrets/connectionstrings--PostgresVectorDb,identityref:$MANAGEDIDENTITYID
             az containerapp update --name $CONTAINER_APP_NAME --resource-group $RESOURCEGROUP --replace-env-vars Authentication__github__clientId=secretref:github-clientid Authentication__github__clientSecret=secretref:github-clientsecret \
               Authentication__microsoft__clientId=secretref:msft-clientid Authentication__microsoft__clientSecret=secretref:msft-clientsecret AuthMessageSender__ApiKey=secretref:emailsender-apikey AuthMessageSender__SecretKey=secretref:emailsender-secret \
               AuthMessageSender__SendFromName=secretref:emailsender-name AuthMessageSender__SendFromEmail=secretref:emailsender-email ConnectionStrings__EssentialCSharpWebContextConnection=secretref:connectionstring ASPNETCORE_ENVIRONMENT=Staging \
-              AZURE_CLIENT_ID=$AZURECLIENTID HCaptcha__SiteKey=secretref:captcha-sitekey HCaptcha__SecretKey=secretref:captcha-secretkey ApplicationInsights__ConnectionString=secretref:appinsights-connectionstring
+              AZURE_CLIENT_ID=$AZURECLIENTID HCaptcha__SiteKey=secretref:captcha-sitekey HCaptcha__SecretKey=secretref:captcha-secretkey ApplicationInsights__ConnectionString=secretref:appinsights-connectionstring \
+              AIOptions__Endpoint=secretref:ai-endpoint AIOptions__ApiKey=secretref:ai-apikey AIOptions__VectorGenerationDeploymentName=secretref:ai-vectordeployment AIOptions__ChatDeploymentName=secretref:ai-chatdeployment \
+              AIOptions__SystemPrompt=secretref:ai-systemprompt ConnectionStrings__PostgresVectorStore=secretref:postgres-vectorstore-connectionstring
 
       - name: Logout of Azure CLI
-        if: "always()"
+        if: always()
         uses: azure/CLI@v2
         with:
           inlineScript: |
@@ -233,14 +239,19 @@ jobs:
               emailsender-secret=keyvaultref:$KEYVAULTURI/secrets/authmessagesender-secretkey,identityref:$MANAGEDIDENTITYID emailsender-name=keyvaultref:$KEYVAULTURI/secrets/authmessagesender-sendfromname,identityref:$MANAGEDIDENTITYID \
               emailsender-email=keyvaultref:$KEYVAULTURI/secrets/authmessagesender-sendfromemail,identityref:$MANAGEDIDENTITYID connectionstring=keyvaultref:$KEYVAULTURI/secrets/connectionstrings-essentialcsharpwebcontextconnection,identityref:$MANAGEDIDENTITYID \
               captcha-sitekey=keyvaultref:$KEYVAULTURI/secrets/captcha-sitekey,identityref:$MANAGEDIDENTITYID captcha-secretkey=keyvaultref:$KEYVAULTURI/secrets/captcha-secretkey,identityref:$MANAGEDIDENTITYID \
-              appinsights-connectionstring=keyvaultref:$KEYVAULTURI/secrets/applicationinsights-connectionstring,identityref:$MANAGEDIDENTITYID
+              appinsights-connectionstring=keyvaultref:$KEYVAULTURI/secrets/applicationinsights-connectionstring,identityref:$MANAGEDIDENTITYID \
+              ai-endpoint=keyvaultref:$KEYVAULTURI/secrets/AIOptions--Endpoint,identityref:$MANAGEDIDENTITYID ai-apikey=keyvaultref:$KEYVAULTURI/secrets/AIOptions--ApiKey,identityref:$MANAGEDIDENTITYID \
+              ai-vectordeployment=keyvaultref:$KEYVAULTURI/secrets/AIOptions--VectorGenerationDeploymentName,identityref:$MANAGEDIDENTITYID ai-chatdeployment=keyvaultref:$KEYVAULTURI/secrets/AIOptions--ChatDeploymentName,identityref:$MANAGEDIDENTITYID \
+              ai-systemprompt=keyvaultref:$KEYVAULTURI/secrets/AIOptions--SystemPrompt,identityref:$MANAGEDIDENTITYID \
+              postgres-vectorstore-connectionstring=keyvaultref:$KEYVAULTURI/secrets/connectionstrings--PostgresVectorDb,identityref:$MANAGEDIDENTITYID
             az containerapp update --name $CONTAINER_APP_NAME --resource-group $RESOURCEGROUP --replace-env-vars Authentication__github__clientId=secretref:github-clientid Authentication__github__clientSecret=secretref:github-clientsecret \
               Authentication__microsoft__clientId=secretref:msft-clientid Authentication__microsoft__clientSecret=secretref:msft-clientsecret AuthMessageSender__ApiKey=secretref:emailsender-apikey AuthMessageSender__SecretKey=secretref:emailsender-secret \
               AuthMessageSender__SendFromName=secretref:emailsender-name AuthMessageSender__SendFromEmail=secretref:emailsender-email ConnectionStrings__EssentialCSharpWebContextConnection=secretref:connectionstring ASPNETCORE_ENVIRONMENT=Production \
-              AZURE_CLIENT_ID=$AZURECLIENTID HCaptcha__SiteKey=secretref:captcha-sitekey HCaptcha__SecretKey=secretref:captcha-secretkey ApplicationInsights__ConnectionString=secretref:appinsights-connectionstring
+              AZURE_CLIENT_ID=$AZURECLIENTID HCaptcha__SiteKey=secretref:captcha-sitekey HCaptcha__SecretKey=secretref:captcha-secretkey ApplicationInsights__ConnectionString=secretref:appinsights-connectionstring \
+              AIOptions__Endpoint=secretref:ai-endpoint AIOptions__ApiKey=secretref:ai-apikey ConnectionStrings__PostgresVectorStore=secretref:postgres-vectorstore-connectionstring
 
       - name: Logout of Azure CLI
-        if: "always()"
+        if: always()
         uses: azure/CLI@v2
         with:
           inlineScript: |

Directory.Packages.props

@@ -36,12 +36,19 @@
     <PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.10" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.10" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel" Version="1.60.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.PgVector" Version="1.60.0-preview" />
+    <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="9.0.0" />
+    <PackageVersion Include="ModelContextProtocol" Version="0.3.0-preview.3" />
+    <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.3.0-preview.3" />
+    <PackageVersion Include="Moq" Version="4.20.72" />
+    <PackageVersion Include="System.CommandLine" Version="2.0.0-beta6.25358.103" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Octokit" Version="14.0.0" />
     <PackageVersion Include="DotnetSitemapGenerator" Version="1.0.4" />
     <PackageVersion Include="xunit" Version="2.9.3" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="3.0.1" />
   </ItemGroup>
-</Project>
+</Project>

EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SemanticKernel" />
+    <PackageReference Include="Microsoft.SemanticKernel.Connectors.PgVector" />
+    <PackageReference Include="ModelContextProtocol" />
+    <PackageReference Include="ModelContextProtocol.AspNetCore" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+</Project>

EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs

@@ -0,0 +1,85 @@
+using Azure.AI.OpenAI;
+using EssentialCSharp.Chat.Common.Services;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.SemanticKernel;
+
+namespace EssentialCSharp.Chat.Common.Extensions;
+
+public static class ServiceCollectionExtensions
+{
+    /// <summary>
+    /// Adds Azure OpenAI and related AI services to the service collection
+    /// </summary>
+    /// <param name="services">The service collection to add services to</param>
+    /// <param name="aiOptions">The AI configuration options</param>
+    /// <param name="postgresConnectionString">The PostgreSQL connection string for the vector store</param>
+    /// <returns>The service collection for chaining</returns>
+    public static IServiceCollection AddAzureOpenAIServices(this IServiceCollection services, AIOptions aiOptions, string postgresConnectionString)
+    {
+        if (string.IsNullOrEmpty(aiOptions.Endpoint) ||
+            string.IsNullOrEmpty(aiOptions.ApiKey))
+            // Register Azure OpenAI services
+#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
+            services.AddAzureOpenAIEmbeddingGenerator(
+                aiOptions.VectorGenerationDeploymentName,
+                aiOptions.Endpoint,
+                aiOptions.ApiKey);
+
+        services.AddAzureOpenAIChatClient(
+            aiOptions.ChatDeploymentName,
+            aiOptions.Endpoint,
+            aiOptions.ApiKey);
+
+        services.AddSingleton(provider =>
+            new AzureOpenAIClient(new Uri(aiOptions.Endpoint), new Azure.AzureKeyCredential(aiOptions.ApiKey)));
+
+        // Register Azure OpenAI services
+        services.AddAzureOpenAIEmbeddingGenerator(
+            aiOptions.VectorGenerationDeploymentName,
+            aiOptions.Endpoint,
+            aiOptions.ApiKey);
+
+        services.AddAzureOpenAIChatCompletion(
+            aiOptions.ChatDeploymentName,
+            aiOptions.Endpoint,
+            aiOptions.ApiKey);
+
+        // Add PostgreSQL vector store
+        services.AddPostgresVectorStore(postgresConnectionString);
+
+#pragma warning restore SKEXP0010
+
+        // Register shared AI services
+        services.AddSingleton<EmbeddingService>();
+        services.AddSingleton<AISearchService>();
+        services.AddSingleton<AIChatService>();
+        services.AddSingleton<MarkdownChunkingService>();
+
+        return services;
+    }
+
+    /// <summary>
+    /// Adds Azure OpenAI and related AI services to the service collection using configuration
+    /// </summary>
+    /// <param name="services">The service collection to add services to</param>
+    /// <param name="configuration">The configuration to read AIOptions from</param>
+    /// <returns>The service collection for chaining</returns>
+    public static IServiceCollection AddAzureOpenAIServices(this IServiceCollection services, IConfiguration configuration)
+    {
+        // Configure AI options from configuration
+        services.Configure<AIOptions>(configuration.GetSection("AIOptions"));
+
+        var aiOptions = configuration.GetSection("AIOptions").Get<AIOptions>();
+        if (aiOptions == null)
+        {
+            throw new InvalidOperationException("AIOptions section is missing from configuration.");
+        }
+
+        // Get PostgreSQL connection string using the standard method
+        var postgresConnectionString = configuration.GetConnectionString("PostgresVectorStore") ??
+            throw new InvalidOperationException("Connection string 'PostgresVectorStore' not found.");
+
+        return services.AddAzureOpenAIServices(aiOptions, postgresConnectionString);
+    }
+}

EssentialCSharp.Chat.Shared/Models/AIOptions.cs

@@ -0,0 +1,29 @@
+namespace EssentialCSharp.Chat;
+
+public class AIOptions
+{
+    /// <summary>
+    /// The Azure OpenAI deployment name for text embedding generation.
+    /// </summary>
+    public string VectorGenerationDeploymentName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The Azure OpenAI deployment name for chat completions.
+    /// </summary>
+    public string ChatDeploymentName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The system prompt to use for the chat model.
+    /// </summary>
+    public string SystemPrompt { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The Azure OpenAI endpoint URL.
+    /// </summary>
+    public string Endpoint { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The API key for accessing Azure OpenAI services.
+    /// </summary>
+    public string ApiKey { get; set; } = string.Empty;
+}

EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs

@@ -0,0 +1,54 @@
+using Microsoft.Extensions.VectorData;
+
+namespace EssentialCSharp.Chat.Common.Models;
+
+/// <summary>
+/// Represents a chunk of book content for vector search
+/// </summary>
+public sealed class BookContentChunk
+{
+    /// <summary>
+    /// Unique identifier for the chunk - serves as the vector store key
+    /// </summary>
+    [VectorStoreKey]
+    public string Id { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Original source file name
+    /// </summary>
+    [VectorStoreData]
+    public string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Heading or title of the markdown chunk
+    /// </summary>
+    [VectorStoreData]
+    public string Heading { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The actual markdown content text for this chunk
+    /// </summary>
+    [VectorStoreData]
+    public string ChunkText { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Chapter number extracted from filename (e.g., "Chapter01.md" -> 1)
+    /// </summary>
+    [VectorStoreData]
+    public int? ChapterNumber { get; set; }
+
+    /// <summary>
+    /// SHA256 hash of the chunk content for change detection
+    /// </summary>
+    [VectorStoreData]
+    public string ContentHash { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Vector embedding for the chunk text - will be generated by embedding service
+    /// Using 1536 dimensions for Azure OpenAI text-embedding-3-small-v1
+    /// Use CosineSimilarity distance function since we are using text-embedding-3 (https://platform.openai.com/docs/guides/embeddings#which-distance-function-should-i-use)
+    /// Postgres supports only Hnsw: https://learn.microsoft.com/en-us/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/postgres-connector?pivots=programming-language-csharp&WT.mc_id=8B97120A00B57354
+    /// </summary>
+    [VectorStoreVector(Dimensions: 1536, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Hnsw)]
+    public ReadOnlyMemory<float>? TextEmbedding { get; set; }
+}