Fix Chat documents ingestion (#338)

Author: MikeAlhayek

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/IAIChatSessionHandler.cs

@@ -1,13 +1,17 @@
 using CrestApps.OrchardCore.AI.Models;
+using CrestApps.OrchardCore.Services;
 
 namespace CrestApps.OrchardCore.AI;
 
 /// <summary>
 /// Handles lifecycle events raised during an AI chat session, such as when
 /// a message exchange completes. Implementations can perform post-processing
 /// tasks like data extraction, analytics, or workflow triggers.
+/// Inherits from <see cref="ICatalogEntryHandler{T}"/> to support standard
+/// lifecycle events (Initializing, Initialized, Creating, Created, Loaded,
+/// Deleting, Deleted, Updating, Updated, Validating, Validated).
 /// </summary>
-public interface IAIChatSessionHandler
+public interface IAIChatSessionHandler : ICatalogEntryHandler<AIChatSession>
 {
     /// <summary>
     /// Called after a user message has been processed and the assistant response

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/Models/DataSourceMetadata.cs

@@ -0,0 +1,12 @@
+namespace CrestApps.OrchardCore.AI.Models;
+
+/// <summary>
+/// Metadata for storing data source configuration on an entity.
+/// </summary>
+public sealed class DataSourceMetadata
+{
+    /// <summary>
+    /// Gets or sets the data source ID.
+    /// </summary>
+    public string DataSourceId { get; set; }
+}

src/Core/CrestApps.OrchardCore.AI.Core/Handlers/AIChatSessionHandlerBase.cs

@@ -0,0 +1,17 @@
+using CrestApps.OrchardCore.AI.Models;
+using CrestApps.OrchardCore.Core.Handlers;
+
+namespace CrestApps.OrchardCore.AI.Core.Handlers;
+
+/// <summary>
+/// Base class for <see cref="IAIChatSessionHandler"/> implementations.
+/// Provides virtual no-op implementations for all lifecycle events
+/// inherited from <see cref="ICatalogEntryHandler{AIChatSession}"/>
+/// and <see cref="IAIChatSessionHandler.MessageCompletedAsync"/>.
+/// </summary>
+public abstract class AIChatSessionHandlerBase : CatalogEntryHandlerBase<AIChatSession>, IAIChatSessionHandler
+{
+    /// <inheritdoc/>
+    public virtual Task MessageCompletedAsync(ChatMessageCompletedContext context)
+        => Task.CompletedTask;
+}

src/Core/CrestApps.OrchardCore.AI.Core/Models/AIProfileDataSourceMetadata.cs

@@ -2,12 +2,8 @@
 using CrestApps.OrchardCore.AI.Core.Indexes;
 using CrestApps.OrchardCore.AI.Models;
 using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using OrchardCore;
-using OrchardCore.Environment.Shell.Scope;
-using OrchardCore.Indexing;
-using OrchardCore.Indexing.Models;
 using OrchardCore.Modules;
 using YesSql;
 using ISession = YesSql.ISession;
@@ -20,23 +16,26 @@ public sealed class DefaultAIChatSessionManager : IAIChatSessionManager
     private readonly IHttpContextAccessor _httpContextAccessor;
     private readonly IClientIPAddressAccessor _clientIPAddressAccessor;
     private readonly ISession _session;
-    private readonly IAIDocumentStore _documentStore;
     private readonly IAIChatSessionPromptStore _promptStore;
+    private readonly IEnumerable<IAIChatSessionHandler> _handlers;
+    private readonly ILogger _logger;
 
     public DefaultAIChatSessionManager(
         IClock clock,
         IHttpContextAccessor httpContextAccessor,
         IClientIPAddressAccessor clientIPAddressAccessor,
         ISession session,
-        IAIDocumentStore documentStore,
-        IAIChatSessionPromptStore promptStore)
+        IAIChatSessionPromptStore promptStore,
+        IEnumerable<IAIChatSessionHandler> handlers,
+        ILogger<DefaultAIChatSessionManager> logger)
     {
         _clock = clock;
         _httpContextAccessor = httpContextAccessor;
         _clientIPAddressAccessor = clientIPAddressAccessor;
         _session = session;
-        _documentStore = documentStore;
         _promptStore = promptStore;
+        _handlers = handlers;
+        _logger = logger;
     }
 
     public async Task<AIChatSession> NewAsync(AIProfile profile, NewAIChatSessionContext context)
@@ -193,13 +192,17 @@ public async Task<bool> DeleteAsync(string sessionId)
             return false;
         }
 
-        await CleanupSessionDocumentsAsync(chatSession);
+        var deletingContext = new CrestApps.OrchardCore.Models.DeletingContext<AIChatSession>(chatSession);
+        await _handlers.InvokeAsync((handler, ctx) => handler.DeletingAsync(ctx), deletingContext, _logger);
 
         // Delete all prompts associated with this session.
         await _promptStore.DeleteAllPromptsAsync(chatSession.SessionId);
 
         _session.Delete(chatSession, collection: AIConstants.CollectionName);
 
+        var deletedContext = new CrestApps.OrchardCore.Models.DeletedContext<AIChatSession>(chatSession);
+        await _handlers.InvokeAsync((handler, ctx) => handler.DeletedAsync(ctx), deletedContext, _logger);
+
         return true;
     }
 
@@ -225,85 +228,20 @@ public async Task<int> DeleteAllAsync(string profileId)
 
         foreach (var session in sessions)
         {
-            await CleanupSessionDocumentsAsync(session);
+            var deletingContext = new CrestApps.OrchardCore.Models.DeletingContext<AIChatSession>(session);
+            await _handlers.InvokeAsync((handler, ctx) => handler.DeletingAsync(ctx), deletingContext, _logger);
 
             // Delete all prompts associated with this session.
             await _promptStore.DeleteAllPromptsAsync(session.SessionId);
 
             _session.Delete(session, collection: AIConstants.CollectionName);
-            totalDeleted++;
-        }
-
-        return totalDeleted;
-    }
-
-    /// <summary>
-    /// Removes all documents associated with the given session from the document store
-    /// and schedules deferred removal of their chunks from all AI document indexes.
-    /// </summary>
-    private async Task CleanupSessionDocumentsAsync(AIChatSession session)
-    {
-        var documents = await _documentStore.GetDocumentsAsync(
-            session.SessionId,
-            AIConstants.DocumentReferenceTypes.ChatSession);
-
-        if (documents.Count == 0)
-        {
-            return;
-        }
-
-        var chunkIds = new List<string>();
-
-        foreach (var doc in documents)
-        {
-            if (doc.Chunks != null)
-            {
-                for (var i = 0; i < doc.Chunks.Count; i++)
-                {
-                    chunkIds.Add($"{doc.ItemId}_{i}");
-                }
-            }
-
-            await _documentStore.DeleteAsync(doc);
-        }
-
-        if (chunkIds.Count > 0)
-        {
-            ShellScope.AddDeferredTask(scope => RemoveDocumentChunksAsync(scope, chunkIds));
-        }
-    }
-
-    private static async Task RemoveDocumentChunksAsync(ShellScope scope, List<string> chunkIds)
-    {
-        var services = scope.ServiceProvider;
-        var indexStore = services.GetRequiredService<IIndexProfileStore>();
 
-        var indexProfiles = await indexStore.GetByTypeAsync(AIConstants.AIDocumentsIndexingTaskType);
+            var deletedContext = new CrestApps.OrchardCore.Models.DeletedContext<AIChatSession>(session);
+            await _handlers.InvokeAsync((handler, ctx) => handler.DeletedAsync(ctx), deletedContext, _logger);
 
-        if (!indexProfiles.Any())
-        {
-            return;
+            totalDeleted++;
         }
 
-        var logger = services.GetRequiredService<ILogger<DefaultAIChatSessionManager>>();
-
-        foreach (var indexProfile in indexProfiles)
-        {
-            var documentIndexManager = services.GetKeyedService<IDocumentIndexManager>(indexProfile.ProviderName);
-
-            if (documentIndexManager == null)
-            {
-                continue;
-            }
-
-            try
-            {
-                await documentIndexManager.DeleteDocumentsAsync(indexProfile, chunkIds);
-            }
-            catch (Exception ex)
-            {
-                logger.LogError(ex, "Error removing session document chunks from index '{IndexName}'.", indexProfile.IndexName);
-            }
-        }
+        return totalDeleted;
     }
 }

src/Core/CrestApps.OrchardCore.AI.Core/Services/DefaultAIChatSessionManager.cs

@@ -23,6 +23,10 @@ The **AI Chat** feature builds upon the **AI Services** feature by adding AI cha
 - **Azure AI Inference Chat** (`CrestApps.OrchardCore.AzureAIInference`): AI services using Azure AI Inference (GitHub models) models.
 - **Ollama AI Chat** (`CrestApps.OrchardCore.Ollama`): AI-powered chat using Ollama service.
 
+### Welcome Message Behavior
+
+When an AI profile has a **Welcome Message** configured, it is displayed to the user as the first message in the chat. The welcome message is also included in the conversation history sent to the AI model as an assistant message, ensuring the model understands the full context of the conversation — including any questions posed in the welcome message that the user may respond to in their first prompt.
+
 ### Admin Chat User Interface
 
 ![Screen cast of the admin chat](/img/docs/admin-ui-sample.gif)

src/CrestApps.OrchardCore.Documentations/docs/ai/chat.md

@@ -55,5 +55,5 @@ To use these files, please convert them to the newer formats (.docx, .xlsx, .ppt
 ### Content Extraction Notes
 
 - **Word**: Extracts text from all paragraphs in the main document body
-- **Excel**: Extracts data row-by-row, with cells separated by tabs
+- **Excel**: Extracts data row-by-row, with cells separated by tabs. Supports shared strings, inline strings, numeric values, and boolean cells.
 - **PowerPoint**: Extracts text from all text elements across all slides

src/CrestApps.OrchardCore.Documentations/docs/ai/documents/openxml.md

@@ -194,6 +194,36 @@ The citation and reference system has been completely reworked so that **every A
 
 ---
 
+### Fix: Welcome Message Now Included in Chat History
+
+When an AI Profile has a `WelcomeMessage`, it is the first message the user sees. Previously, the model had no knowledge of this welcome message, so it could not understand the user's first response (e.g., answering a question posed in the welcome message). The welcome message is now prepended as the first assistant message in the conversation history, giving the model full context of the conversation from the start.
+
+---
+
+### Fix: Chat Session Handler Lifecycle Events
+
+`IAIChatSessionHandler` now extends `ICatalogEntryHandler<AIChatSession>`, adding full lifecycle events: `InitializingAsync`, `InitializedAsync`, `CreatingAsync`, `CreatedAsync`, `LoadedAsync`, `DeletingAsync`, `DeletedAsync`, `UpdatingAsync`, `UpdatedAsync`, `ValidatingAsync`, and `ValidatedAsync`. A new `AIChatSessionHandlerBase` base class (extending `CatalogEntryHandlerBase<AIChatSession>`) provides virtual no-op implementations for all lifecycle methods plus the existing `MessageCompletedAsync`, so handler implementations only need to override the events they care about.
+
+`DefaultAIChatSessionManager` no longer depends on `IAIDocumentStore` directly. Instead, it invokes `DeletingAsync`/`DeletedAsync` lifecycle events on all registered `IAIChatSessionHandler` implementations when sessions are deleted. Document cleanup is now handled by a dedicated `AIChatSessionDocumentCleanupHandler` registered in the AI Documents feature, resolving a dependency injection exception when the AI Documents feature was not enabled.
+
+**Breaking:** `IAIChatSessionHandler` now inherits from `ICatalogEntryHandler<AIChatSession>`. Existing implementations should extend `AIChatSessionHandlerBase` instead of implementing the interface directly to avoid having to implement all lifecycle methods.
+
+---
+
+### Fix: OpenXml Excel Data Extraction
+
+Fixed `OpenXmlIngestionDocumentReader.GetCellValue` to correctly handle Excel cells stored as **inline strings** (`InlineString` cell type) and **boolean** values. Previously, inline string cells returned empty text because the code only checked `CellValue`, which is `null` for inline strings — the text is stored in the `InlineString` element instead. Shared string table lookup was also changed from LINQ `ElementAtOrDefault` to direct `ChildElements` indexer for O(1) access.
+
+---
+
+### Refactor: Unified DataSourceMetadata Type
+
+`AIProfileDataSourceMetadata` and `ChatInteractionDataSourceMetadata` have been merged into a single `DataSourceMetadata` type in `CrestApps.OrchardCore.AI.Models`. Both types were identical (containing only a `DataSourceId` property) and served the same purpose on different entity types. Data migrations automatically rename the stored JSON property keys from the legacy names to `DataSourceMetadata`.
+
+**Breaking:** `AIProfileDataSourceMetadata` (from `CrestApps.OrchardCore.AI.Core.Models`) and `ChatInteractionDataSourceMetadata` (from `CrestApps.OrchardCore.AI.Chat.Interactions`) have been removed. Use `DataSourceMetadata` (from `CrestApps.OrchardCore.AI.Models`) instead.
+
+---
+
 ## Breaking Changes
 
 ### Changed: Navigation Paths

src/CrestApps.OrchardCore.Documentations/docs/changelog/v2.0.0.md

@@ -40,7 +40,7 @@ public override IDisplayResult Edit(ChatInteraction interaction, BuildEditorCont
         {
             var dataSourceSettings = await _siteService.GetSettingsAsync<AIDataSourceSettings>();
 
-            var metadata = interaction.As<ChatInteractionDataSourceMetadata>();
+            var metadata = interaction.As<DataSourceMetadata>();
             model.DataSourceId = metadata?.DataSourceId;
 
             var ragMetadata = interaction.As<AIDataSourceRagMetadata>();
@@ -66,7 +66,7 @@ public override async Task<IDisplayResult> UpdateAsync(ChatInteraction interacti
 
             if (dataSource != null)
             {
-                interaction.Put(new ChatInteractionDataSourceMetadata
+                interaction.Put(new DataSourceMetadata
                 {
                     DataSourceId = dataSource.ItemId,
                 });
@@ -75,7 +75,7 @@ public override async Task<IDisplayResult> UpdateAsync(ChatInteraction interacti
         else
         {
             // Clear the metadata if no data source is selected
-            interaction.Put(new ChatInteractionDataSourceMetadata());
+            interaction.Put(new DataSourceMetadata());
         }
 
         var dataSourceSettings = await _siteService.GetSettingsAsync<AIDataSourceSettings>();