Add Knowledge Base References (#333)

Author: MikeAlhayek

Directory.Packages.props

@@ -9,20 +9,24 @@
     <PackageVersion Include="CrestApps.AgentSkills.OrchardCore" Version="1.0.0-preview-0001" />
     <PackageVersion Include="CrestApps.AgentSkills.Mcp.OrchardCore" Version="1.0.0-preview-0001" />
     <PackageVersion Include="DocumentFormat.OpenXml" Version="3.4.1" />
-    <PackageVersion Include="FluentFTP" Version="52.1.0" />
+    <PackageVersion Include="FluentFTP" Version="53.0.2" />
     <PackageVersion Include="JsonSchema.Net" Version="7.4.0" />
     <PackageVersion Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00017" />
+    <PackageVersion Include="Markdig" Version="1.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion" Version="10.3.0-preview.1.26109.11" />
+    <PackageVersion Include="Microsoft.Extensions.DataIngestion.Markdig" Version="10.3.0-preview.1.26109.11" />
+    <PackageVersion Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="2.0.0" />
     <PackageVersion Include="MongoDB.Driver" Version="3.4.0" />
     <PackageVersion Include="NuGet.Versioning" Version="7.0.1" />
     <PackageVersion Include="OllamaSharp" Version="5.4.16" />
-    <PackageVersion Include="SSH.NET" Version="2024.2.0" />
+    <PackageVersion Include="SSH.NET" Version="2025.1.0" />
     <PackageVersion Include="YesSql.Abstractions" Version="5.4.7" />
     <PackageVersion Include="YesSql.Core" Version="5.4.7" />
-    <PackageVersion Include="ModelContextProtocol" Version="0.8.0-preview.1" />
-    <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.8.0-preview.1" />
-    <PackageVersion Include="ModelContextProtocol.Core" Version="0.8.0-preview.1" />
+    <PackageVersion Include="ModelContextProtocol" Version="1.0.0" />
+    <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="1.0.0" />
+    <PackageVersion Include="ModelContextProtocol.Core" Version="1.0.0" />
     <PackageVersion Include="PdfPig" Version="0.1.13" />
-    <PackageVersion Include="GitHub.Copilot.SDK" Version="0.1.25" />
+    <PackageVersion Include="GitHub.Copilot.SDK" Version="0.1.26" />
   </ItemGroup>
   <ItemGroup>
     <!-- OrchardCore Packages -->
@@ -87,27 +91,27 @@
     <PackageVersion Include="Azure.Messaging.EventGrid" Version="5.0.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.7.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="10.0.3" />
-    <PackageVersion Include="Microsoft.Extensions.AI" Version="10.2.0" />
-    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.2.0" />
-    <PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.2.0-preview.1.26063.2" />
+    <PackageVersion Include="Microsoft.Extensions.AI" Version="10.3.0" />
+    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.3.0" />
+    <PackageVersion Include="Microsoft.Extensions.AI.OpenAI" Version="10.3.0" />
     <PackageVersion Include="Microsoft.Extensions.AI.AzureAIInference" Version="10.0.0-preview.1.25559.3" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
-    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.2.0" />
+    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.3.0" />
   </ItemGroup>
   <ItemGroup>
     <!-- Testing Packages -->
     <PackageVersion Include="Moq" Version="4.20.72" />
-    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.3.0" />
     <PackageVersion Include="xunit.v3" Version="3.2.2" />
     <PackageVersion Include="xunit.analyzers" Version="1.27.0" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
     <PackageVersion Include="xunit.runner.inproc" Version="3.2.0" />
   </ItemGroup>
   <ItemGroup>
     <!-- Aspire Packages -->
-    <PackageVersion Include="Aspire.Hosting.AppHost" Version="13.1.0" />
+    <PackageVersion Include="Aspire.Hosting.AppHost" Version="13.1.1" />
     <PackageVersion Include="Aspire.Hosting.Elasticsearch" Version="13.1.0" />
-    <PackageVersion Include="Aspire.Hosting.Redis" Version="13.1.0" />
+    <PackageVersion Include="Aspire.Hosting.Redis" Version="13.1.1" />
     <PackageVersion Include="CommunityToolkit.Aspire.Hosting.Ollama" Version="13.1.1" />
   </ItemGroup>
 </Project>

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/AICompletionContextKeys.cs

@@ -0,0 +1,17 @@
+namespace CrestApps.OrchardCore.AI;
+
+/// <summary>
+/// Provides well-known keys for <see cref="Models.AICompletionContext.AdditionalProperties"/>.
+/// These keys are used by orchestration handlers to communicate context availability
+/// to downstream consumers such as <see cref="IToolRegistryProvider"/> implementations.
+/// </summary>
+public static class AICompletionContextKeys
+{
+    /// <summary>
+    /// When set to <see langword="true"/> in <see cref="Models.AICompletionContext.AdditionalProperties"/>,
+    /// indicates that documents are available for the current session. This enables
+    /// document processing system tools (e.g., <c>search_documents</c>, <c>list_documents</c>)
+    /// to be included in the tool registry.
+    /// </summary>
+    public const string HasDocuments = "HasDocuments";
+}

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/AIInvocationContext.cs

@@ -0,0 +1,68 @@
+using CrestApps.OrchardCore.AI.Models;
+
+namespace CrestApps.OrchardCore.AI;
+
+/// <summary>
+/// Per-invocation context for AI operations, providing isolation between concurrent
+/// SignalR hub method calls. Each hub invocation creates its own instance via
+/// <see cref="AIInvocationScope.Begin"/>, and AI tools retrieve it via
+/// <see cref="AIInvocationScope.Current"/>.
+///
+/// <para>
+/// <b>Why this exists:</b> In SignalR, <c>HttpContext</c> is shared across all hub
+/// method invocations on the same WebSocket connection. If a user sends multiple
+/// messages concurrently, writing to <c>HttpContext.Items</c> causes data leaks
+/// between invocations. This class uses <see cref="System.Threading.AsyncLocal{T}"/>
+/// (via <see cref="AIInvocationScope"/>) to provide truly invocation-scoped storage
+/// that flows correctly through async/await chains without any cross-invocation
+/// contamination.
+/// </para>
+///
+/// <para>
+/// <b>For AI tools:</b> Tools are registered as singletons and the AI model does not
+/// pass any invocation identifier when calling them. Tools retrieve the current
+/// invocation context by calling <c>AIInvocationScope.Current</c>, which returns the
+/// context for the async execution flow that is calling the tool — even when multiple
+/// invocations are in flight simultaneously on different threads or continuations.
+/// </para>
+/// </summary>
+public sealed class AIInvocationContext
+{
+    private int _referenceIndex;
+
+    /// <summary>
+    /// Gets or sets the <see cref="AIToolExecutionContext"/> for the current invocation,
+    /// providing provider, connection, and resource information.
+    /// </summary>
+    public AIToolExecutionContext ToolExecutionContext { get; set; }
+
+    /// <summary>
+    /// Gets or sets the data source identifier for the current invocation.
+    /// Used by <c>DataSourceSearchTool</c> to scope searches to the correct data source.
+    /// </summary>
+    public string DataSourceId { get; set; }
+
+    /// <summary>
+    /// Gets the dictionary of citation references collected during tool execution
+    /// (e.g., from <c>DataSourceSearchTool</c> and <c>SearchDocumentsTool</c>).
+    /// Keyed by the citation marker (e.g., "[doc:1]") with the reference metadata as value.
+    /// </summary>
+    public Dictionary<string, AICompletionReference> ToolReferences { get; } = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// Gets a general-purpose property bag for extensibility.
+    /// Handlers and tools can store arbitrary per-invocation data here.
+    /// </summary>
+    public Dictionary<string, object> Items { get; } = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// Returns the next unique reference index for citation markers (e.g., [doc:1], [doc:2], ...).
+    /// This method is thread-safe and ensures a monotonically increasing counter across all
+    /// handlers and tools within the same invocation, preventing index collisions between
+    /// <c>DataSourcePreemptiveRagHandler</c>, <c>DocumentPreemptiveRagHandler</c>,
+    /// <c>DataSourceSearchTool</c>, and <c>SearchDocumentsTool</c>.
+    /// </summary>
+    /// <returns>The next 1-based reference index.</returns>
+    public int NextReferenceIndex()
+        => Interlocked.Increment(ref _referenceIndex);
+}

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/AIInvocationScope.cs

@@ -0,0 +1,85 @@
+namespace CrestApps.OrchardCore.AI;
+
+/// <summary>
+/// Manages an <see cref="AsyncLocal{T}"/>-backed ambient context for AI invocations.
+/// Each SignalR hub method call (or any entry point) creates a scope with
+/// <see cref="Begin"/>, and any code running in that async flow can access the
+/// context via <see cref="Current"/>.
+///
+/// <para>
+/// <b>Usage in hubs:</b>
+/// <code>
+/// using var scope = AIInvocationScope.Begin();
+/// // scope.Context is the AIInvocationContext for this invocation.
+/// // All downstream code (handlers, tools) can call AIInvocationScope.Current.
+/// </code>
+/// </para>
+///
+/// <para>
+/// <b>Usage in AI tools (singletons):</b>
+/// <code>
+/// var context = AIInvocationScope.Current;
+/// var dataSourceId = context?.DataSourceId;
+/// </code>
+/// Because <see cref="AsyncLocal{T}"/> flows through async/await continuations,
+/// each concurrent invocation sees its own context — even when multiple invocations
+/// share the same <c>HttpContext</c> in a SignalR WebSocket connection.
+/// </para>
+/// </summary>
+public static class AIInvocationScope
+{
+    private static readonly AsyncLocal<AIInvocationContext> _current = new();
+
+    /// <summary>
+    /// Gets the <see cref="AIInvocationContext"/> for the current async execution flow,
+    /// or <c>null</c> if no scope has been started.
+    /// </summary>
+    public static AIInvocationContext Current => _current.Value;
+
+    /// <summary>
+    /// Begins a new invocation scope with a fresh <see cref="AIInvocationContext"/>.
+    /// The returned <see cref="Scope"/> must be disposed to clear the context.
+    /// </summary>
+    /// <returns>A disposable scope that clears the context on disposal.</returns>
+    public static Scope Begin()
+        => new(new AIInvocationContext());
+
+    /// <summary>
+    /// Begins a new invocation scope with the specified <see cref="AIInvocationContext"/>.
+    /// The returned <see cref="Scope"/> must be disposed to clear the context.
+    /// </summary>
+    /// <param name="context">The invocation context to make current.</param>
+    /// <returns>A disposable scope that clears the context on disposal.</returns>
+    public static Scope Begin(AIInvocationContext context)
+    {
+        ArgumentNullException.ThrowIfNull(context);
+
+        return new Scope(context);
+    }
+
+    /// <summary>
+    /// A disposable wrapper that sets and clears the <see cref="AIInvocationScope.Current"/> context.
+    /// </summary>
+    public readonly struct Scope : IDisposable
+    {
+        /// <summary>
+        /// Gets the <see cref="AIInvocationContext"/> associated with this scope.
+        /// </summary>
+        public AIInvocationContext Context { get; }
+
+        internal Scope(AIInvocationContext context)
+        {
+            Context = context;
+            _current.Value = context;
+        }
+
+        /// <summary>
+        /// Clears the current invocation context, preventing data from leaking
+        /// into subsequent operations on the same thread.
+        /// </summary>
+        public void Dispose()
+        {
+            _current.Value = null;
+        }
+    }
+}

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/AIToolExecutionContext.cs

@@ -3,7 +3,8 @@ namespace CrestApps.OrchardCore.AI;
 /// <summary>
 /// Ambient context that provides AI tool implementations with
 /// the provider, connection, and resource information of the current request.
-/// Stored in <c>HttpContext.Items</c> by the caller (e.g., AIChatHub).
+/// Stored in <see cref="AIInvocationContext.ToolExecutionContext"/> and accessible
+/// via <c>AIInvocationScope.Current?.ToolExecutionContext</c>.
 /// </summary>
 public sealed class AIToolExecutionContext
 {

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/DataSourceSearchResult.cs

@@ -25,6 +25,12 @@ public sealed class DataSourceSearchResult
     /// </summary>
     public int ChunkIndex { get; set; }
 
+    /// <summary>
+    /// Gets or sets the reference type that identifies the kind of source
+    /// (e.g., "Content" for Orchard Core content items, or the source index profile type).
+    /// </summary>
+    public string ReferenceType { get; set; }
+
     /// <summary>
     /// Gets or sets the similarity score.
     /// </summary>

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/DocumentChunkSearchResult.cs

@@ -10,6 +10,16 @@ public sealed class DocumentChunkSearchResult
     /// </summary>
     public Models.ChatInteractionDocumentChunk Chunk { get; set; }
 
+    /// <summary>
+    /// Gets or sets the unique key that identifies the source document.
+    /// </summary>
+    public string DocumentKey { get; set; }
+
+    /// <summary>
+    /// Gets or sets the original file name of the source document.
+    /// </summary>
+    public string FileName { get; set; }
+
     /// <summary>
     /// Gets or sets the similarity score.
     /// </summary>

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/IAILinkGenerator.cs

@@ -0,0 +1,17 @@
+namespace CrestApps.OrchardCore.AI;
+
+/// <summary>
+/// Resolves links for AI completion references based on the reference type.
+/// Implementations are registered as keyed services using the reference type as the key,
+/// allowing different strategies for different source types (e.g., content items, documents, custom).
+/// </summary>
+public interface IAIReferenceLinkResolver
+{
+    /// <summary>
+    /// Resolves a link URL for the given reference.
+    /// </summary>
+    /// <param name="referenceId">The unique identifier of the referenced resource.</param>
+    /// <param name="metadata">Optional metadata associated with the reference.</param>
+    /// <returns>The resolved link URL, or <c>null</c> if no link can be generated.</returns>
+    string ResolveLink(string referenceId, IDictionary<string, object> metadata);
+}

src/Abstractions/CrestApps.OrchardCore.AI.Abstractions/IAIReferenceLinkResolver.cs

@@ -2,11 +2,7 @@
 
 namespace CrestApps.OrchardCore.AI;
 
-/// <summary>
-/// Interface for searching data source embeddings in various index providers.
-/// Implementations should be registered as keyed services using the provider name.
-/// </summary>
-public interface IDataSourceVectorSearchService
+public interface IDataSourceContentManager
 {
     /// <summary>
     /// Searches for document chunks that are similar to the provided embedding vector.
@@ -25,4 +21,16 @@ Task<IEnumerable<DataSourceSearchResult>> SearchAsync(
         int topN,
         string filter = null,
         CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Deletes all document chunks belonging to the specified data source from the index.
+    /// </summary>
+    /// <param name="indexProfile">The index profile to delete from.</param>
+    /// <param name="dataSourceId">The data source ID whose documents should be deleted.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>The number of documents deleted.</returns>
+    Task<long> DeleteByDataSourceIdAsync(
+        IndexProfile indexProfile,
+        string dataSourceId,
+        CancellationToken cancellationToken = default);
 }