.NET: Processes support for sub-processes (microsoft#9095)

### Description

#### Enables use of sub-processes by adding a process to another process
as a step.

##### Event bubbling and visibility:
This PR adds the concept of visibility to events within a process. There
are two options, `Internal` which keeps the event within the process
it's fired in, and `Public` which allows the event to bubble out of the
process to parent processes or external systems. Events default to
`Internal`. This allows a parent process to receive events that
originate from within a sub-process.

##### Targets for process' external events:
Processes now expose targets for their external events. When a processes
defines a route for an external event by calling
`processBuilder.OnExternalEvent(...)...`, the target for this event can
now be retrieved by calling `process.GetTargetForExternalEvent(...)`.
This allows a parent process to route a step event to the specified
entry event of the sub-process.

#### Retrieve state of a running or completed process
The `*KernelProcessContext` now exposes a `GetStateAsync` method that
allows the state of the process to be retrieved on a started process.

Closes microsoft#9097 

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

---------

Co-authored-by: Ben Thomas <[email protected]>

Author: alliscode

dotnet/samples/GettingStartedWithProcesses/Step01/Step01_Processes.cs

@@ -38,17 +38,17 @@ public async Task UseSimpleProcessAsync()
 
         // Define the behavior when the process receives an external event
         process
-            .OnExternalEvent(ChatBotEvents.StartProcess)
+            .OnInputEvent(ChatBotEvents.StartProcess)
             .SendEventTo(new ProcessFunctionTargetBuilder(introStep));
 
         // When the intro is complete, notify the userInput step
         introStep
             .OnFunctionResult(nameof(IntroStep.PrintIntroMessage))
             .SendEventTo(new ProcessFunctionTargetBuilder(userInputStep));
 
-        // When the userInput step emits an exit event, send it to the end steprt
+        // When the userInput step emits an exit event, send it to the end step
         userInputStep
-            .OnFunctionResult("GetUserInput")
+            .OnEvent(ChatBotEvents.Exit)
             .StopProcess();
 
         // When the userInput step emits a user input event, send it to the assistantResponse step

dotnet/samples/GettingStartedWithProcesses/Step02/Step02_AccountOpening.cs

@@ -30,7 +30,7 @@ private KernelProcess SetupAccountOpeningProcess<TUserInputStep>() where TUserIn
         var crmRecordStep = process.AddStepFromType<CRMRecordCreationStep>();
         var welcomePacketStep = process.AddStepFromType<WelcomePacketStep>();
 
-        process.OnExternalEvent(AccountOpeningEvents.StartProcess)
+        process.OnInputEvent(AccountOpeningEvents.StartProcess)
             .SendEventTo(new ProcessFunctionTargetBuilder(newCustomerFormStep, CompleteNewCustomerFormStep.Functions.NewAccountWelcome));
 
         // When the welcome message is generated, send message to displayAssistantMessageStep

dotnet/src/Experimental/Process.Abstractions/KernelProcessEvent.cs

@@ -16,4 +16,9 @@ public sealed record KernelProcessEvent
     /// An optional data payload associated with the event.
     /// </summary>
     public object? Data { get; set; }
+
+    /// <summary>
+    /// The visibility of the event. Defaults to <see cref="KernelProcessEventVisibility.Internal"/>.
+    /// </summary>
+    public KernelProcessEventVisibility Visibility { get; set; } = KernelProcessEventVisibility.Internal;
 }

dotnet/src/Experimental/Process.Abstractions/KernelProcessEventVisibility.cs

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.SemanticKernel;
+
+/// <summary>
+/// An enumeration representing the visibility of a <see cref="KernelProcessEvent"/>. This is used to determine
+/// if the event is kept within the process it's emitted in, or exposed to external processes and systems.
+/// </summary>
+public enum KernelProcessEventVisibility
+{
+    /// <summary>
+    /// The event is only visible to steps within the same process.
+    /// </summary>
+    Internal,
+
+    /// <summary>
+    /// The event is visible inside the process as well as outside the process. This is useful
+    /// when the event is intended to be consumed by other processes or external systems.
+    /// </summary>
+    Public
+}

dotnet/src/Experimental/Process.Abstractions/KernelProcessFunctionTarget.cs

@@ -10,14 +10,15 @@ public record KernelProcessFunctionTarget
     /// <summary>
     /// Creates an instance of the <see cref="KernelProcessFunctionTarget"/> class.
     /// </summary>
-    public KernelProcessFunctionTarget(string stepId, string functionName, string? parameterName = null)
+    public KernelProcessFunctionTarget(string stepId, string functionName, string? parameterName = null, string? targetEventId = null)
     {
         Verify.NotNullOrWhiteSpace(stepId);
         Verify.NotNullOrWhiteSpace(functionName);
 
         this.StepId = stepId;
         this.FunctionName = functionName;
         this.ParameterName = parameterName;
+        this.TargetEventId = targetEventId;
     }
 
     /// <summary>
@@ -34,4 +35,9 @@ public KernelProcessFunctionTarget(string stepId, string functionName, string? p
     /// The name of the parameter to target. This may be null if the function has no parameters.
     /// </summary>
     public string? ParameterName { get; init; }
+
+    /// <summary>
+    /// The unique identifier for the event to target. This may be null if the target is not a sub-process.
+    /// </summary>
+    public string? TargetEventId { get; init; }
 }

dotnet/src/Experimental/Process.Core/Internal/EndStep.cs

@@ -34,12 +34,6 @@ internal EndStep()
     {
     }
 
-    internal override string GetScopedEventId(string eventId)
-    {
-        // No event scoping for the end step.
-        return eventId;
-    }
-
     internal override Dictionary<string, KernelFunctionMetadata> GetFunctionMetadataMap()
     {
         // The end step has no functions.

dotnet/src/Experimental/Process.Core/ProcessBuilder.cs

@@ -11,9 +11,19 @@ namespace Microsoft.SemanticKernel;
 /// </summary>
 public sealed class ProcessBuilder : ProcessStepBuilder
 {
-    private readonly List<ProcessStepBuilder> _steps;
-    private readonly List<ProcessStepBuilder> _entrySteps;
-    private readonly Dictionary<string, ProcessStepBuilder> _stepsMap;
+    /// <summary>The collection of steps within this process.</summary>
+    private readonly List<ProcessStepBuilder> _steps = [];
+
+    /// <summary>The collection of entry steps within this process.</summary>
+    private readonly List<ProcessStepBuilder> _entrySteps = [];
+
+    /// <summary>Maps external event Ids to the target entry step for the event.</summary>
+    private readonly Dictionary<string, ProcessFunctionTargetBuilder> _externalEventTargetMap = [];
+
+    /// <summary>
+    /// A boolean indicating if the current process is a step within another process.
+    /// </summary>
+    internal bool HasParentProcess { get; set; }
 
     /// <summary>
     /// Used to resolve the target function and parameter for a given optional function name and parameter name.
@@ -56,18 +66,15 @@ internal override KernelProcessFunctionTarget ResolveFunctionTarget(string? func
     /// <inheritdoc/>
     internal override void LinkTo(string eventId, ProcessStepEdgeBuilder edgeBuilder)
     {
+        Verify.NotNull(edgeBuilder?.Source, nameof(edgeBuilder.Source));
+        Verify.NotNull(edgeBuilder?.Target, nameof(edgeBuilder.Target));
+
         // Keep track of the entry point steps
         this._entrySteps.Add(edgeBuilder.Source);
+        this._externalEventTargetMap[eventId] = edgeBuilder.Target;
         base.LinkTo(eventId, edgeBuilder);
     }
 
-    /// <inheritdoc/>
-    internal override string GetScopedEventId(string eventId)
-    {
-        // The event id is scoped to the process name
-        return $"{this.Name}.{eventId}";
-    }
-
     /// <inheritdoc/>
     internal override Dictionary<string, KernelFunctionMetadata> GetFunctionMetadataMap()
     {
@@ -104,7 +111,6 @@ public ProcessStepBuilder AddStepFromType<TStep>(string? name = null) where TSte
     {
         var stepBuilder = new ProcessStepBuilder<TStep>(name);
         this._steps.Add(stepBuilder);
-        this._stepsMap[stepBuilder.Name] = stepBuilder;
 
         return stepBuilder;
     }
@@ -114,15 +120,10 @@ public ProcessStepBuilder AddStepFromType<TStep>(string? name = null) where TSte
     /// </summary>
     /// <param name="kernelProcess">The process to add as a step.</param>
     /// <returns>An instance of <see cref="ProcessStepBuilder"/></returns>
-    public ProcessStepBuilder AddStepFromProcess(ProcessBuilder kernelProcess)
+    public ProcessBuilder AddStepFromProcess(ProcessBuilder kernelProcess)
     {
-        // TODO: Could this method be converted to an "AddStepFromObject" method takes an
-        // instance of ProcessStepBase and adds it to the process?
-        // This would work for processes.
-        // This would benefit steps because the initial value of state could be captured?
-
+        kernelProcess.HasParentProcess = true;
         this._steps.Add(kernelProcess);
-        this._stepsMap[kernelProcess.Name] = kernelProcess;
         return kernelProcess;
     }
 
@@ -132,11 +133,31 @@ public ProcessStepBuilder AddStepFromProcess(ProcessBuilder kernelProcess)
     /// </summary>
     /// <param name="eventId">The Id of the external event.</param>
     /// <returns>An instance of <see cref="ProcessStepEdgeBuilder"/></returns>
-    public ProcessEdgeBuilder OnExternalEvent(string eventId)
+    public ProcessEdgeBuilder OnInputEvent(string eventId)
     {
         return new ProcessEdgeBuilder(this, eventId);
     }
 
+    /// <summary>
+    /// Retrieves the target for a given external event. The step associated with the target is the process itself (this).
+    /// </summary>
+    /// <param name="eventId">The Id of the event</param>
+    /// <returns>An instance of <see cref="ProcessFunctionTargetBuilder"/></returns>
+    /// <exception cref="KernelException"></exception>
+    public ProcessFunctionTargetBuilder WhereInputEventIs(string eventId)
+    {
+        Verify.NotNullOrWhiteSpace(eventId);
+
+        if (!this._externalEventTargetMap.TryGetValue(eventId, out var target))
+        {
+            throw new KernelException($"The process named '{this.Name}' does not expose an event with Id '{eventId}'.");
+        }
+
+        // Targets for external events on a process should be scoped to the process itself rather than the step inside the process.
+        var processTarget = target with { Step = this, TargetEventId = eventId };
+        return processTarget;
+    }
+
     /// <summary>
     /// Builds the process.
     /// </summary>
@@ -151,7 +172,7 @@ public KernelProcess Build()
         var builtSteps = this._steps.Select(step => step.BuildStep()).ToList();
 
         // Create the process
-        var state = new KernelProcessState(this.Name);
+        var state = new KernelProcessState(this.Name, id: this.HasParentProcess ? this.Id : null);
         var process = new KernelProcess(state, builtSteps, builtEdges);
         return process;
     }
@@ -163,9 +184,6 @@ public KernelProcess Build()
     public ProcessBuilder(string name)
         : base(name)
     {
-        this._steps = [];
-        this._entrySteps = [];
-        this._stepsMap = [];
     }
 
     #endregion

dotnet/src/Experimental/Process.Core/ProcessFunctionTargetBuilder.cs

@@ -5,7 +5,7 @@ namespace Microsoft.SemanticKernel;
 /// <summary>
 /// Provides functionality for incrementally defining a process function target.
 /// </summary>
-public sealed class ProcessFunctionTargetBuilder
+public sealed record ProcessFunctionTargetBuilder
 {
     /// <summary>
     /// Initializes a new instance of the <see cref="ProcessFunctionTargetBuilder"/> class.
@@ -41,7 +41,7 @@ public ProcessFunctionTargetBuilder(ProcessStepBuilder step, string? functionNam
     internal KernelProcessFunctionTarget Build()
     {
         Verify.NotNull(this.Step.Id);
-        return new KernelProcessFunctionTarget(this.Step.Id, this.FunctionName, this.ParameterName);
+        return new KernelProcessFunctionTarget(this.Step.Id, this.FunctionName, this.ParameterName, this.TargetEventId);
     }
 
     /// <summary>
@@ -58,4 +58,9 @@ internal KernelProcessFunctionTarget Build()
     /// The name of the parameter to target. This may be null if the function has no parameters.
     /// </summary>
     public string? ParameterName { get; init; }
+
+    /// <summary>
+    /// The unique identifier for the event to target. This may be null if the target is not a sub-process.
+    /// </summary>
+    public string? TargetEventId { get; init; }
 }

dotnet/src/Experimental/Process.Core/ProcessStepBuilder.cs

@@ -48,6 +48,9 @@ public virtual ProcessStepEdgeBuilder OnFunctionResult(string functionName)
 
     #endregion
 
+    /// <summary>The namespace for events that are scoped to this step.</summary>
+    private readonly string _eventNamespace;
+
     /// <summary>
     /// A mapping of function names to the functions themselves.
     /// </summary>
@@ -143,19 +146,23 @@ internal virtual KernelProcessFunctionTarget ResolveFunctionTarget(string? funct
         );
     }
 
-    /// <summary>
-    /// Given an event Id, returns a scoped event Id that is unique to this instance of the step.
-    /// </summary>
-    /// <param name="eventId">The Id of the event.</param>
-    /// <returns>An Id that represents the provided event Id scoped to this step instance.</returns>
-    internal abstract string GetScopedEventId(string eventId);
-
     /// <summary>
     /// Loads a mapping of function names to the associated functions metadata.
     /// </summary>
     /// <returns>A <see cref="Dictionary{TKey, TValue}"/> where TKey is <see cref="string"/> and TValue is <see cref="KernelFunctionMetadata"/></returns>
     internal abstract Dictionary<string, KernelFunctionMetadata> GetFunctionMetadataMap();
 
+    /// <summary>
+    /// Given an event Id, returns a scoped event Id that is unique to this instance of the step.
+    /// </summary>
+    /// <param name="eventId">The Id of the event.</param>
+    /// <returns>An Id that represents the provided event Id scoped to this step instance.</returns>
+    protected string GetScopedEventId(string eventId)
+    {
+        // Scope the event to this instance of this step by prefixing the event Id with the step's namespace.
+        return $"{this._eventNamespace}.{eventId}";
+    }
+
     /// <summary>
     /// Initializes a new instance of the <see cref="ProcessStepBuilder"/> class.
     /// </summary>
@@ -167,6 +174,7 @@ protected ProcessStepBuilder(string name)
 
         this.FunctionsDict = [];
         this.Id = Guid.NewGuid().ToString("n");
+        this._eventNamespace = $"{this.Name}_{this.Id}";
         this.Edges = new Dictionary<string, List<ProcessStepEdgeBuilder>>(StringComparer.OrdinalIgnoreCase);
     }
 }
@@ -176,16 +184,12 @@ protected ProcessStepBuilder(string name)
 /// </summary>
 public sealed class ProcessStepBuilder<TStep> : ProcessStepBuilder where TStep : KernelProcessStep
 {
-    /// <summary>The namespace for events that are scoped to this step.</summary>
-    private readonly string _eventNamespace;
-
     /// <summary>
     /// Creates a new instance of the <see cref="ProcessStepBuilder"/> class. If a name is not provided, the name will be derived from the type of the step.
     /// </summary>
     public ProcessStepBuilder(string? name = null)
         : base(name ?? typeof(TStep).Name)
     {
-        this._eventNamespace = $"{this.Name}_{this.Id}";
         this.FunctionsDict = this.GetFunctionMetadataMap();
     }
 
@@ -225,13 +229,6 @@ internal override KernelProcessStepInfo BuildStep()
         return builtStep;
     }
 
-    /// <inheritdoc/>
-    internal override string GetScopedEventId(string eventId)
-    {
-        // Scope the event to this instance of this step by prefixing the event Id with the step's namespace.
-        return $"{this._eventNamespace}.{eventId}";
-    }
-
     /// <inheritdoc/>
     internal override Dictionary<string, KernelFunctionMetadata> GetFunctionMetadataMap()
     {

dotnet/src/Experimental/Process.Core/ProcessStepEdgeBuilder.cs

@@ -49,15 +49,15 @@ internal KernelProcessEdge Build()
     /// <summary>
     /// Signals that the output of the source step should be sent to the specified target when the associated event fires.
     /// </summary>
-    /// <param name="outputTarget">The output target.</param>
-    public void SendEventTo(ProcessFunctionTargetBuilder outputTarget)
+    /// <param name="target">The output target.</param>
+    public void SendEventTo(ProcessFunctionTargetBuilder target)
     {
         if (this.Target is not null)
         {
             throw new InvalidOperationException("An output target has already been set.");
         }
 
-        this.Target = outputTarget;
+        this.Target = target;
         this.Source.LinkTo(this.EventId, this);
     }