Merge pull request #2840 from Cratis/copilot/support-migration-of-events

Support migration of events between generations (up & down casting)

Author: einari

.vscode/settings.json

@@ -92,6 +92,7 @@
         "Insurtech",
         "Intrinsics",
         "IPII",
+        "jmes",
         "maxcpucount",
         "Meziantou",
         "middlewares",

Directory.Packages.props

@@ -7,7 +7,6 @@
     <!-- System -->
     <PackageVersion Include="protobuf-net.Grpc.AspNetCore.Reflection" Version="1.2.2" />
     <PackageVersion Include="protobuf-net.Grpc.Reflection" Version="1.2.2" />
-    <PackageVersion Include="protobuf-net.Reflection" Version="3.2.52" />
     <PackageVersion Include="System.Reactive" Version="6.1.0" />
     <PackageVersion Include="System.Text.Encoding.Extensions" Version="4.3.0" />
     <PackageVersion Include="System.Text.Json" Version="10.0.5" />
@@ -67,8 +66,6 @@
     <PackageVersion Include="OpenTelemetry.Exporter.InMemory" Version="1.15.0" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.6.0" />
     <!-- Roslyn-->
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="5.3.0" />
-    <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.3.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="5.3.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="5.3.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.3.0" />
@@ -87,6 +84,7 @@
     <PackageVersion Include="humanizer" Version="3.0.10" />
     <PackageVersion Include="Mono.Cecil" Version="0.11.6" />
     <PackageVersion Include="NJsonSchema" Version="11.5.2" />
+    <PackageVersion Include="JsonCons.JmesPath" Version="1.0.0" />
     <PackageVersion Include="protobuf-net.BuildTools" Version="3.2.52">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>

Documentation/concepts/event-type-migrations.md

@@ -0,0 +1,122 @@
+# Event Type Migrations
+
+Event type migrations enable you to evolve your event schemas over time while maintaining
+compatibility with existing events. When an event type changes, you can define **upcasters**
+and **downcasters** that automatically transform events between different generations.
+
+## Why Migrations?
+
+In evolving systems, event schemas naturally change:
+- Properties are added or removed
+- Properties are renamed
+- Complex properties are split or combined
+
+Chronicle's migration system allows you to:
+1. Define declarative transformation rules
+2. Automatically store all generations of an event when appending
+3. Read events in any generation format
+
+## Defining Migrations
+
+To define a migration, implement the `IEventTypeMigrationFor<TEvent>` interface:
+
+```csharp
+public record AuthorRegisteredV1(string Name);
+
+[EventType("author-registered", 2)]
+public record AuthorRegistered(string FirstName, string LastName);
+
+public class AuthorRegisteredMigrator : IEventTypeMigrationFor<AuthorRegistered>
+{
+    public EventTypeGeneration From => 1;
+    public EventTypeGeneration To => 2;
+
+    public void Upcast(IEventMigrationBuilder builder)
+    {
+        builder.Properties(pb =>
+        {
+            pb.Split("Name", " ", 0);   // FirstName from first part of Name
+            pb.Split("Name", " ", 1);   // LastName from second part of Name
+        });
+    }
+
+    public void Downcast(IEventMigrationBuilder builder)
+    {
+        builder.Properties(pb =>
+        {
+            pb.Combine("FirstName", "LastName");  // Combine back to Name
+        });
+    }
+}
+```
+
+## Migration Operations
+
+The migration builder supports the following operations:
+
+### Split
+
+Splits a source property into parts using a separator:
+
+```csharp
+pb.Split("FullName", " ", 0);  // Gets first part
+pb.Split("FullName", " ", 1);  // Gets second part
+```
+
+### Combine
+
+Combines multiple properties into a single value:
+
+```csharp
+pb.Combine("FirstName", "LastName");  // Joins with space separator
+```
+
+### Rename
+
+Renames a property from a previous name:
+
+```csharp
+pb.RenamedFrom("OldPropertyName");
+```
+
+### Default Value
+
+Sets a default value for a new property:
+
+```csharp
+pb.DefaultValue(42);
+pb.DefaultValue("default string");
+```
+
+## How Migrations Work
+
+When an event is appended to the event store:
+
+1. Chronicle identifies the event's current generation
+2. The migration system retrieves all registered migrations for the event type
+3. **Upcasting**: If there are higher generations, the event is transformed upward (1→2→3)
+4. **Downcasting**: If there are lower generations, the event is transformed downward (3→2→1)
+5. All generations are stored in the event sequence
+
+This ensures that:
+- Older consumers can still read events in their expected format
+- Newer consumers can read events with the latest schema
+- No data is lost during schema evolution
+
+## Registration
+
+Migrations are automatically discovered and registered when you connect to Chronicle.
+Simply implement `IEventTypeMigrationFor<TEvent>` in your client application, and
+Chronicle will:
+
+1. Discover all migrators via `IClientArtifactsProvider`
+2. Build migration definitions with JmesPath transformations
+3. Send the definitions to the kernel during event type registration
+
+## Best Practices
+
+1. **Incremental generations**: Always migrate between consecutive generations (1→2, 2→3, not 1→3)
+2. **Reversible transformations**: Ensure downcast can recreate the original structure where possible
+3. **Default values**: Use `DefaultValue()` for new properties that didn't exist in older generations
+4. **Test migrations**: Verify both upcast and downcast transformations work correctly
+5. **Document changes**: Keep track of what changed between generations in your event types

Documentation/concepts/event-type.md

@@ -14,3 +14,4 @@ The concept of generations is important when working with systems that evolve ov
 Each generation registers its own JSON schema, which Chronicle uses to validate and store
 events correctly for that generation.
 
+For detailed information on how to define and use migrations, see [Event Type Migrations](./event-type-migrations.md).

Documentation/concepts/toc.yml

@@ -2,6 +2,8 @@
   href: event.md
 - name: Event Type
   href: event-type.md
+- name: Event Type Migrations
+  href: event-type-migrations.md
 - name: Event Source
   href: event-source.md
 - name: Event Store

Documentation/migrations/dotnet-client.md

@@ -0,0 +1,192 @@
+# C# client usage
+
+This guide covers how to declare event type migrations in a .NET client, the operations available to you, and what happens when your migrators are registered with the Chronicle Kernel.
+
+## Prerequisites
+
+- A Chronicle-enabled .NET application
+- An event type marked with `[EventType]` that has evolved beyond generation 1
+
+## Marking an event type with a generation
+
+Every `[EventType]` that has evolved past its first version must declare its current generation. The generation is part of the event type identity:
+
+```csharp
+// Generation 1 (original) — no explicit generation needed, defaults to 1
+[EventType]
+public record AuthorRegistered(string Name);
+```
+
+When the schema changes, the new record carries the higher generation. The old record can be kept for documentation or removed — Chronicle identifies events by the type name, not the .NET class:
+
+```csharp
+// Generation 2 — Name has been split into FirstName and LastName
+[EventType(2)]
+public record AuthorRegistered(string FirstName, string LastName);
+```
+
+## Defining a migrator
+
+Implement `IEventTypeMigrationFor<TEvent>` where `TEvent` is the **latest generation** of the event type. The interface requires:
+
+- `From` — the generation this migrator reads from
+- `To` — the generation this migrator produces
+- `Upcast(IEventMigrationBuilder)` — transformation from `From` to `To`
+- `Downcast(IEventMigrationBuilder)` — transformation from `To` back to `From`
+
+```csharp
+using Cratis.Chronicle.Events;
+using Cratis.Chronicle.Events.Migrations;
+
+public class AuthorRegisteredMigrator : IEventTypeMigrationFor<AuthorRegistered>
+{
+    public EventTypeGeneration From => 1;
+    public EventTypeGeneration To => 2;
+
+    public void Upcast(IEventMigrationBuilder builder)
+    {
+        builder.Properties(pb =>
+        {
+            var firstName = pb.Split("Name", separator: " ", part: SplitPartIndex.First);
+            var lastName  = pb.Split("Name", separator: " ", part: SplitPartIndex.Second);
+        });
+    }
+
+    public void Downcast(IEventMigrationBuilder builder)
+    {
+        builder.Properties(pb =>
+        {
+            var name = pb.Combine("FirstName", "LastName");
+        });
+    }
+}
+```
+
+Migrators are discovered automatically at startup — no explicit registration is needed.
+
+## Migration operations
+
+All operations return a `PropertyExpression` that identifies the expression in the migration definition. The property name you assign to the returned expression in `Properties()` becomes the output property name in the transformed event.
+
+### Split
+
+Extracts one segment of a string property by splitting it on a separator.
+
+```csharp
+builder.Properties(pb =>
+{
+    var firstName = pb.Split("FullName", separator: " ", part: 0);  // first segment
+    var lastName  = pb.Split("FullName", separator: " ", part: 1);  // second segment
+});
+```
+
+Use `SplitPartIndex.First` and `SplitPartIndex.Second` for the most common cases:
+
+```csharp
+var firstName = pb.Split("FullName", " ", SplitPartIndex.First);
+var lastName  = pb.Split("FullName", " ", SplitPartIndex.Second);
+```
+
+### Combine
+
+Joins multiple source properties into a single string value, separated by a space.
+
+```csharp
+builder.Properties(pb =>
+{
+    var fullName = pb.Combine("FirstName", "LastName");
+});
+```
+
+### RenamedFrom
+
+Reads a property value from an old property name. Use this when a property is being renamed between generations.
+
+```csharp
+builder.Properties(pb =>
+{
+    var email = pb.RenamedFrom("EmailAddress");  // was EmailAddress, now Email
+});
+```
+
+### DefaultValue
+
+Provides a literal default value for a property that did not exist in the source generation.
+
+```csharp
+builder.Properties(pb =>
+{
+    var status = pb.DefaultValue("active");
+    var retries = pb.DefaultValue(0);
+    var enabled = pb.DefaultValue(true);
+});
+```
+
+## Multi-generation migrations
+
+If your event type spans more than two generations, define one migrator per generation pair. Chronicle chains the migrators automatically:
+
+```csharp
+// Generation 1 → 2 migrator
+public class PersonRegisteredV1ToV2 : IEventTypeMigrationFor<PersonRegistered>
+{
+    public EventTypeGeneration From => 1;
+    public EventTypeGeneration To => 2;
+
+    public void Upcast(IEventMigrationBuilder builder) =>
+        builder.Properties(pb =>
+        {
+            var email = pb.RenamedFrom("EmailAddress");
+        });
+
+    public void Downcast(IEventMigrationBuilder builder) =>
+        builder.Properties(pb =>
+        {
+            var emailAddress = pb.RenamedFrom("Email");
+        });
+}
+
+// Generation 2 → 3 migrator
+public class PersonRegisteredV2ToV3 : IEventTypeMigrationFor<PersonRegistered>
+{
+    public EventTypeGeneration From => 2;
+    public EventTypeGeneration To => 3;
+
+    public void Upcast(IEventMigrationBuilder builder) =>
+        builder.Properties(pb =>
+        {
+            var firstName = pb.Split("Name", " ", SplitPartIndex.First);
+            var lastName  = pb.Split("Name", " ", SplitPartIndex.Second);
+        });
+
+    public void Downcast(IEventMigrationBuilder builder) =>
+        builder.Properties(pb =>
+        {
+            var name = pb.Combine("FirstName", "LastName");
+        });
+}
+```
+
+When a generation 1 event arrives, the Kernel chains the upcasts: 1→2, then 2→3, and stores all three generations.
+
+## How registration works
+
+When your application connects to Chronicle, the client:
+
+1. Discovers all `IEventTypeMigrationFor<T>` implementations via `IClientArtifactsProvider`
+2. Invokes `Upcast` and `Downcast` on each migrator to capture the transformation declarations
+3. Converts the declarations into JmesPath expressions
+4. Sends the complete `EventTypeDefinition` — including all generations and their migration definitions — to the Kernel during event type registration
+
+From that point on, the Kernel applies the migrations autonomously on every event append, without any further involvement from the client.
+
+## Validation: missing migrators
+
+If an event type is declared with a generation higher than 1 but has no migrators covering all generations up to the current one, Chronicle throws `MissingEventTypeMigrators` during startup. This prevents silent data loss from an incomplete migration chain.
+
+```text
+Cratis.Chronicle.Events.Migrations.MissingEventTypeMigrators:
+  Event type 'AuthorRegistered' is at generation 3 but no migrators are registered for it.
+```
+
+Ensure every generation gap has a corresponding `IEventTypeMigrationFor<T>` implementation before deploying an event type with a new generation.