Add support for instructions in markdown files with YAML front-matter

kzu · kzu · commit 8da697a76faa · 2025-10-29T02:18:19.000-03:00
If instructions are long and the primary content that defines an agent, using TOML is still an inconvenience. For longer text, markdown is king.

We add support for markdown instructions by just relying on YAML front-matter to provide the section data to go along with it.

Example:

```
---
id: ai.agents.notes
description: Provides free-form memory
client: Grok
options:
  modelid: grok-4-fast
---
You organize and keep notes for the user
```
diff --git a/.netconfig b/.netconfig
@@ -164,3 +164,6 @@
 	sha = c0a15b3c5e42a6f5e73b8e43ad3a335d7d6f3787
 	etag = fe8de7929a8ecdb631911233ae3c6bad034b26b9802e62c3521918207f6d4068
 	weak
+[file "src/Agents/Extensions/YamlConfigurationStreamParser.cs"]
+    url = https://github.com/andrewlock/NetEscapades.Configuration/blob/master/src/NetEscapades.Configuration.Yaml/YamlConfigurationStreamParser.cs
+    weak
diff --git a/readme.md b/readme.md
@@ -133,6 +133,34 @@ This can be used by leveraging [Tomlyn.Extensions.Configuration](https://www.nug
 > avoiding unnecessary tokens being used for indentation while allowing flexible 
 > formatting in the config file.
 
+For longer instructions, markdown format plus YAML front-matter can be used for better readability:
+
+```yaml
+---
+id: ai.agents.notes
+description: Provides free-form memory
+client: grok
+options: 
+  modelid: grok-4-fast
+---
+You organize and keep notes for the user.
+# Some header
+More content
+
+## Another header
+...
+```
+
+Use the provided `AddInstructionsFile` extension method to load instructions from files as follows:
+
+```csharp
+var host = new HostApplicationBuilder(args);
+host.Configuration.AddInstructionsFile("notes.md", optional: false, reloadOnChange: true);
+```
+
+The `id` field in the front-matter is required and specifies the configuration section name, and 
+all other fields are added as if they were specified under it in the configuration.
+
 ### Extensible AI Contexts
 
 The Microsoft [agent framework](https://learn.microsoft.com/en-us/agent-framework/overview/agent-framework-overview) allows extending 
diff --git a/sample/Client/Client.csproj b/sample/Client/Client.csproj
@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="9.10.0" />
     <PackageReference Include="Smith" Version="0.2.5" />
     <PackageReference Include="Spectre.Console" Version="0.52.0" />
     <PackageReference Include="Spectre.Console.Json" Version="0.52.0" />
@@ -22,6 +23,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\src\Agents\Agents.csproj" />
     <ProjectReference Include="..\..\src\Extensions\Extensions.csproj" />
     <ProjectReference Include="..\..\src\Extensions.CodeAnalysis\Extensions.CodeAnalysis.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
   </ItemGroup>
diff --git a/sample/Client/Program.cs b/sample/Client/Program.cs
@@ -1,17 +1,16 @@
 ﻿using System.Net.Http.Json;
-using System.Text.Json.Serialization;
 using Devlooped.Extensions.AI.OpenAI;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Trace;
-using Spectre.Console;
 
 var builder = App.CreateBuilder(args);
 #if DEBUG
 builder.Environment.EnvironmentName = Environments.Development;
 #endif
 
 builder.AddServiceDefaults();
-builder.Services.AddHttpClient();
+builder.Services.AddHttpClient()
+    .ConfigureHttpClientDefaults(b => b.AddStandardResilienceHandler());
 
 var app = builder.Build(async (IServiceProvider services, CancellationToken cancellation) =>
 {
diff --git a/sample/Client/appsettings.json b/sample/Client/appsettings.json
@@ -7,5 +7,12 @@
         "Endpoint": "http://localhost:5117/notes/v1"
       }
     }
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Warning",
+      "System.Net.Http.HttpClient": "Error",
+      "Microsoft.Hosting": "Error"
+    }
   }
 }
diff --git a/sample/Directory.Build.targets b/sample/Directory.Build.targets
@@ -12,7 +12,7 @@
 
   <ItemGroup>
     <None Update=".env" CopyToOutputDirectory="PreserveNewest" CopyToPublishDirectory="Never" />
-    <Content Include="*.json;*.ini;*.toml" Exclude="@(Content)" CopyToOutputDirectory="PreserveNewest" />
+    <Content Include="*.json;*.ini;*.toml;*.md" Exclude="@(Content)" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(IsAspireHost)' != 'true'">
diff --git a/sample/Server/appsettings.Development.json b/sample/Server/appsettings.Development.json
@@ -2,7 +2,9 @@
   "Logging": {
     "LogLevel": {
       "Default": "Information",
-      "Microsoft.AspNetCore": "Warning"
+      "Microsoft.AspNetCore": "Warning",
+      "Microsoft.Hosting": "Warning",
+      "System.Net.Http.HttpClient": "Warning"
     }
   }
 }
diff --git a/sample/Server/appsettings.json b/sample/Server/appsettings.json
@@ -7,22 +7,12 @@
   },
   "AllowedHosts": "*",
   "AI": {
-    "Agents": {
-      "Notes": {
-        "Name": "notes",
-        "Description": "Provides free-form memory",
-        "Instructions": "You organize and keep notes for the user, using JSON-LD",
-        "Client": "Grok",
-        "Options": {
-          "ModelId": "grok-4"
-        }
-      }
-    },
     "Clients": {
       "Grok": {
-        "Endpoint": "https://api.grok.ai/v1",
+        "Endpoint": "https://api.x.ai/v1",
         "ModelId": "grok-4-fast-non-reasoning"
       }
     }
-  }
+  },
+  "OpenTelemetry:ConsoleExporter": true
 }
diff --git a/sample/Server/notes.md b/sample/Server/notes.md
@@ -0,0 +1,8 @@
+---
+id: ai.agents.notes
+description: Provides free-form memory
+client: Grok
+options: 
+  modelid: grok-4-fast
+---
+You organize and keep notes for the user, using JSON-LD
diff --git a/sample/ServiceDefaults.cs b/sample/ServiceDefaults.cs
@@ -1,11 +1,13 @@
-﻿using DotNetEnv.Configuration;
+﻿using Devlooped.Agents.AI;
+using DotNetEnv.Configuration;
 using OpenTelemetry;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
 using Tomlyn.Extensions.Configuration;
 
 
+
 #if WEB
 using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.AspNetCore.Diagnostics.HealthChecks;
@@ -54,7 +56,10 @@ public static TBuilder ConfigureOpenTelemetry<TBuilder>(this TBuilder builder)
                           || httpContext.Request.Path.StartsWithSegments(AlivenessEndpointPath)));
 #endif
                 tracing.AddHttpClientInstrumentation();
-                tracing.AddConsoleExporter();
+
+                // Only add console exporter if explicitly enabled in configuration
+                if (builder.Configuration.GetValue<bool>("OpenTelemetry:ConsoleExporter"))
+                    tracing.AddConsoleExporter();
             })
             .WithMetrics(metrics =>
             {
@@ -83,8 +88,12 @@ public static TBuilder ConfigureReload<TBuilder>(this TBuilder builder)
         {
             foreach (var toml in Directory.EnumerateFiles(AppContext.BaseDirectory, "*.toml", SearchOption.AllDirectories))
                 builder.Configuration.AddTomlFile(toml, optional: false, reloadOnChange: true);
+
             foreach (var json in Directory.EnumerateFiles(AppContext.BaseDirectory, "*.json", SearchOption.AllDirectories))
                 builder.Configuration.AddJsonFile(json, optional: false, reloadOnChange: true);
+
+            foreach (var md in Directory.EnumerateFiles(AppContext.BaseDirectory, "*.md", SearchOption.AllDirectories))
+                builder.Configuration.AddInstructionsFile(md, optional: false, reloadOnChange: true);
         }
         else
         {
@@ -95,11 +104,14 @@ public static TBuilder ConfigureReload<TBuilder>(this TBuilder builder)
             // Only use configs outside of bin/ and obj/ directories since we want reload to happen from source files not output files
             bool IsSource(string path) => !path.StartsWith(outDir) && !path.StartsWith(objDir);
 
+            foreach (var toml in Directory.EnumerateFiles(baseDir, "*.toml", SearchOption.AllDirectories).Where(IsSource))
+                builder.Configuration.AddTomlFile(toml, optional: false, reloadOnChange: true);
+
             foreach (var json in Directory.EnumerateFiles(baseDir, "*.json", SearchOption.AllDirectories).Where(IsSource))
                 builder.Configuration.AddJsonFile(json, optional: false, reloadOnChange: true);
 
-            foreach (var toml in Directory.EnumerateFiles(baseDir, "*.toml", SearchOption.AllDirectories).Where(IsSource))
-                builder.Configuration.AddTomlFile(toml, optional: false, reloadOnChange: true);
+            foreach (var md in Directory.EnumerateFiles(baseDir, "*.md", SearchOption.AllDirectories).Where(IsSource))
+                builder.Configuration.AddInstructionsFile(md, optional: false, reloadOnChange: true);
         }
 
         return builder;
diff --git a/src/Agents/Agents.csproj b/src/Agents/Agents.csproj
@@ -28,14 +28,20 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.10" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.10" />
     <PackageReference Include="PolySharp" Version="1.15.0" PrivateAssets="all" />
+    <PackageReference Include="YamlDotNet" Version="16.3.0" />
   </ItemGroup>
 
   <ItemGroup>
     <None Include="..\..\osmfeula.txt" Link="osmfeula.txt" PackagePath="OSMFEULA.txt" />
+    <None Include="C:\Code\AI\src\Agents\Extensions\.editorconfig" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Extensions\Extensions.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <EditorConfigFiles Remove="C:\Code\AI\src\Agents\Extensions\.editorconfig" />
+  </ItemGroup>
+
 </Project>
diff --git a/src/Agents/ConfigurableAIAgent.cs b/src/Agents/ConfigurableAIAgent.cs
@@ -1,6 +1,5 @@
 ﻿using System.ComponentModel;
 using System.Diagnostics;
-using System.Runtime.CompilerServices;
 using System.Text.Json;
 using Devlooped.Extensions.AI;
 using Devlooped.Extensions.AI.Grok;
diff --git a/src/Agents/ConfigurableInstructionsExtensions.cs b/src/Agents/ConfigurableInstructionsExtensions.cs
@@ -0,0 +1,86 @@
+﻿using System.ComponentModel;
+using System.Text;
+using Microsoft.Extensions.Configuration;
+using NetEscapades.Configuration.Yaml;
+
+namespace Devlooped.Agents.AI;
+
+[EditorBrowsable(EditorBrowsableState.Never)]
+public static class ConfigurableInstructionsExtensions
+{
+    /// <summary>
+    /// Adds an instructions markdown file with optional YAML front-matter to the configuration sources.
+    /// </summary>
+    public static IConfigurationBuilder AddInstructionsFile(this IConfigurationBuilder builder, string path, bool optional = false, bool reloadOnChange = false)
+        => builder.Add<InstructionsConfigurationSource>(source =>
+        {
+            source.Path = path;
+            source.Optional = optional;
+            source.ReloadOnChange = reloadOnChange;
+            source.ResolveFileProvider();
+        });
+
+    /// <summary>
+    /// Adds an instructions markdown stream with optional YAML front-matter to the configuration sources.
+    /// </summary>
+    public static IConfigurationBuilder AddInstructionsStream(this IConfigurationBuilder builder, Stream stream)
+        => Throw.IfNull(builder).Add((InstructionsStreamConfigurationSource source) => source.Stream = stream);
+
+    static class InstructionsParser
+    {
+        public static Dictionary<string, string?> Parse(Stream stream)
+        {
+            using var reader = new StreamReader(stream);
+            var frontMatter = new StringBuilder();
+            var line = reader.ReadLine();
+            // First line must be the front-matter according to spec.
+            if (line == "---")
+            {
+                while ((line = reader.ReadLine()) != "---" && !reader.EndOfStream)
+                    frontMatter.AppendLine(line);
+            }
+
+            if (frontMatter.Length > 0 && line != "---")
+                throw new FormatException("Instructions markdown front-matter is not properly closed with '---'.");
+
+            var instructions = reader.ReadToEnd().Trim();
+            var data = new YamlConfigurationStreamParser().Parse(new MemoryStream(Encoding.UTF8.GetBytes(frontMatter.ToString())));
+            if (!data.TryGetValue("id", out var value) || Convert.ToString(value) is not { Length: > 1 } id)
+                throw new FormatException("Instructions markdown file must contain YAML front-matter with an 'id' key that specifies the section identifier.");
+
+            data.Remove("id");
+            // id should use the config delimiter rather than dot (which is a typical mistake when coming from TOML)
+            id = id.Replace(".", ConfigurationPath.KeyDelimiter);
+            var result = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
+            foreach (var entry in data)
+                result[$"{id}{ConfigurationPath.KeyDelimiter}{entry.Key}"] = entry.Value;
+
+            result[$"{id}{ConfigurationPath.KeyDelimiter}instructions"] = instructions;
+            return result;
+        }
+    }
+
+    class InstructionsStreamConfigurationSource : StreamConfigurationSource
+    {
+        public override IConfigurationProvider Build(IConfigurationBuilder builder) => new InstructionsStreamConfigurationProvider(this);
+    }
+
+    class InstructionsStreamConfigurationProvider(InstructionsStreamConfigurationSource source) : StreamConfigurationProvider(source)
+    {
+        public override void Load(Stream stream) => Data = InstructionsParser.Parse(stream);
+    }
+
+    class InstructionsConfigurationSource : FileConfigurationSource
+    {
+        public override IConfigurationProvider Build(IConfigurationBuilder builder)
+        {
+            EnsureDefaults(builder);
+            return new InstructionsConfigurationProvider(this);
+        }
+    }
+
+    class InstructionsConfigurationProvider(FileConfigurationSource source) : FileConfigurationProvider(source)
+    {
+        public override void Load(Stream stream) => Data = InstructionsParser.Parse(stream);
+    }
+}
diff --git a/src/Agents/Extensions/.editorconfig b/src/Agents/Extensions/.editorconfig
@@ -0,0 +1,2 @@
+[*.cs]
+generated_code = true
diff --git a/src/Agents/Extensions/Resources.cs b/src/Agents/Extensions/Resources.cs
@@ -0,0 +1,16 @@
+﻿// <auto-generated />
+using System.Globalization;
+
+namespace NetEscapades.Configuration.Yaml
+{
+    internal static class Resources
+    {
+        /// <summary>
+        /// A duplicate key '{0}' was found.
+        /// </summary>
+        internal static string FormatError_KeyIsDuplicated(object p0)
+        {
+            return string.Format(CultureInfo.CurrentCulture, "A duplicate key '{0}' was found.", p0);
+        }
+    }
+}
diff --git a/src/Agents/Extensions/YamlConfigurationStreamParser.cs b/src/Agents/Extensions/YamlConfigurationStreamParser.cs
diff --git a/src/Extensions/ConfigurableChatClient.cs b/src/Extensions/ConfigurableChatClient.cs
diff --git a/src/Tests/ConfigurableAgentTests.cs b/src/Tests/ConfigurableAgentTests.cs
diff --git a/src/Tests/Content/ai.md b/src/Tests/Content/ai.md
diff --git a/src/Tests/Misc.cs b/src/Tests/Misc.cs
diff --git a/src/Tests/test.toml b/src/Tests/test.toml

Original file line number	Diff line number	Diff line change
`@@ -7,5 +7,12 @@`
`7`	`7`	`"Endpoint": "http://localhost:5117/notes/v1"`
`8`	`8`	`}`
`9`	`9`	`}`
	`10`	`+ },`
	`11`	`+ "Logging": {`
	`12`	`+ "LogLevel": {`
	`13`	`+ "Default": "Warning",`
	`14`	`+ "System.Net.Http.HttpClient": "Error",`
	`15`	`+ "Microsoft.Hosting": "Error"`
	`16`	`+ }`
`10`	`17`	`}`
`11`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,9 @@`
`2`	`2`	`"Logging": {`
`3`	`3`	`"LogLevel": {`
`4`	`4`	`"Default": "Information",`
`5`		`- "Microsoft.AspNetCore": "Warning"`
	`5`	`+ "Microsoft.AspNetCore": "Warning",`
	`6`	`+ "Microsoft.Hosting": "Warning",`
	`7`	`+ "System.Net.Http.HttpClient": "Warning"`
`6`	`8`	`}`
`7`	`9`	`}`
`8`	`10`	`}`