Index OpenAPI endpoints for Elasticsearch and Kibana (#2236)

* Index OpenAPI endpoints

* Enhance OpenAPI exporter with version filtering and update usage in Markdown exporters.

* Ensure api docs are LLM ready and we have stripped markdown version

* clean up html in some descriptions

* Abstract from stripped body

* Tweak API titles

* Ensure type is available on the frontend

* Tweak tests

* fix codeql suggestion

Author: Mpdreamz

Directory.Packages.props

@@ -26,6 +26,7 @@
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.2.3" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.7.14" />
     <PackageVersion Include="Elastic.OpenTelemetry" Version="1.1.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="10.0.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.13.0" />
     <PackageVersion Include="Generator.Equals" Version="3.2.1" PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive" />
@@ -37,7 +38,7 @@
     <PackageVersion Include="InMemoryLogger" Version="1.0.66" />
     <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.6.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0" />
-    <PackageVersion Include="Microsoft.OpenApi" Version="3.0.0" />
+    <PackageVersion Include="Microsoft.OpenApi" Version="3.0.1" />
     <PackageVersion Include="TUnit" Version="0.25.21" />
     <PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
   </ItemGroup>
@@ -105,4 +106,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

aspire/AppHost.cs

@@ -44,8 +44,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 		.WithArgs(globalArguments)
 		.WithEnvironment("ENVIRONMENT", "dev")
 		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
-		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
-		.WithExplicitStart();
+		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath);
 
 	// ReSharper disable once RedundantAssignment
 	api = startElasticsearch
@@ -55,12 +54,14 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 			.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
 			.WithParentRelationship(elasticsearchLocal)
 			.WaitFor(elasticsearchLocal)
+			.WithExplicitStart()
 		: api.WithReference(elasticsearchRemote)
 			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
-			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
+			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
+			.WithExplicitStart();
 
-	var indexElasticsearch = builder.AddProject<Projects.docs_builder>(ElasticsearchIndexerPlain)
-		.WithArgs(["assembler", "build", "--exporters", "elasticsearch", .. globalArguments])
+	var indexElasticsearch = builder.AddProject<Projects.docs_builder>(ElasticsearchIngest)
+		.WithArgs(["assembler", "index", .. globalArguments])
 		.WithExplicitStart()
 		.WaitForCompletion(cloneAll);
 
@@ -78,27 +79,6 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skip
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
 			.WithParentRelationship(elasticsearchRemote);
 
-	var indexElasticsearchSemantic = builder.AddProject<Projects.docs_builder>(ElasticsearchIndexerSemantic)
-		.WithArgs(["assembler", "build", "--exporters", "semantic", .. globalArguments])
-		.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
-		.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
-		.WithExplicitStart()
-		.WaitForCompletion(cloneAll);
-
-	// ReSharper disable once RedundantAssignment
-	indexElasticsearchSemantic = startElasticsearch
-		? indexElasticsearchSemantic
-			.WaitFor(elasticsearchLocal)
-			.WithReference(elasticsearchLocal)
-			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
-			.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
-			.WithParentRelationship(elasticsearchLocal)
-		: indexElasticsearchSemantic
-			.WithReference(elasticsearchRemote)
-			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
-			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
-			.WithParentRelationship(elasticsearchRemote);
-
 	var serveStatic = builder.AddProject<Projects.docs_builder>(AssemblerServe)
 		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
 		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)

aspire/ResourceNames.cs

@@ -12,6 +12,5 @@ public static class ResourceNames
 	public const string ElasticsearchLocal = "elasticsearch-local";
 	public const string ElasticsearchRemote = "elasticsearch-remote";
 	public const string LambdaApi = "lambda-api";
-	public const string ElasticsearchIndexerPlain = "elasticsearch-indexer-plain";
-	public const string ElasticsearchIndexerSemantic = "elasticsearch-indexer-semantic";
+	public const string ElasticsearchIngest = "elasticsearch-ingest";
 }

src/Elastic.ApiExplorer/Elastic.ApiExplorer.csproj

@@ -23,4 +23,8 @@
     <ProjectReference Include="..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj"/>
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="Elasticsearch\" />
+  </ItemGroup>
+
 </Project>