Allow specifying custom path locally in assembler.yml

.github/workflows/ci.yml

@@ -98,7 +98,6 @@ jobs:
 
   integration:
     runs-on: docs-builder-latest-16
-    if: false
     steps:
       - uses: actions/checkout@v4
 

aspire/AppHost.cs

@@ -0,0 +1,136 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+
+using ConsoleAppFramework;
+using Elastic.Documentation;
+using Elastic.Documentation.Configuration;
+using Microsoft.Extensions.Logging;
+using static Elastic.Documentation.Aspire.ResourceNames;
+
+// ReSharper disable UnusedVariable
+// ReSharper disable RedundantAssignment
+// ReSharper disable NotAccessedVariable
+
+var logLevel = LogLevel.Information;
+GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
+var globalArguments = new List<string>();
+if (skipPrivateRepositories)
+	globalArguments.Add("--skip-private-repositories");
+
+if (logLevel != LogLevel.Information)
+{
+	globalArguments.Add("--log-level");
+	globalArguments.Add(logLevel.ToString());
+}
+
+await ConsoleApp.RunAsync(args, BuildAspireHost);
+return;
+
+// ReSharper disable once RedundantLambdaParameterType
+async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skipPrivateRepositories, Cancel ctx)
+{
+	var builder = DistributedApplication.CreateBuilder(args);
+	skipPrivateRepositories = globalArguments.Contains("--skip-private-repositories");
+
+	var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true);
+	var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true);
+
+	var elasticsearchUrl = builder.AddParameter("DocumentationElasticUrl", secret: true);
+	var elasticsearchApiKey = builder.AddParameter("DocumentationElasticApiKey", secret: true);
+
+	var cloneAll = builder.AddProject<Projects.docs_assembler>(AssemblerClone);
+	string[] cloneArgs = assumeCloned ? ["--assume-cloned"] : [];
+	cloneAll = cloneAll.WithArgs(["repo", "clone-all", .. globalArguments, .. cloneArgs]);
+
+	var buildAll = builder.AddProject<Projects.docs_assembler>(AssemblerBuild)
+		.WithArgs(["repo", "build-all", .. globalArguments])
+		.WaitForCompletion(cloneAll)
+		.WithParentRelationship(cloneAll);
+
+	var elasticsearchLocal = builder.AddElasticsearch(ElasticsearchLocal)
+		.WithEnvironment("LICENSE", "trial");
+	if (!startElasticsearch)
+		elasticsearchLocal = elasticsearchLocal.WithExplicitStart();
+
+	var elasticsearchRemote = builder.AddExternalService(ElasticsearchRemote, elasticsearchUrl);
+
+	var api = builder.AddProject<Projects.Elastic_Documentation_Api_Lambda>(LambdaApi)
+		.WithArgs(globalArguments)
+		.WithEnvironment("ENVIRONMENT", "dev")
+		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
+		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
+		.WithExplicitStart();
+
+	api = startElasticsearch
+		? api
+			.WithReference(elasticsearchLocal)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
+			.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
+			.WithParentRelationship(elasticsearchLocal)
+			.WaitFor(elasticsearchLocal)
+		: api.WithReference(elasticsearchRemote)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
+
+	var indexElasticsearch = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerPlain)
+		.WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments])
+		.WithExplicitStart()
+		.WaitForCompletion(cloneAll);
+	indexElasticsearch = startElasticsearch
+		? indexElasticsearch
+			.WaitFor(elasticsearchLocal)
+			.WithReference(elasticsearchLocal)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
+			.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
+			.WithParentRelationship(elasticsearchLocal)
+		: indexElasticsearch
+			.WithReference(elasticsearchRemote)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
+			.WithParentRelationship(elasticsearchRemote);
+
+	var indexElasticsearchSemantic = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerSemantic)
+		.WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments])
+		.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
+		.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
+		.WithExplicitStart()
+		.WaitForCompletion(cloneAll);
+	indexElasticsearchSemantic = startElasticsearch
+		? indexElasticsearchSemantic
+			.WaitFor(elasticsearchLocal)
+			.WithReference(elasticsearchLocal)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
+			.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
+			.WithParentRelationship(elasticsearchLocal)
+		: indexElasticsearchSemantic
+			.WithReference(elasticsearchRemote)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
+			.WithParentRelationship(elasticsearchRemote);
+
+	var serveStatic = builder.AddProject<Projects.docs_builder>(AssemblerServe)
+		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
+		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
+		.WithHttpEndpoint(port: 4000, isProxied: false)
+		.WithArgs(["serve-static", .. globalArguments])
+		.WithHttpHealthCheck("/", 200)
+		.WaitForCompletion(buildAll)
+		.WithParentRelationship(cloneAll);
+
+	serveStatic = startElasticsearch
+		? serveStatic
+			.WithReference(elasticsearchLocal)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
+			.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
+		: serveStatic
+			.WithReference(elasticsearchRemote)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
+			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
+
+
+	serveStatic = startElasticsearch ? serveStatic.WaitFor(elasticsearchLocal) : serveStatic.WaitFor(buildAll);
+
+	await builder.Build().RunAsync(ctx);
+}

aspire/README.md

@@ -0,0 +1,96 @@
+# Aspire for Elastic Documentation
+
+We use [Aspire](https://learn.microsoft.com/en-us/dotnet/aspire/get-started/aspire-overview) for local development purposes to spin up all services in a controlled fashion.
+
+> Aspire provides tools, templates, and packages for building observable, production-ready distributed apps. At the center is the app model—a code-first, single source of truth that defines your app's services, resources, and connections.  
+>Aspire gives you a unified toolchain: launch and debug your entire app locally with one command, then deploy anywhere—Kubernetes, the cloud, or your own servers—using the same composition.
+
+We do not use Aspire to generate production deployment scripts since [this is not fully baked for AWS and terraform yet](https://github.com/dotnet/aspire/issues/6559)
+
+![service-graph.png](service-graph.png)
+
+## Run all services locally
+
+You may need to install the Aspire workload first. We also recommend installing the aspire plugin 
+
+* [For Rider](https://plugins.jetbrains.com/plugin/23289--net-aspire)
+
+```bash
+sudo dotnet workload install aspire
+```
+
+Aspire is just another CLI program so can be run like all the other tools
+
+```bash
+dotnet run --project aspire
+```
+
+This will automatically:
+
+* clone all repositories according to `config/assembler.yml` using `docs-assembler repo clone-all`
+* do a full site build of all repositories using `docs-assembler repo build-all`
+* Serve a copy of the fully assembled documentation using `docs-builder serve-static`.
+
+This should start a management UI over at: https://localhost:17166. This UI exposes all logs, traces, and metrics for each service
+
+![management-ui.png](management-ui.png)
+
+### Run without authorization tokens
+
+If you do not have access to clone to private repositories you can use `--skip-private-repositories`
+
+```bash
+dotnet run --project aspire -- --skip-private-repositories
+```
+
+This will automagically scrub the private repositories from assembler.yml and navigation.yml.
+
+Our integration tests, for instance, use this to run tests on CI tokenless. When specifying this option locally we automatically inject `docs-builder`'s own docs into the `navigation.yml`. This allows us to test changes to documentation sets and their effect on assembler during PR's
+
+## Elasticsearch Instance
+
+By default, we assume local [dotnet user secrets](#user-secrets) have been set to communicate to an external Elasticsearch instance.
+
+However, you can start a local Elasticsearch instance using 
+
+```bash
+dotnet run --project aspire -- --start-elasticsearch
+```
+
+This will run a local Elasticsearch docker image and expose that to Aspire service discovery instead.
+
+### Elasticsearch indexing
+
+Furthermore, it makes the following indexers available in the Aspire UI
+
+* Plain Elasticsearch, index elasticsearch documents.
+* Semantic Elasticsearch, same but with semantic fields.
+
+These have to be run manually and can be run multiple times.
+
+## User secrets
+
+We use dotnet user secrets to provide parameters to aspire. These are all optional but needed if you want
+the AI prompts and external Elasticsearch searches to work.
+
+NOTE: using `--start-elasticsearch` the url and random password are automatically wired.
+
+```bash
+dotnet user-secrets --project aspire list
+```
+
+Should have these secrets
+
+> Parameters:LlmGatewayUrl = https://****  
+> Parameters:LlmGatewayServiceAccountPath = <PATH_TO_GCP_SERVICE_CREDENTIALS_FILE>   
+> Parameters:DocumentationElasticUrl = https://*.elastic.cloud:443  
+> Parameters:DocumentationElasticApiKey = ****
+
+To set them:
+
+```bash
+dotnet user-secrets --project aspire set Parameters:DocumentationElasticApiKey <VALUE>
+```
+
+Do note `dotnet user-secrets` should only be used on local development machines and not on CI.
+

aspire/ResourceNames.cs

@@ -0,0 +1,17 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Documentation.Aspire;
+
+public static class ResourceNames
+{
+	public const string AssemblerClone = "assembler-clone";
+	public const string AssemblerBuild = "assembler-build";
+	public const string AssemblerServe = "assembler-serve";
+	public const string ElasticsearchLocal = "elasticsearch-local";
+	public const string ElasticsearchRemote = "elasticsearch-remote";
+	public const string LambdaApi = "lambda-api";
+	public const string ElasticsearchIndexerPlain = "elasticsearch-indexer-plain";
+	public const string ElasticsearchIndexerSemantic = "elasticsearch-indexer-semantic";
+}

aspire/aspire.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <Sdk Name="Aspire.AppHost.Sdk" Version="9.4.0"/>
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <UserSecretsId>72f50f33-6fb9-4d08-bff3-39568fe370b3</UserSecretsId>
+    <IsTestProject>false</IsTestProject>
+    <RootNamespace>Elastic.Documentation.Aspire</RootNamespace>
+    <NoWarn>IDE0350</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="ConsoleAppFramework" />
+    <PackageReference Include="Aspire.Hosting.AppHost"/>
+    <PackageReference Include="Elastic.Aspire.Hosting.Elasticsearch"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\src\api\Elastic.Documentation.Api.Lambda\Elastic.Documentation.Api.Lambda.csproj" />
+    <ProjectReference Include="..\src\tooling\docs-assembler\docs-assembler.csproj"/>
+    <ProjectReference Include="..\src\tooling\docs-builder\docs-builder.csproj"/>
+
+    <ProjectReference Include="..\src\tooling\Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj"  IsAspireProjectResource="false" />
+    <ProjectReference Include="..\src\Elastic.Documentation\Elastic.Documentation.csproj" IsAspireProjectResource="false" />
+  </ItemGroup>
+
+</Project>

docs-builder.sln

@@ -129,7 +129,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests-integration", "tests-
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Assembler.IntegrationTests", "tests-integration\Elastic.Assembler.IntegrationTests\Elastic.Assembler.IntegrationTests.csproj", "{A272D3EC-FAAF-4795-A796-302725382AFF}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Aspire", "tests-integration\Elastic.Documentation.Aspire\Elastic.Documentation.Aspire.csproj", "{4DFECE72-4A1F-4B58-918E-DCD07B585231}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "aspire", "aspire\aspire.csproj", "{4DFECE72-4A1F-4B58-918E-DCD07B585231}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.ServiceDefaults", "src\Elastic.Documentation.ServiceDefaults\Elastic.Documentation.ServiceDefaults.csproj", "{2A83ED35-B631-4F02-8D4C-15611D0DB72C}"
 EndProject
@@ -286,7 +286,6 @@ Global
 		{111E7029-BB29-4039-9B45-04776798A8DD} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{164F55EC-9412-4CD4-81AD-3598B57632A6} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
 		{A272D3EC-FAAF-4795-A796-302725382AFF} = {BCAD38D5-6C83-46E2-8398-4BE463931098}
-		{4DFECE72-4A1F-4B58-918E-DCD07B585231} = {BCAD38D5-6C83-46E2-8398-4BE463931098}
 		{2A83ED35-B631-4F02-8D4C-15611D0DB72C} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{B042CC78-5060-4091-B95A-79C71BA3908A} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{F30B90AD-1A01-4A6F-9699-809FA6875B22} = {B042CC78-5060-4091-B95A-79C71BA3908A}