Skip to content

Commit 6242e2a

Browse files
MpdreamzMpdreamz
authored
Allow specifying custom path locally in assembler.yml (#1756)
* Rename aspire project to aspire * Migrate Aspire project to root directory and refactor AppHost implementation. * Add README file for Aspire documentation setup and usage Includes setup instructions, local services management, Elasticsearch instance configuration, user secrets, and tokenless CI workflows. * fix integration tests to account for new AppHost setup * Add ability to set path for repositories in assembler.yml Only works locally. If you specify `--skip-private-repositories` locally and we detect a solution file we automatically inject docs-builder in to the assembler navigation. This allows our integration tests to tests new features in the assembler. * Re-enable integration tests in CI workflow * Expand README with local `--skip-private-repositories` behavior and remove unused import in integration tests * format * add more details if serve-static does not become healthy * Add Parameters:: on CI as empty strings * Override parameters on CI * Another attempt at empty parameters in test * ensure DocumentationElasticUrl is a valid example url
1 parent 3a95c53 commit 6242e2a

File tree

24 files changed

+480
-151
lines changed

24 files changed

+480
-151
lines changed

‎.github/workflows/ci.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,6 @@ jobs:
9898

9999
integration:
100100
runs-on: docs-builder-latest-16
101-
if: false
102101
steps:
103102
- uses: actions/checkout@v4
104103

‎aspire/AppHost.cs

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
6+
using ConsoleAppFramework;
7+
using Elastic.Documentation;
8+
using Elastic.Documentation.Configuration;
9+
using Microsoft.Extensions.Logging;
10+
using static Elastic.Documentation.Aspire.ResourceNames;
11+
12+
// ReSharper disable UnusedVariable
13+
// ReSharper disable RedundantAssignment
14+
// ReSharper disable NotAccessedVariable
15+
16+
var logLevel = LogLevel.Information;
17+
GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
18+
var globalArguments = new List<string>();
19+
if (skipPrivateRepositories)
20+
globalArguments.Add("--skip-private-repositories");
21+
22+
if (logLevel != LogLevel.Information)
23+
{
24+
globalArguments.Add("--log-level");
25+
globalArguments.Add(logLevel.ToString());
26+
}
27+
28+
await ConsoleApp.RunAsync(args, BuildAspireHost);
29+
return;
30+
31+
// ReSharper disable once RedundantLambdaParameterType
32+
async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skipPrivateRepositories, Cancel ctx)
33+
{
34+
var builder = DistributedApplication.CreateBuilder(args);
35+
skipPrivateRepositories = globalArguments.Contains("--skip-private-repositories");
36+
37+
var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true);
38+
var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true);
39+
40+
var elasticsearchUrl = builder.AddParameter("DocumentationElasticUrl", secret: true);
41+
var elasticsearchApiKey = builder.AddParameter("DocumentationElasticApiKey", secret: true);
42+
43+
var cloneAll = builder.AddProject<Projects.docs_assembler>(AssemblerClone);
44+
string[] cloneArgs = assumeCloned ? ["--assume-cloned"] : [];
45+
cloneAll = cloneAll.WithArgs(["repo", "clone-all", .. globalArguments, .. cloneArgs]);
46+
47+
var buildAll = builder.AddProject<Projects.docs_assembler>(AssemblerBuild)
48+
.WithArgs(["repo", "build-all", .. globalArguments])
49+
.WaitForCompletion(cloneAll)
50+
.WithParentRelationship(cloneAll);
51+
52+
var elasticsearchLocal = builder.AddElasticsearch(ElasticsearchLocal)
53+
.WithEnvironment("LICENSE", "trial");
54+
if (!startElasticsearch)
55+
elasticsearchLocal = elasticsearchLocal.WithExplicitStart();
56+
57+
var elasticsearchRemote = builder.AddExternalService(ElasticsearchRemote, elasticsearchUrl);
58+
59+
var api = builder.AddProject<Projects.Elastic_Documentation_Api_Lambda>(LambdaApi)
60+
.WithArgs(globalArguments)
61+
.WithEnvironment("ENVIRONMENT", "dev")
62+
.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
63+
.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
64+
.WithExplicitStart();
65+
66+
api = startElasticsearch
67+
? api
68+
.WithReference(elasticsearchLocal)
69+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
70+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
71+
.WithParentRelationship(elasticsearchLocal)
72+
.WaitFor(elasticsearchLocal)
73+
: api.WithReference(elasticsearchRemote)
74+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
75+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
76+
77+
var indexElasticsearch = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerPlain)
78+
.WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments])
79+
.WithExplicitStart()
80+
.WaitForCompletion(cloneAll);
81+
indexElasticsearch = startElasticsearch
82+
? indexElasticsearch
83+
.WaitFor(elasticsearchLocal)
84+
.WithReference(elasticsearchLocal)
85+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
86+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
87+
.WithParentRelationship(elasticsearchLocal)
88+
: indexElasticsearch
89+
.WithReference(elasticsearchRemote)
90+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
91+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
92+
.WithParentRelationship(elasticsearchRemote);
93+
94+
var indexElasticsearchSemantic = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerSemantic)
95+
.WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments])
96+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
97+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
98+
.WithExplicitStart()
99+
.WaitForCompletion(cloneAll);
100+
indexElasticsearchSemantic = startElasticsearch
101+
? indexElasticsearchSemantic
102+
.WaitFor(elasticsearchLocal)
103+
.WithReference(elasticsearchLocal)
104+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
105+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
106+
.WithParentRelationship(elasticsearchLocal)
107+
: indexElasticsearchSemantic
108+
.WithReference(elasticsearchRemote)
109+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
110+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
111+
.WithParentRelationship(elasticsearchRemote);
112+
113+
var serveStatic = builder.AddProject<Projects.docs_builder>(AssemblerServe)
114+
.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
115+
.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
116+
.WithHttpEndpoint(port: 4000, isProxied: false)
117+
.WithArgs(["serve-static", .. globalArguments])
118+
.WithHttpHealthCheck("/", 200)
119+
.WaitForCompletion(buildAll)
120+
.WithParentRelationship(cloneAll);
121+
122+
serveStatic = startElasticsearch
123+
? serveStatic
124+
.WithReference(elasticsearchLocal)
125+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
126+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
127+
: serveStatic
128+
.WithReference(elasticsearchRemote)
129+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
130+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
131+
132+
133+
serveStatic = startElasticsearch ? serveStatic.WaitFor(elasticsearchLocal) : serveStatic.WaitFor(buildAll);
134+
135+
await builder.Build().RunAsync(ctx);
136+
}

‎tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json renamed to ‎aspire/Properties/launchSettings.json

File renamed without changes.

‎aspire/README.md

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Aspire for Elastic Documentation
2+
3+
We use [Aspire](https://learn.microsoft.com/en-us/dotnet/aspire/get-started/aspire-overview) for local development purposes to spin up all services in a controlled fashion.
4+
5+
> Aspire provides tools, templates, and packages for building observable, production-ready distributed apps. At the center is the app model—a code-first, single source of truth that defines your app's services, resources, and connections.
6+
>Aspire gives you a unified toolchain: launch and debug your entire app locally with one command, then deploy anywhere—Kubernetes, the cloud, or your own servers—using the same composition.
7+
8+
We do not use Aspire to generate production deployment scripts since [this is not fully baked for AWS and terraform yet](https://github.com/dotnet/aspire/issues/6559)
9+
10+
![service-graph.png](service-graph.png)
11+
12+
## Run all services locally
13+
14+
You may need to install the Aspire workload first. We also recommend installing the aspire plugin
15+
16+
* [For Rider](https://plugins.jetbrains.com/plugin/23289--net-aspire)
17+
18+
```bash
19+
sudo dotnet workload install aspire
20+
```
21+
22+
Aspire is just another CLI program so can be run like all the other tools
23+
24+
```bash
25+
dotnet run --project aspire
26+
```
27+
28+
This will automatically:
29+
30+
* clone all repositories according to `config/assembler.yml` using `docs-assembler repo clone-all`
31+
* do a full site build of all repositories using `docs-assembler repo build-all`
32+
* Serve a copy of the fully assembled documentation using `docs-builder serve-static`.
33+
34+
This should start a management UI over at: https://localhost:17166. This UI exposes all logs, traces, and metrics for each service
35+
36+
![management-ui.png](management-ui.png)
37+
38+
### Run without authorization tokens
39+
40+
If you do not have access to clone to private repositories you can use `--skip-private-repositories`
41+
42+
```bash
43+
dotnet run --project aspire -- --skip-private-repositories
44+
```
45+
46+
This will automagically scrub the private repositories from assembler.yml and navigation.yml.
47+
48+
Our integration tests, for instance, use this to run tests on CI tokenless. When specifying this option locally we automatically inject `docs-builder`'s own docs into the `navigation.yml`. This allows us to test changes to documentation sets and their effect on assembler during PR's
49+
50+
## Elasticsearch Instance
51+
52+
By default, we assume local [dotnet user secrets](#user-secrets) have been set to communicate to an external Elasticsearch instance.
53+
54+
However, you can start a local Elasticsearch instance using
55+
56+
```bash
57+
dotnet run --project aspire -- --start-elasticsearch
58+
```
59+
60+
This will run a local Elasticsearch docker image and expose that to Aspire service discovery instead.
61+
62+
### Elasticsearch indexing
63+
64+
Furthermore, it makes the following indexers available in the Aspire UI
65+
66+
* Plain Elasticsearch, index elasticsearch documents.
67+
* Semantic Elasticsearch, same but with semantic fields.
68+
69+
These have to be run manually and can be run multiple times.
70+
71+
## User secrets
72+
73+
We use dotnet user secrets to provide parameters to aspire. These are all optional but needed if you want
74+
the AI prompts and external Elasticsearch searches to work.
75+
76+
NOTE: using `--start-elasticsearch` the url and random password are automatically wired.
77+
78+
```bash
79+
dotnet user-secrets --project aspire list
80+
```
81+
82+
Should have these secrets
83+
84+
> Parameters:LlmGatewayUrl = https://****
85+
> Parameters:LlmGatewayServiceAccountPath = <PATH_TO_GCP_SERVICE_CREDENTIALS_FILE>
86+
> Parameters:DocumentationElasticUrl = https://*.elastic.cloud:443
87+
> Parameters:DocumentationElasticApiKey = ****
88+
89+
To set them:
90+
91+
```bash
92+
dotnet user-secrets --project aspire set Parameters:DocumentationElasticApiKey <VALUE>
93+
```
94+
95+
Do note `dotnet user-secrets` should only be used on local development machines and not on CI.
96+

‎aspire/ResourceNames.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace Elastic.Documentation.Aspire;
6+
7+
public static class ResourceNames
8+
{
9+
public const string AssemblerClone = "assembler-clone";
10+
public const string AssemblerBuild = "assembler-build";
11+
public const string AssemblerServe = "assembler-serve";
12+
public const string ElasticsearchLocal = "elasticsearch-local";
13+
public const string ElasticsearchRemote = "elasticsearch-remote";
14+
public const string LambdaApi = "lambda-api";
15+
public const string ElasticsearchIndexerPlain = "elasticsearch-indexer-plain";
16+
public const string ElasticsearchIndexerSemantic = "elasticsearch-indexer-semantic";
17+
}

‎tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json renamed to ‎aspire/appsettings.Development.json

File renamed without changes.

‎tests-integration/Elastic.Documentation.Aspire/appsettings.json renamed to ‎aspire/appsettings.json

File renamed without changes.

‎aspire/aspire.csproj

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<Sdk Name="Aspire.AppHost.Sdk" Version="9.4.0"/>
4+
5+
<PropertyGroup>
6+
<OutputType>Exe</OutputType>
7+
<TargetFramework>net9.0</TargetFramework>
8+
<ImplicitUsings>enable</ImplicitUsings>
9+
<Nullable>enable</Nullable>
10+
<UserSecretsId>72f50f33-6fb9-4d08-bff3-39568fe370b3</UserSecretsId>
11+
<IsTestProject>false</IsTestProject>
12+
<RootNamespace>Elastic.Documentation.Aspire</RootNamespace>
13+
<NoWarn>IDE0350</NoWarn>
14+
</PropertyGroup>
15+
16+
<ItemGroup>
17+
<PackageReference Include="ConsoleAppFramework" />
18+
<PackageReference Include="Aspire.Hosting.AppHost"/>
19+
<PackageReference Include="Elastic.Aspire.Hosting.Elasticsearch"/>
20+
</ItemGroup>
21+
22+
<ItemGroup>
23+
<ProjectReference Include="..\src\api\Elastic.Documentation.Api.Lambda\Elastic.Documentation.Api.Lambda.csproj" />
24+
<ProjectReference Include="..\src\tooling\docs-assembler\docs-assembler.csproj"/>
25+
<ProjectReference Include="..\src\tooling\docs-builder\docs-builder.csproj"/>
26+
27+
<ProjectReference Include="..\src\tooling\Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj" IsAspireProjectResource="false" />
28+
<ProjectReference Include="..\src\Elastic.Documentation\Elastic.Documentation.csproj" IsAspireProjectResource="false" />
29+
</ItemGroup>
30+
31+
</Project>

‎aspire/management-ui.png

144 KB
Loading

‎aspire/service-graph.png

137 KB
Loading

0 commit comments

Comments
 (0)