Skip to content

Commit dadc672

Browse files
MpdreamzMpdreamz
authored
Merge branch 'main' into feature/search-enhancements-again
2 parents d333221 + 6242e2a commit dadc672

File tree

93 files changed

+2380
-889
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+2380
-889
lines changed

‎updatecli/updatecli.d/versions.yml‎ renamed to ‎.github/updatecli/updatecli.d/versions.yml‎

File renamed without changes.

‎updatecli/values.d/scm.yml‎ renamed to ‎.github/updatecli/values.d/scm.yml‎

File renamed without changes.

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -107,17 +107,10 @@ jobs:
107107

108108

109109
integration:
110-
if: false
111-
runs-on: ubuntu-latest
110+
runs-on: docs-builder-latest-16
112111
steps:
113112
- uses: actions/checkout@v4
114113

115-
- name: Free Disk Space
116-
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
117-
with:
118-
tool-cache: false
119-
dotnet: true
120-
121114
- name: Bootstrap Action Workspace
122115
id: bootstrap
123116
uses: ./.github/actions/bootstrap

‎.github/workflows/updatecli.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
3535
- uses: elastic/oblt-actions/updatecli/run@v1
3636
with:
37-
command: apply --config updatecli/updatecli.d/versions.yml --values updatecli/values.d/scm.yml
37+
command: apply --config .github/updatecli/updatecli.d/versions.yml --values .github/updatecli/values.d/scm.yml
3838
env:
3939
GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
4040

‎Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
<PackageVersion Include="Proc" Version="0.9.1" />
5858
<PackageVersion Include="RazorSlices" Version="0.9.4" />
5959
<PackageVersion Include="Samboy063.Tomlet" Version="6.0.0" />
60+
<PackageVersion Include="Sep" Version="0.11.0" />
6061
<PackageVersion Include="Slugify.Core" Version="4.0.1" />
6162
<PackageVersion Include="SoftCircuits.IniFileParser" Version="2.7.0" />
6263
<PackageVersion Include="System.IO.Abstractions" Version="22.0.15" />

‎aspire/AppHost.cs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
6+
using ConsoleAppFramework;
7+
using Elastic.Documentation;
8+
using Elastic.Documentation.Configuration;
9+
using Microsoft.Extensions.Logging;
10+
using static Elastic.Documentation.Aspire.ResourceNames;
11+
12+
// ReSharper disable UnusedVariable
13+
// ReSharper disable RedundantAssignment
14+
// ReSharper disable NotAccessedVariable
15+
16+
var logLevel = LogLevel.Information;
17+
GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories);
18+
var globalArguments = new List<string>();
19+
if (skipPrivateRepositories)
20+
globalArguments.Add("--skip-private-repositories");
21+
22+
if (logLevel != LogLevel.Information)
23+
{
24+
globalArguments.Add("--log-level");
25+
globalArguments.Add(logLevel.ToString());
26+
}
27+
28+
await ConsoleApp.RunAsync(args, BuildAspireHost);
29+
return;
30+
31+
// ReSharper disable once RedundantLambdaParameterType
32+
async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skipPrivateRepositories, Cancel ctx)
33+
{
34+
var builder = DistributedApplication.CreateBuilder(args);
35+
skipPrivateRepositories = globalArguments.Contains("--skip-private-repositories");
36+
37+
var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true);
38+
var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true);
39+
40+
var elasticsearchUrl = builder.AddParameter("DocumentationElasticUrl", secret: true);
41+
var elasticsearchApiKey = builder.AddParameter("DocumentationElasticApiKey", secret: true);
42+
43+
var cloneAll = builder.AddProject<Projects.docs_assembler>(AssemblerClone);
44+
string[] cloneArgs = assumeCloned ? ["--assume-cloned"] : [];
45+
cloneAll = cloneAll.WithArgs(["repo", "clone-all", .. globalArguments, .. cloneArgs]);
46+
47+
var buildAll = builder.AddProject<Projects.docs_assembler>(AssemblerBuild)
48+
.WithArgs(["repo", "build-all", .. globalArguments])
49+
.WaitForCompletion(cloneAll)
50+
.WithParentRelationship(cloneAll);
51+
52+
var elasticsearchLocal = builder.AddElasticsearch(ElasticsearchLocal)
53+
.WithEnvironment("LICENSE", "trial");
54+
if (!startElasticsearch)
55+
elasticsearchLocal = elasticsearchLocal.WithExplicitStart();
56+
57+
var elasticsearchRemote = builder.AddExternalService(ElasticsearchRemote, elasticsearchUrl);
58+
59+
var api = builder.AddProject<Projects.Elastic_Documentation_Api_Lambda>(LambdaApi)
60+
.WithArgs(globalArguments)
61+
.WithEnvironment("ENVIRONMENT", "dev")
62+
.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
63+
.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
64+
.WithExplicitStart();
65+
66+
api = startElasticsearch
67+
? api
68+
.WithReference(elasticsearchLocal)
69+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
70+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
71+
.WithParentRelationship(elasticsearchLocal)
72+
.WaitFor(elasticsearchLocal)
73+
: api.WithReference(elasticsearchRemote)
74+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
75+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
76+
77+
var indexElasticsearch = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerPlain)
78+
.WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments])
79+
.WithExplicitStart()
80+
.WaitForCompletion(cloneAll);
81+
indexElasticsearch = startElasticsearch
82+
? indexElasticsearch
83+
.WaitFor(elasticsearchLocal)
84+
.WithReference(elasticsearchLocal)
85+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
86+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
87+
.WithParentRelationship(elasticsearchLocal)
88+
: indexElasticsearch
89+
.WithReference(elasticsearchRemote)
90+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
91+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
92+
.WithParentRelationship(elasticsearchRemote);
93+
94+
var indexElasticsearchSemantic = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerSemantic)
95+
.WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments])
96+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
97+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
98+
.WithExplicitStart()
99+
.WaitForCompletion(cloneAll);
100+
indexElasticsearchSemantic = startElasticsearch
101+
? indexElasticsearchSemantic
102+
.WaitFor(elasticsearchLocal)
103+
.WithReference(elasticsearchLocal)
104+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
105+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
106+
.WithParentRelationship(elasticsearchLocal)
107+
: indexElasticsearchSemantic
108+
.WithReference(elasticsearchRemote)
109+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
110+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
111+
.WithParentRelationship(elasticsearchRemote);
112+
113+
var serveStatic = builder.AddProject<Projects.docs_builder>(AssemblerServe)
114+
.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
115+
.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
116+
.WithHttpEndpoint(port: 4000, isProxied: false)
117+
.WithArgs(["serve-static", .. globalArguments])
118+
.WithHttpHealthCheck("/", 200)
119+
.WaitForCompletion(buildAll)
120+
.WithParentRelationship(cloneAll);
121+
122+
serveStatic = startElasticsearch
123+
? serveStatic
124+
.WithReference(elasticsearchLocal)
125+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
126+
.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
127+
: serveStatic
128+
.WithReference(elasticsearchRemote)
129+
.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
130+
.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
131+
132+
133+
serveStatic = startElasticsearch ? serveStatic.WaitFor(elasticsearchLocal) : serveStatic.WaitFor(buildAll);
134+
135+
await builder.Build().RunAsync(ctx);
136+
}

‎tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json‎ renamed to ‎aspire/Properties/launchSettings.json‎

File renamed without changes.

‎aspire/README.md‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Aspire for Elastic Documentation
2+
3+
We use [Aspire](https://learn.microsoft.com/en-us/dotnet/aspire/get-started/aspire-overview) for local development purposes to spin up all services in a controlled fashion.
4+
5+
> Aspire provides tools, templates, and packages for building observable, production-ready distributed apps. At the center is the app model—a code-first, single source of truth that defines your app's services, resources, and connections.
6+
>Aspire gives you a unified toolchain: launch and debug your entire app locally with one command, then deploy anywhere—Kubernetes, the cloud, or your own servers—using the same composition.
7+
8+
We do not use Aspire to generate production deployment scripts since [this is not fully baked for AWS and terraform yet](https://github.com/dotnet/aspire/issues/6559)
9+
10+
![service-graph.png](service-graph.png)
11+
12+
## Run all services locally
13+
14+
You may need to install the Aspire workload first. We also recommend installing the aspire plugin
15+
16+
* [For Rider](https://plugins.jetbrains.com/plugin/23289--net-aspire)
17+
18+
```bash
19+
sudo dotnet workload install aspire
20+
```
21+
22+
Aspire is just another CLI program so can be run like all the other tools
23+
24+
```bash
25+
dotnet run --project aspire
26+
```
27+
28+
This will automatically:
29+
30+
* clone all repositories according to `config/assembler.yml` using `docs-assembler repo clone-all`
31+
* do a full site build of all repositories using `docs-assembler repo build-all`
32+
* Serve a copy of the fully assembled documentation using `docs-builder serve-static`.
33+
34+
This should start a management UI over at: https://localhost:17166. This UI exposes all logs, traces, and metrics for each service
35+
36+
![management-ui.png](management-ui.png)
37+
38+
### Run without authorization tokens
39+
40+
If you do not have access to clone to private repositories you can use `--skip-private-repositories`
41+
42+
```bash
43+
dotnet run --project aspire -- --skip-private-repositories
44+
```
45+
46+
This will automagically scrub the private repositories from assembler.yml and navigation.yml.
47+
48+
Our integration tests, for instance, use this to run tests on CI tokenless. When specifying this option locally we automatically inject `docs-builder`'s own docs into the `navigation.yml`. This allows us to test changes to documentation sets and their effect on assembler during PR's
49+
50+
## Elasticsearch Instance
51+
52+
By default, we assume local [dotnet user secrets](#user-secrets) have been set to communicate to an external Elasticsearch instance.
53+
54+
However, you can start a local Elasticsearch instance using
55+
56+
```bash
57+
dotnet run --project aspire -- --start-elasticsearch
58+
```
59+
60+
This will run a local Elasticsearch docker image and expose that to Aspire service discovery instead.
61+
62+
### Elasticsearch indexing
63+
64+
Furthermore, it makes the following indexers available in the Aspire UI
65+
66+
* Plain Elasticsearch, index elasticsearch documents.
67+
* Semantic Elasticsearch, same but with semantic fields.
68+
69+
These have to be run manually and can be run multiple times.
70+
71+
## User secrets
72+
73+
We use dotnet user secrets to provide parameters to aspire. These are all optional but needed if you want
74+
the AI prompts and external Elasticsearch searches to work.
75+
76+
NOTE: using `--start-elasticsearch` the url and random password are automatically wired.
77+
78+
```bash
79+
dotnet user-secrets --project aspire list
80+
```
81+
82+
Should have these secrets
83+
84+
> Parameters:LlmGatewayUrl = https://****
85+
> Parameters:LlmGatewayServiceAccountPath = <PATH_TO_GCP_SERVICE_CREDENTIALS_FILE>
86+
> Parameters:DocumentationElasticUrl = https://*.elastic.cloud:443
87+
> Parameters:DocumentationElasticApiKey = ****
88+
89+
To set them:
90+
91+
```bash
92+
dotnet user-secrets --project aspire set Parameters:DocumentationElasticApiKey <VALUE>
93+
```
94+
95+
Do note `dotnet user-secrets` should only be used on local development machines and not on CI.
96+

‎aspire/ResourceNames.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace Elastic.Documentation.Aspire;
6+
7+
public static class ResourceNames
8+
{
9+
public const string AssemblerClone = "assembler-clone";
10+
public const string AssemblerBuild = "assembler-build";
11+
public const string AssemblerServe = "assembler-serve";
12+
public const string ElasticsearchLocal = "elasticsearch-local";
13+
public const string ElasticsearchRemote = "elasticsearch-remote";
14+
public const string LambdaApi = "lambda-api";
15+
public const string ElasticsearchIndexerPlain = "elasticsearch-indexer-plain";
16+
public const string ElasticsearchIndexerSemantic = "elasticsearch-indexer-semantic";
17+
}

‎tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json‎ renamed to ‎aspire/appsettings.Development.json‎

File renamed without changes.

0 commit comments

Comments
 (0)