fix integration tests to account for new AppHost setup

Author: Mpdreamz

aspire/AppHost.cs

@@ -7,6 +7,7 @@
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration;
 using Microsoft.Extensions.Logging;
+using static Elastic.Documentation.Aspire.ResourceNames;
 
 // ReSharper disable UnusedVariable
 // ReSharper disable RedundantAssignment
@@ -28,35 +29,34 @@
 return;
 
 // ReSharper disable once RedundantLambdaParameterType
-async Task BuildAspireHost(bool startElasticsearch, bool cloneAlways, bool skipPrivateRepositories, Cancel ctx)
+async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool skipPrivateRepositories, Cancel ctx)
 {
 	var builder = DistributedApplication.CreateBuilder(args);
+	skipPrivateRepositories = globalArguments.Contains("--skip-private-repositories");
 
 	var llmUrl = builder.AddParameter("LlmGatewayUrl", secret: true);
 	var llmServiceAccountPath = builder.AddParameter("LlmGatewayServiceAccountPath", secret: true);
 
 	var elasticsearchUrl = builder.AddParameter("DocumentationElasticUrl", secret: true);
 	var elasticsearchApiKey = builder.AddParameter("DocumentationElasticApiKey", secret: true);
 
-	var cloneAll = builder.AddProject<Projects.docs_assembler>("assembler-clone");
-	var cloneDirectory = Path.Combine(Paths.GetSolutionDirectory()!.FullName, ".artifacts", "checkouts", "current");
-	cloneAll = cloneAlways || !Directory.Exists(cloneDirectory)
-		? cloneAll.WithArgs(["repo", "clone-all", .. globalArguments])
-		: cloneAll.WithArgs(["repo", "clone-all", "--help"]);
+	var cloneAll = builder.AddProject<Projects.docs_assembler>(AssemblerClone);
+	string[] cloneArgs = assumeCloned ? ["--assume-cloned"] : [];
+	cloneAll = cloneAll.WithArgs(["repo", "clone-all", .. globalArguments, .. cloneArgs]);
 
-	var buildAll = builder.AddProject<Projects.docs_assembler>("assembler-build")
+	var buildAll = builder.AddProject<Projects.docs_assembler>(AssemblerBuild)
 		.WithArgs(["repo", "build-all", .. globalArguments])
 		.WaitForCompletion(cloneAll)
 		.WithParentRelationship(cloneAll);
 
-	var elasticsearchLocal = builder.AddElasticsearch("elasticsearch-local")
+	var elasticsearchLocal = builder.AddElasticsearch(ElasticsearchLocal)
 		.WithEnvironment("LICENSE", "trial");
 	if (!startElasticsearch)
 		elasticsearchLocal = elasticsearchLocal.WithExplicitStart();
 
-	var elasticsearchRemote = builder.AddExternalService("elasticsearch-remote", elasticsearchUrl);
+	var elasticsearchRemote = builder.AddExternalService(ElasticsearchRemote, elasticsearchUrl);
 
-	var api = builder.AddProject<Projects.Elastic_Documentation_Api_Lambda>("api-lambda")
+	var api = builder.AddProject<Projects.Elastic_Documentation_Api_Lambda>(LambdaApi)
 		.WithArgs(globalArguments)
 		.WithEnvironment("ENVIRONMENT", "dev")
 		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
@@ -74,7 +74,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool cloneAlways, bool skipP
 			.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchUrl)
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey);
 
-	var indexElasticsearch = builder.AddProject<Projects.docs_assembler>("elasticsearch-indexer-plain")
+	var indexElasticsearch = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerPlain)
 		.WithArgs(["repo", "build-all", "--exporters", "elasticsearch", .. globalArguments])
 		.WithExplicitStart()
 		.WaitForCompletion(cloneAll);
@@ -91,7 +91,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool cloneAlways, bool skipP
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
 			.WithParentRelationship(elasticsearchRemote);
 
-	var indexElasticsearchSemantic = builder.AddProject<Projects.docs_assembler>("elasticsearch-indexer-semantic")
+	var indexElasticsearchSemantic = builder.AddProject<Projects.docs_assembler>(ElasticsearchIndexerSemantic)
 		.WithArgs(["repo", "build-all", "--exporters", "semantic", .. globalArguments])
 		.WithEnvironment("DOCUMENTATION_ELASTIC_URL", elasticsearchLocal.GetEndpoint("http"))
 		.WithEnvironment(context => context.EnvironmentVariables["DOCUMENTATION_ELASTIC_PASSWORD"] = elasticsearchLocal.Resource.PasswordParameter)
@@ -110,7 +110,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool cloneAlways, bool skipP
 			.WithEnvironment("DOCUMENTATION_ELASTIC_APIKEY", elasticsearchApiKey)
 			.WithParentRelationship(elasticsearchRemote);
 
-	var serveStatic = builder.AddProject<Projects.docs_builder>("assembler-serve")
+	var serveStatic = builder.AddProject<Projects.docs_builder>(AssemblerServe)
 		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
 		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
 		.WithHttpEndpoint(port: 4000, isProxied: false)

aspire/README.md

@@ -11,11 +11,21 @@ We do not use Aspire to generate production deployment scripts since [this is no
 
 ## Run all services locally
 
+You may need to install the Aspire workload first. We also recommend installing the aspire plugin 
+
+* [For Rider](https://plugins.jetbrains.com/plugin/23289--net-aspire)
+
+```bash
+sudo dotnet workload install aspire
+```
+
+Aspire is just another CLI program so can be run like all the other tools
+
 ```bash
 dotnet run --project aspire
 ```
 
-Will automatically:
+This will automatically:
 
 * clone all repositories according to `config/assembler.yml` using `docs-assembler repo clone-all`
 * do a full site build of all repositories using `docs-assembler repo build-all`
@@ -58,7 +68,6 @@ Furthermore, it makes the following indexers available in the Aspire UI
 
 These have to be run manually and can be run multiple times.
 
-
 ## User secrets
 
 We use dotnet user secrets to provide parameters to aspire. These are all optional but needed if you want

aspire/ResourceNames.cs

@@ -0,0 +1,17 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Documentation.Aspire;
+
+public static class ResourceNames
+{
+	public const string AssemblerClone = "assembler-clone";
+	public const string AssemblerBuild = "assembler-build";
+	public const string AssemblerServe = "assembler-serve";
+	public const string ElasticsearchLocal = "elasticsearch-local";
+	public const string ElasticsearchRemote = "elasticsearch-remote";
+	public const string LambdaApi = "lambda-api";
+	public const string ElasticsearchIndexerPlain = "elasticsearch-indexer-plain";
+	public const string ElasticsearchIndexerSemantic = "elasticsearch-indexer-semantic";
+}

aspire/aspire.csproj

@@ -20,12 +20,12 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="$(SolutionRoot)\src\api\Elastic.Documentation.Api.Lambda\Elastic.Documentation.Api.Lambda.csproj" />
-    <ProjectReference Include="$(SolutionRoot)\src\tooling\docs-assembler\docs-assembler.csproj"/>
-    <ProjectReference Include="$(SolutionRoot)\src\tooling\docs-builder\docs-builder.csproj"/>
+    <ProjectReference Include="..\src\api\Elastic.Documentation.Api.Lambda\Elastic.Documentation.Api.Lambda.csproj" />
+    <ProjectReference Include="..\src\tooling\docs-assembler\docs-assembler.csproj"/>
+    <ProjectReference Include="..\src\tooling\docs-builder\docs-builder.csproj"/>
 
-    <ProjectReference Include="$(SolutionRoot)\src\tooling\Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj"  IsAspireProjectResource="false" />
-    <ProjectReference Include="$(SolutionRoot)\src\Elastic.Documentation\Elastic.Documentation.csproj" IsAspireProjectResource="false" />
+    <ProjectReference Include="..\src\tooling\Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj"  IsAspireProjectResource="false" />
+    <ProjectReference Include="..\src\Elastic.Documentation\Elastic.Documentation.csproj" IsAspireProjectResource="false" />
   </ItemGroup>
 
 </Project>

src/Elastic.Documentation/GlobalCommandLine.cs

@@ -21,10 +21,7 @@ public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out
 				i++;
 			}
 			else if (args[i] == "--skip-private-repositories")
-			{
 				skipPrivateRepositories = true;
-				i++;
-			}
 			else
 				newArgs.Add(args[i]);
 		}

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -78,12 +78,14 @@ public async Task<int> CloneConfigurationFolder(string? gitRef = null, Cancel ct
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="environment"> The environment to build</param>
 	/// <param name="fetchLatest"> If true, fetch the latest commit of the branch instead of the link registry entry ref</param>
+	/// <param name="assumeCloned"> If true, assume the repository folder already exists on disk assume it's cloned already, primarily used for testing</param>
 	/// <param name="ctx"></param>
 	[Command("clone-all")]
 	public async Task<int> CloneAll(
 		bool? strict = null,
 		string? environment = null,
 		bool? fetchLatest = null,
+		bool? assumeCloned = null,
 		Cancel ctx = default
 	)
 	{
@@ -97,7 +99,7 @@ public async Task<int> CloneAll(
 		var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, fs, fs, null, null);
 		var cloner = new AssemblerRepositorySourcer(logFactory, assembleContext);
 
-		_ = await cloner.CloneAll(fetchLatest ?? false, ctx);
+		_ = await cloner.CloneAll(fetchLatest ?? false, assumeCloned ?? false, ctx);
 
 		await collector.StopAsync(ctx);
 

src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs

@@ -57,7 +57,7 @@ public CheckoutResult GetAll()
 		};
 	}
 
-	public async Task<CheckoutResult> CloneAll(bool fetchLatest, Cancel ctx = default)
+	public async Task<CheckoutResult> CloneAll(bool fetchLatest, bool assumeCloned, Cancel ctx = default)
 	{
 		_logger.LogInformation("Cloning all repositories for environment {EnvironmentName} using '{ContentSourceStrategy}' content sourcing strategy",
 			PublishEnvironment.Name,
@@ -93,7 +93,9 @@ await Task.Run(() =>
 						}
 						gitRef = entryInfo.GitReference;
 					}
-					checkouts.Add(RepositorySourcer.CloneRef(repo.Value, gitRef, fetchLatest));
+
+					var cloneInformation = RepositorySourcer.CloneRef(repo.Value, gitRef, fetchLatest, assumeCloned: assumeCloned);
+					checkouts.Add(cloneInformation);
 				}, c);
 			}).ConfigureAwait(false);
 		await context.WriteFileSystem.File.WriteAllTextAsync(
@@ -125,13 +127,24 @@ public class RepositorySourcer(ILoggerFactory logFactory, IDirectoryInfo checkou
 	// </summary>
 	// <param name="repository">The repository to clone.</param>
 	// <param name="gitRef">The git reference to check out. Branch, commit or tag</param>
-	public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true)
+	public Checkout CloneRef(Repository repository, string gitRef, bool pull = false, int attempt = 1, bool appendRepositoryName = true, bool assumeCloned = false)
 	{
 		var checkoutFolder =
 			appendRepositoryName
 				? readFileSystem.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, repository.Name))
 				: checkoutDirectory;
 		IGitRepository git = new SingleCommitOptimizedGitRepository(collector, checkoutFolder);
+		if (assumeCloned && checkoutFolder.Exists)
+		{
+			_logger.LogInformation("{RepositoryName}: Assuming {RepositoryName}@{Commit} is already checked out to {CheckoutFolder}", repository.Name, repository.Name, gitRef, checkoutFolder.FullName);
+			return new Checkout
+			{
+				Directory = checkoutFolder,
+				HeadReference = git.GetCurrentCommit(),
+				Repository = repository,
+			};
+		}
+
 		if (attempt > 3)
 		{
 			collector.EmitError("", $"Failed to clone repository {repository.Name}@{gitRef} after 3 attempts");

tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs

@@ -9,6 +9,7 @@
 using InMemLogger;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using static Elastic.Documentation.Aspire.ResourceNames;
 
 [assembly: CaptureConsole, AssemblyFixture(typeof(Elastic.Assembler.IntegrationTests.DocumentationFixture))]
 
@@ -24,22 +25,23 @@ public class DocumentationFixture : IAsyncLifetime
 	public async ValueTask InitializeAsync()
 	{
 		var builder = await DistributedApplicationTestingBuilder.CreateAsync<Projects.aspire>(
-			["--skip-private-repositories"],
+			["--skip-private-repositories", "--assume-cloned"],
 			(options, settings) =>
 			{
 				options.DisableDashboard = true;
 				options.AllowUnsecuredTransport = true;
+				options.EnableResourceLogging = true;
 			}
 		);
 		_ = builder.Services.AddElasticDocumentationLogging(LogLevel.Information);
 		_ = builder.Services.AddLogging(c => c.AddXUnit());
 		_ = builder.Services.AddLogging(c => c.AddInMemory());
-		// TODO expose this as secrets for now not needed integration tests
-		_ = builder.AddParameter("LlmGatewayUrl", "");
-		_ = builder.AddParameter("LlmGatewayServiceAccountPath", "");
 		DistributedApplication = await builder.BuildAsync();
 		InMemoryLogger = DistributedApplication.Services.GetService<InMemoryLogger>()!;
-		await DistributedApplication.StartAsync();
+		_ = DistributedApplication.StartAsync().WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken);
+		_ = await DistributedApplication.ResourceNotifications
+			.WaitForResourceHealthyAsync(AssemblerServe, cancellationToken: TestContext.Current.CancellationToken)
+			.WaitAsync(TimeSpan.FromMinutes(5), TestContext.Current.CancellationToken);
 	}
 
 	/// <inheritdoc />
@@ -56,9 +58,7 @@ public class ServeStaticTests(DocumentationFixture fixture, ITestOutputHelper ou
 	[Fact]
 	public async Task AssertRequestToRootReturnsData()
 	{
-		_ = await fixture.DistributedApplication.ResourceNotifications
-			.WaitForResourceHealthyAsync("DocsBuilderServeStatic", cancellationToken: TestContext.Current.CancellationToken);
-		var client = fixture.DistributedApplication.CreateHttpClient("DocsBuilderServeStatic", "http");
+		var client = fixture.DistributedApplication.CreateHttpClient(AssemblerServe, "http");
 		var root = await client.GetStringAsync("/", TestContext.Current.CancellationToken);
 		_ = root.Should().NotBeNullOrEmpty();
 	}