diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml index 8a74e471d..169d50c22 100644 --- a/.github/actions/bootstrap/action.yml +++ b/.github/actions/bootstrap/action.yml @@ -18,6 +18,12 @@ runs: run: | git fetch --prune --unshallow --tags git tag --list + + - name: Git config + shell: bash + run: | + git config --global init.defaultBranch main + - uses: actions/setup-dotnet@v4 with: global-json-file: global.json @@ -37,4 +43,3 @@ runs: cache: npm cache-dependency-path: src/Elastic.Documentation.Site/package-lock.json node-version-file: .nvmrc - \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 599f464e2..612b835d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,11 +84,36 @@ jobs: id: bootstrap uses: ./.github/actions/bootstrap + - name: Install Aspire workload + run: dotnet workload install aspire + - name: Build run: dotnet run --project build -c release - name: Test - run: dotnet run --project build -c release -- test + run: dotnet run --project build -c release -- unit-test - name: Publish AOT run: dotnet run --project build -c release -- publishbinaries + + integration: + if: false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Free Disk Space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: false + dotnet: true + + - name: Bootstrap Action Workspace + id: bootstrap + uses: ./.github/actions/bootstrap + + - name: Install Aspire workload + run: dotnet workload install aspire + + - name: Integration Tests + run: dotnet run --project build -c release -- integrate diff --git a/Directory.Packages.props b/Directory.Packages.props index d5894c880..d785e6180 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,6 +3,8 @@ true true + enable + enable @@ -15,14 +17,21 @@ + + + + + + + @@ -40,8 +49,8 @@ - - + + @@ -58,6 +67,17 @@ + + + + + + + + + + + diff --git a/build/CommandLine.fs b/build/CommandLine.fs index d3244adb8..87feb1d96 100644 --- a/build/CommandLine.fs +++ b/build/CommandLine.fs @@ -9,13 +9,22 @@ open Microsoft.FSharp.Reflection open System open Bullseye +type TestSuite = All | Unit | Integration + with + member this.SuitName = + match FSharpValue.GetUnionFields(this, typeof) with + | case, _ -> case.Name.ToLowerInvariant() + type Build = | [] Clean | [] Version | [] Compile | [] Build + | [] Test - + | [] Unit_Test + | [] Integrate + | [] Format | [] Watch @@ -33,6 +42,7 @@ type Build = | [] Single_Target | [] Token of string | [] Skip_Dirty_Check + | [] Test_Suite of TestSuite with interface IArgParserTemplate with member this.Usage = @@ -41,8 +51,11 @@ with | Clean -> "clean known output locations" | Version -> "print version information" | Build -> "Run build" - - | Test -> "runs a clean build and then runs all the tests " + + | Unit_Test -> "alias to providing: test --test-suite=unit" + | Integrate -> "alias to providing: test --test-suite=integration" + | Test -> "runs a clean build and then runs all the tests unless --test-suite is provided" + | Release -> "runs build, tests, and create and validates the packages shy of publishing them" | Publish -> "Publishes artifacts" | Format -> "runs dotnet format" @@ -62,6 +75,7 @@ with | Single_Target -> "Runs the provided sub command without running their dependencies" | Token _ -> "Token to be used to authenticate with github" | Skip_Dirty_Check -> "Skip the clean checkout check that guards the release/publish targets" + | Test_Suite _ -> "Specify the test suite to run, defaults to all" member this.StepName = match FSharpValue.GetUnionFields(this, typeof) with @@ -87,4 +101,4 @@ with let dependsOn = if singleTarget then [] else dependsOn let steps = dependsOn @ composedOf |> List.map _.StepName - Targets.Target(target.StepName, steps, Action(fun _ -> action parsed)) \ No newline at end of file + Targets.Target(target.StepName, steps, Action(fun _ -> action parsed)) diff --git a/build/Targets.fs b/build/Targets.fs index efc5f7852..47baaf75a 100644 --- a/build/Targets.fs +++ b/build/Targets.fs @@ -107,10 +107,17 @@ let private publishContainers _ = createImage "docs-builder" createImage "docs-assembler" -let private runTests _ = +let private runTests (testSuite: TestSuite) _ = + let testFilter = + match testSuite with + | All -> [] + | Unit -> ["--filter"; "FullyQualifiedName~.Tests"] + | Integration -> ["--filter"; "FullyQualifiedName~.IntegrationTests"] + exec { run "dotnet" ( ["test"; "-c"; "release"; "--no-restore"; "--no-build"; "--logger"; "GitHubActions"] + @ testFilter @ ["--"; "RunConfiguration.CollectSourceInformation=true"] ) } @@ -132,8 +139,10 @@ let Setup (parsed:ParseResults) = Build.Cmd [Clean; Lint; Compile] [] build - | Test -> Build.Cmd [Compile] [] runTests - + | Test -> Build.Cmd [Compile] [] <| runTests TestSuite.All + | Unit_Test -> Build.Cmd [Compile] [] <| runTests TestSuite.Unit + | Integrate -> Build.Cmd [Compile] [] <| runTests TestSuite.Integration + | Release -> Build.Cmd [PristineCheck; Build] @@ -159,6 +168,7 @@ let Setup (parsed:ParseResults) = // flags | Single_Target + | Test_Suite _ | Token _ | Skip_Dirty_Check -> Build.Ignore diff --git a/config/assembler.yml b/config/assembler.yml index c56fa05c4..21330a29f 100644 --- a/config/assembler.yml +++ b/config/assembler.yml @@ -94,6 +94,7 @@ references: elastic-otel-python: elastic-serverless-forwarder: integration-docs: + private: true integrations: logstash-docs-md: opentelemetry: @@ -104,7 +105,12 @@ references: # @elastic/admin-docs cloud-on-k8s: - cloud: *master + cloud: + current: master + next: master + edge: master + private: true + curator: *master ecctl: *master diff --git a/docs-builder.sln b/docs-builder.sln index bc53345b7..1f46db51b 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -25,6 +25,9 @@ EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = ".github", ".github\.github.csproj", "{1A8659C1-222A-4824-B562-ED8F88658C05}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{67B576EE-02FA-4F9B-94BC-3630BC09ECE5}" + ProjectSection(SolutionItems) = preProject + tests\Directory.Build.props = tests\Directory.Build.props + EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Markdown.Tests", "tests\Elastic.Markdown.Tests\Elastic.Markdown.Tests.csproj", "{B27C5107-128B-465A-B8F8-8985399E4CFB}" EndProject @@ -119,6 +122,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{6FAB56 config\navigation.yml = config\navigation.yml EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests-integration", "tests-integration", "{BCAD38D5-6C83-46E2-8398-4BE463931098}" + ProjectSection(SolutionItems) = preProject + tests-integration\Directory.Build.props = tests-integration\Directory.Build.props + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Assembler.IntegrationTests", "tests-integration\Elastic.Assembler.IntegrationTests\Elastic.Assembler.IntegrationTests.csproj", "{A272D3EC-FAAF-4795-A796-302725382AFF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Aspire", "tests-integration\Elastic.Documentation.Aspire\Elastic.Documentation.Aspire.csproj", "{4DFECE72-4A1F-4B58-918E-DCD07B585231}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.ServiceDefaults", "src\Elastic.Documentation.ServiceDefaults\Elastic.Documentation.ServiceDefaults.csproj", "{2A83ED35-B631-4F02-8D4C-15611D0DB72C}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "api", "api", "{B042CC78-5060-4091-B95A-79C71BA3908A}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Api.Core", "src\api\Elastic.Documentation.Api.Core\Elastic.Documentation.Api.Core.csproj", "{F30B90AD-1A01-4A6F-9699-809FA6875B22}" @@ -212,6 +226,18 @@ Global {164F55EC-9412-4CD4-81AD-3598B57632A6}.Debug|Any CPU.Build.0 = Debug|Any CPU {164F55EC-9412-4CD4-81AD-3598B57632A6}.Release|Any CPU.ActiveCfg = Release|Any CPU {164F55EC-9412-4CD4-81AD-3598B57632A6}.Release|Any CPU.Build.0 = Release|Any CPU + {A272D3EC-FAAF-4795-A796-302725382AFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A272D3EC-FAAF-4795-A796-302725382AFF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A272D3EC-FAAF-4795-A796-302725382AFF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A272D3EC-FAAF-4795-A796-302725382AFF}.Release|Any CPU.Build.0 = Release|Any CPU + {4DFECE72-4A1F-4B58-918E-DCD07B585231}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4DFECE72-4A1F-4B58-918E-DCD07B585231}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4DFECE72-4A1F-4B58-918E-DCD07B585231}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4DFECE72-4A1F-4B58-918E-DCD07B585231}.Release|Any CPU.Build.0 = Release|Any CPU + {2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A83ED35-B631-4F02-8D4C-15611D0DB72C}.Release|Any CPU.Build.0 = Release|Any CPU {F30B90AD-1A01-4A6F-9699-809FA6875B22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F30B90AD-1A01-4A6F-9699-809FA6875B22}.Debug|Any CPU.Build.0 = Debug|Any CPU {F30B90AD-1A01-4A6F-9699-809FA6875B22}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -254,6 +280,9 @@ Global {89B83007-71E6-4B57-BA78-2544BFA476DB} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {111E7029-BB29-4039-9B45-04776798A8DD} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {164F55EC-9412-4CD4-81AD-3598B57632A6} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5} + {A272D3EC-FAAF-4795-A796-302725382AFF} = {BCAD38D5-6C83-46E2-8398-4BE463931098} + {4DFECE72-4A1F-4B58-918E-DCD07B585231} = {BCAD38D5-6C83-46E2-8398-4BE463931098} + {2A83ED35-B631-4F02-8D4C-15611D0DB72C} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {B042CC78-5060-4091-B95A-79C71BA3908A} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} {F30B90AD-1A01-4A6F-9699-809FA6875B22} = {B042CC78-5060-4091-B95A-79C71BA3908A} {AE3FC78E-167F-4B6E-88EC-84743EB748B7} = {B042CC78-5060-4091-B95A-79C71BA3908A} diff --git a/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs b/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs index 472b7715e..7e37ea96b 100644 --- a/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs +++ b/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs @@ -4,7 +4,6 @@ using System.Text.RegularExpressions; using Elastic.Documentation.Extensions; -using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; using YamlStaticContext = Elastic.Documentation.Configuration.Serialization.YamlStaticContext; @@ -13,9 +12,9 @@ namespace Elastic.Documentation.Configuration.Assembler; public record AssemblyConfiguration { public static AssemblyConfiguration Create(ConfigurationFileProvider provider) => - Deserialize(provider.AssemblerFile.ReadToEnd()); + Deserialize(provider.AssemblerFile.ReadToEnd(), skipPrivateRepositories: provider.SkipPrivateRepositories); - public static AssemblyConfiguration Deserialize(string yaml) + public static AssemblyConfiguration Deserialize(string yaml, bool skipPrivateRepositories = false) { var input = new StringReader(yaml); @@ -28,13 +27,28 @@ public static AssemblyConfiguration Deserialize(string yaml) var config = deserializer.Deserialize(input); foreach (var (name, r) in config.ReferenceRepositories) { + if (name == "cloud") + { + } var repository = RepositoryDefaults(r, name); config.ReferenceRepositories[name] = repository; } + var privateRepositories = config.ReferenceRepositories.Where(r => r.Value.Private).ToList(); + foreach (var (name, _) in privateRepositories) + { + if (skipPrivateRepositories) + _ = config.ReferenceRepositories.Remove(name); + } foreach (var (name, env) in config.Environments) env.Name = name; config.Narrative = RepositoryDefaults(config.Narrative, NarrativeRepository.RepositoryName); + + config.AvailableRepositories = config.ReferenceRepositories.Values + .Where(r => !r.Skip) + .Concat([config.Narrative]).ToDictionary(kvp => kvp.Name, kvp => kvp); + + config.PrivateRepositories = privateRepositories.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); return config; } catch (Exception e) @@ -80,6 +94,15 @@ private static TRepository RepositoryDefaults(TRepository r, string [YamlMember(Alias = "references")] public Dictionary ReferenceRepositories { get; set; } = []; + /// All available repositories, combines and and will filter private repositories if `skip-private-repositories` + /// is specified + [YamlIgnore] + public IReadOnlyDictionary AvailableRepositories { get; private set; } = new Dictionary(); + + /// Repositories marked as private, these are listed under if `--skip-private-repositories` is not specified + [YamlIgnore] + public IReadOnlyDictionary PrivateRepositories { get; private set; } = new Dictionary(); + [YamlMember(Alias = "environments")] public Dictionary Environments { get; set; } = []; diff --git a/src/Elastic.Documentation.Configuration/Assembler/Repository.cs b/src/Elastic.Documentation.Configuration/Assembler/Repository.cs index cabdbf837..389fc257d 100644 --- a/src/Elastic.Documentation.Configuration/Assembler/Repository.cs +++ b/src/Elastic.Documentation.Configuration/Assembler/Repository.cs @@ -48,6 +48,9 @@ public record Repository [YamlMember(Alias = "sparse_paths")] public string[] SparsePaths { get; set; } = ["docs"]; + [YamlMember(Alias = "private")] + public bool Private { get; set; } + public string GetBranch(ContentSource contentSource) => contentSource switch { ContentSource.Current => GitReferenceCurrent, diff --git a/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs b/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs index 31cb19e68..045e6832f 100644 --- a/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs +++ b/src/Elastic.Documentation.Configuration/ConfigurationFileProvider.cs @@ -3,8 +3,11 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using System.Text.RegularExpressions; +using Elastic.Documentation.Configuration.Assembler; using Microsoft.Extensions.DependencyInjection; using NetEscapades.EnumGenerators; +using YamlDotNet.RepresentationModel; namespace Elastic.Documentation.Configuration; @@ -16,7 +19,7 @@ public enum ConfigurationSource Embedded } -public class ConfigurationFileProvider +public partial class ConfigurationFileProvider { private readonly IFileSystem _fileSystem; private readonly string _assemblyName; @@ -24,10 +27,11 @@ public class ConfigurationFileProvider public ConfigurationSource ConfigurationSource { get; private set; } = ConfigurationSource.Embedded; public string? GitReference { get; } - public ConfigurationFileProvider(IFileSystem fileSystem) + public ConfigurationFileProvider(IFileSystem fileSystem, bool skipPrivateRepositories = false) { _fileSystem = fileSystem; _assemblyName = typeof(ConfigurationFileProvider).Assembly.GetName().Name!; + SkipPrivateRepositories = skipPrivateRepositories; TemporaryDirectory = fileSystem.Directory.CreateTempSubdirectory("docs-builder-config"); VersionFile = CreateTemporaryConfigurationFile("versions.yml"); @@ -39,9 +43,11 @@ public ConfigurationFileProvider(IFileSystem fileSystem) GitReference = _fileSystem.File.ReadAllText(path); } + public bool SkipPrivateRepositories { get; } + private IDirectoryInfo TemporaryDirectory { get; } - public IFileInfo NavigationFile { get; } + public IFileInfo NavigationFile { get; private set; } public IFileInfo VersionFile { get; } @@ -49,6 +55,61 @@ public ConfigurationFileProvider(IFileSystem fileSystem) public IFileInfo LegacyUrlMappingsFile { get; } + public IFileInfo CreateNavigationFile(IReadOnlyDictionary privateRepositories) + { + if (privateRepositories.Count == 0 || !SkipPrivateRepositories) + return NavigationFile; + + var targets = string.Join("|", privateRepositories.Keys); + + var tempFile = Path.Combine(TemporaryDirectory.FullName, "navigation.filtered.yml"); + if (_fileSystem.File.Exists(tempFile)) + return NavigationFile; + + // This routine removes `toc: `'s linking to private repositories and reindents any later lines if needed. + // This will make any public children in the nav move up one place. + var spacing = -1; + var reindenting = -1; + foreach (var l in _fileSystem.File.ReadAllLines(NavigationFile.FullName)) + { + var line = l; + if (spacing > -1 && !string.IsNullOrWhiteSpace(line) && !line.StartsWith(new string(' ', spacing), StringComparison.Ordinal)) + { + spacing = -1; + reindenting = -1; + } + + if (spacing != -1 && Regex.IsMatch(line, $@"^\s{{{spacing}}}\S")) + { + spacing = -1; + reindenting = -1; + } + + else if (spacing != -1 && Regex.IsMatch(line, $@"^(\s{{{spacing + 3},}})\S")) + { + var matches = Regex.Match(line, $@"^(?\s{{{spacing}}})(?.+)$"); + line = $"{new string(' ', Math.Max(0, spacing - 4))}{matches.Groups["remainder"].Value}"; + reindenting = spacing; + } + + else if (spacing == -1 && TocPrefixRegex().IsMatch(line)) + { + var matches = Regex.Match(line, $@"^(?\s+)-\s?toc:\s?(?:{targets})\:"); + if (matches.Success) + spacing = matches.Groups["spacing"].Value.Length; + if (spacing == 0) + spacing = -1; + } + + if (spacing == -1 || reindenting > 0) + _fileSystem.File.AppendAllLines(tempFile, [line]); + } + NavigationFile = _fileSystem.FileInfo.New(tempFile); + return NavigationFile; + + + } + private IFileInfo CreateTemporaryConfigurationFile(string fileName) { using var stream = GetLocalOrEmbedded(fileName); @@ -90,14 +151,19 @@ private StreamReader GetEmbeddedStream(string fileName) private static string GetLocalPath(string file) => Path.Combine(LocalConfigurationDirectory, file); private static string GetAppDataPath(string file) => Path.Combine(AppDataConfigurationDirectory, file); + [GeneratedRegex(@"^\s+-?\s?toc:\s?")] + private static partial Regex TocPrefixRegex(); } public static class ConfigurationFileProviderServiceCollectionExtensions { - public static IServiceCollection AddConfigurationFileProvider(this IServiceCollection services, - Action configure) + public static IServiceCollection AddConfigurationFileProvider( + this IServiceCollection services, + bool skipPrivateRepositories, + Action configure + ) { - var provider = new ConfigurationFileProvider(new FileSystem()); + var provider = new ConfigurationFileProvider(new FileSystem(), skipPrivateRepositories); _ = services.AddSingleton(provider); configure(services, provider); return services; diff --git a/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs b/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs new file mode 100644 index 000000000..bcfda4fd1 --- /dev/null +++ b/src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.ServiceDefaults.Logging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; + +namespace Elastic.Documentation.ServiceDefaults; + +public static class AppDefaultsExtensions +{ + public static TBuilder AddDocumentationServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + var args = Array.Empty(); + return builder.AddDocumentationServiceDefaults(ref args); + } + public static TBuilder AddDocumentationServiceDefaults(this TBuilder builder, ref string[] args, Action configure) where TBuilder : IHostApplicationBuilder => + builder.AddDocumentationServiceDefaults(ref args, null, configure); + + public static TBuilder AddDocumentationServiceDefaults(this TBuilder builder, ref string[] args, LogLevel? defaultLogLevel = null, Action? configure = null) where TBuilder : IHostApplicationBuilder + { + var logLevel = defaultLogLevel ?? LogLevel.Information; + GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories); + + var services = builder.Services; + _ = services + .AddConfigurationFileProvider(skipPrivateRepositories, (s, p) => + { + _ = s.AddSingleton(p.CreateVersionConfiguration()); + configure?.Invoke(s, p); + }); + _ = builder.Services.AddElasticDocumentationLogging(logLevel); + + return builder.AddServiceDefaults(); + } + + public static TServiceCollection AddElasticDocumentationLogging(this TServiceCollection services, LogLevel logLevel) + where TServiceCollection : IServiceCollection + { + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + _ = services.AddLogging(x => x + .ClearProviders() + .SetMinimumLevel(logLevel) + .AddConsole(c => c.FormatterName = "condensed") + ); + return services; + } + +} diff --git a/src/Elastic.Documentation.ServiceDefaults/Elastic.Documentation.ServiceDefaults.csproj b/src/Elastic.Documentation.ServiceDefaults/Elastic.Documentation.ServiceDefaults.csproj new file mode 100644 index 000000000..1091489a9 --- /dev/null +++ b/src/Elastic.Documentation.ServiceDefaults/Elastic.Documentation.ServiceDefaults.csproj @@ -0,0 +1,28 @@ + + + + net9.0 + enable + enable + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Elastic.Documentation.ServiceDefaults/Extensions.cs b/src/Elastic.Documentation.ServiceDefaults/Extensions.cs new file mode 100644 index 000000000..4af55c492 --- /dev/null +++ b/src/Elastic.Documentation.ServiceDefaults/Extensions.cs @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace Elastic.Documentation.ServiceDefaults; + +// Adds common .NET Aspire services: service discovery, resilience, health checks, and OpenTelemetry. +// This project should be referenced by each service project in your solution. +// To learn more about using this project, see https://aka.ms/dotnet/aspire/service-defaults +public static class Extensions +{ + private const string HealthEndpointPath = "/health"; + private const string AlivenessEndpointPath = "/alive"; + + public static TBuilder AddServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + _ = builder + .ConfigureOpenTelemetry() + .AddDefaultHealthChecks(); + + _ = builder.Services + .AddServiceDiscovery() + .ConfigureHttpClientDefaults(http => + { + _ = http.AddStandardResilienceHandler(); + _ = http.AddServiceDiscovery(); + }); + return builder; + } + + public static TBuilder ConfigureOpenTelemetry(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + _ = builder.Logging.AddOpenTelemetry(logging => + { + logging.IncludeFormattedMessage = true; + logging.IncludeScopes = true; + }); + + _ = builder.Services.AddOpenTelemetry() + .WithMetrics(metrics => + { + _ = metrics.AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation(); + }) + .WithTracing(tracing => + { + _ = tracing.AddSource(builder.Environment.ApplicationName) + .AddAspNetCoreInstrumentation(tracing => + // Exclude health check requests from tracing + tracing.Filter = context => + !context.Request.Path.StartsWithSegments(HealthEndpointPath) + && !context.Request.Path.StartsWithSegments(AlivenessEndpointPath) + ) + // Uncomment the following line to enable gRPC instrumentation (requires the OpenTelemetry.Instrumentation.GrpcNetClient package) + //.AddGrpcClientInstrumentation() + .AddHttpClientInstrumentation(); + }); + + _ = builder.AddOpenTelemetryExporters(); + + return builder; + } + + private static TBuilder AddOpenTelemetryExporters(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + var useOtlpExporter = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); + + if (useOtlpExporter) + _ = builder.Services.AddOpenTelemetry().UseOtlpExporter(); + + // Uncomment the following lines to enable the Azure Monitor exporter (requires the Azure.Monitor.OpenTelemetry.AspNetCore package) + //if (!string.IsNullOrEmpty(builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"])) + //{ + // builder.Services.AddOpenTelemetry() + // .UseAzureMonitor(); + //} + + return builder; + } + + public static TBuilder AddDefaultHealthChecks(this TBuilder builder) where TBuilder : IHostApplicationBuilder + { + _ = builder.Services.AddHealthChecks() + .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]); + + return builder; + } + + public static WebApplication MapDefaultEndpoints(this WebApplication app) + { + // Adding health checks endpoints to applications in non-development environments has security implications. + // See https://aka.ms/dotnet/aspire/healthchecks for details before enabling these endpoints in non-development environments. + if (app.Environment.IsDevelopment()) + { + // All health checks must pass for app to be considered ready to accept traffic after starting + _ = app.MapHealthChecks(HealthEndpointPath); + + // Only health checks tagged with the "live" tag must pass for app to be considered alive + _ = app.MapHealthChecks(AlivenessEndpointPath, new HealthCheckOptions + { + Predicate = r => r.Tags.Contains("live") + }); + } + + return app; + } +} diff --git a/src/tooling/Elastic.Documentation.Tooling/Logging/CondensedConsoleLogger.cs b/src/Elastic.Documentation.ServiceDefaults/Logging/CondensedConsoleLogger.cs similarity index 97% rename from src/tooling/Elastic.Documentation.Tooling/Logging/CondensedConsoleLogger.cs rename to src/Elastic.Documentation.ServiceDefaults/Logging/CondensedConsoleLogger.cs index 852087708..ef00eabff 100644 --- a/src/tooling/Elastic.Documentation.Tooling/Logging/CondensedConsoleLogger.cs +++ b/src/Elastic.Documentation.ServiceDefaults/Logging/CondensedConsoleLogger.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging.Console; using static Crayon.Output; -namespace Elastic.Documentation.Tooling.Logging; +namespace Elastic.Documentation.ServiceDefaults.Logging; public class CondensedConsoleFormatter() : ConsoleFormatter("condensed") { diff --git a/src/Elastic.Documentation/GlobalCommandLine.cs b/src/Elastic.Documentation/GlobalCommandLine.cs new file mode 100644 index 000000000..9ebedb554 --- /dev/null +++ b/src/Elastic.Documentation/GlobalCommandLine.cs @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Microsoft.Extensions.Logging; + +namespace Elastic.Documentation; + +public static class GlobalCommandLine +{ + public static void Process(ref string[] args, ref LogLevel defaultLogLevel, out bool skipPrivateRepositories) + { + skipPrivateRepositories = false; + var newArgs = new List(); + for (var i = 0; i < args.Length; i++) + { + if (args[i] == "--log-level") + { + if (args.Length > i + 1) + defaultLogLevel = GetLogLevel(args[i + 1]); + i++; + } + else if (args[i] == "--skip-private-repositories") + { + skipPrivateRepositories = true; + i++; + } + else + newArgs.Add(args[i]); + } + + args = [.. newArgs]; + } + + private static LogLevel GetLogLevel(string? logLevel) => logLevel switch + { + "trace" => LogLevel.Trace, + "debug" => LogLevel.Debug, + "information" => LogLevel.Information, + "info" => LogLevel.Information, + "warning" => LogLevel.Warning, + "error" => LogLevel.Error, + "critical" => LogLevel.Critical, + _ => LogLevel.Information + }; + +} diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj b/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj index 009c0da0a..effd23891 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Elastic.Documentation.Api.Infrastructure.csproj @@ -19,10 +19,4 @@ - - - ..\..\..\..\..\..\..\usr\local\share\dotnet\shared\Microsoft.AspNetCore.App\9.0.5\Microsoft.Extensions.Configuration.Abstractions.dll - - - diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs index 3d314dcbc..43862f029 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs @@ -42,9 +42,7 @@ public static class ServicesExtension public static void AddElasticDocsApiUsecases(this IServiceCollection services, string? appEnvironment) { if (AppEnvExtensions.TryParse(appEnvironment, out var parsedEnvironment, true)) - { AddElasticDocsApiUsecases(services, parsedEnvironment); - } else { var logger = GetLogger(services); @@ -107,9 +105,9 @@ private static void AddAskAiUsecase(IServiceCollection services, AppEnv appEnv) var logger = GetLogger(services); logger?.LogInformation("Configuring AskAi use case for environment {AppEnvironment}", appEnv); _ = services.AddSingleton(); - _ = services.AddSingleton, LlmGatewayAskAiGateway>(); _ = services.AddScoped(); _ = services.AddScoped(); + _ = services.AddScoped, LlmGatewayAskAiGateway>(); } private static void AddSearchUsecase(IServiceCollection services, AppEnv appEnv) { diff --git a/src/api/Elastic.Documentation.Api.Lambda/Elastic.Documentation.Api.Lambda.csproj b/src/api/Elastic.Documentation.Api.Lambda/Elastic.Documentation.Api.Lambda.csproj index 19d4574f6..3c9a5b990 100644 --- a/src/api/Elastic.Documentation.Api.Lambda/Elastic.Documentation.Api.Lambda.csproj +++ b/src/api/Elastic.Documentation.Api.Lambda/Elastic.Documentation.Api.Lambda.csproj @@ -1,32 +1,33 @@ - - Exe - net9.0 - enable - enable - true + + Exe + net9.0 + enable + enable + true - bootstrap - Lambda + bootstrap + Lambda - true - true - true - true - false - Linux + true + true + true + true + false + Linux - Elastic.Documentation.Api.Lambda - + Elastic.Documentation.Api.Lambda + - - - - + + + + + - - - + + + diff --git a/src/api/Elastic.Documentation.Api.Lambda/Program.cs b/src/api/Elastic.Documentation.Api.Lambda/Program.cs index 46b94b063..425438fb7 100644 --- a/src/api/Elastic.Documentation.Api.Lambda/Program.cs +++ b/src/api/Elastic.Documentation.Api.Lambda/Program.cs @@ -8,11 +8,15 @@ using Elastic.Documentation.Api.Core.AskAi; using Elastic.Documentation.Api.Core.Search; using Elastic.Documentation.Api.Infrastructure; +using Elastic.Documentation.ServiceDefaults; var builder = WebApplication.CreateSlimBuilder(args); +builder.AddDocumentationServiceDefaults(ref args); + builder.Services.AddAWSLambdaHosting(LambdaEventSource.RestApi, new SourceGeneratorLambdaJsonSerializer()); builder.Services.AddElasticDocsApiUsecases(Environment.GetEnvironmentVariable("ENVIRONMENT")); +builder.WebHost.UseKestrelHttpsConfiguration(); var app = builder.Build(); diff --git a/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs b/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs index a85980927..341db9003 100644 --- a/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs +++ b/src/tooling/Elastic.Documentation.Tooling/DocumentationTooling.cs @@ -7,9 +7,10 @@ using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Configuration.Versions; using Elastic.Documentation.Diagnostics; -using Elastic.Documentation.Tooling.Logging; +using Elastic.Documentation.ServiceDefaults.Logging; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Console; @@ -17,69 +18,12 @@ namespace Elastic.Documentation.Tooling; public static class DocumentationTooling { - public static ServiceProvider CreateServiceProvider(ref string[] args, Action? configure = null) + public static TBuilder AddDocumentationToolingDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder { - var defaultLogLevel = LogLevel.Information; - ProcessCommandLineArguments(ref args, ref defaultLogLevel); - - var services = new ServiceCollection(); - CreateServiceCollection(services, defaultLogLevel, configure); - return services.BuildServiceProvider(); - } - - public static void CreateServiceCollection( - IServiceCollection services, - LogLevel defaultLogLevel, - Action? configure = null - ) - { - _ = services + _ = builder.Services .AddGitHubActionsCore() .AddSingleton() - .AddSingleton() - .AddSingleton() - .AddConfigurationFileProvider((s, p) => - { - _ = s.AddSingleton(p.CreateVersionConfiguration()); - configure?.Invoke(s, p); - }); - services.TryAddEnumerable(ServiceDescriptor.Singleton()); - _ = services.AddLogging(x => x - .ClearProviders() - .SetMinimumLevel(defaultLogLevel) - .AddConsole(c => c.FormatterName = "condensed") - ); - - } - - private static void ProcessCommandLineArguments(ref string[] args, ref LogLevel defaultLogLevel) - { - var newArgs = new List(); - for (var i = 0; i < args.Length; i++) - { - if (args[i] == "--log-level") - { - if (args.Length > i + 1) - defaultLogLevel = GetLogLevel(args[i + 1]); - - i++; - } - else - newArgs.Add(args[i]); - } - - args = [.. newArgs]; + .AddSingleton(); + return builder; } - - private static LogLevel GetLogLevel(string? logLevel) => logLevel switch - { - "trace" => LogLevel.Trace, - "debug" => LogLevel.Debug, - "information" => LogLevel.Information, - "info" => LogLevel.Information, - "warning" => LogLevel.Warning, - "error" => LogLevel.Error, - "critical" => LogLevel.Critical, - _ => LogLevel.Information - }; } diff --git a/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj b/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj index 772393998..9f9e2d11f 100644 --- a/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj +++ b/src/tooling/Elastic.Documentation.Tooling/Elastic.Documentation.Tooling.csproj @@ -10,13 +10,12 @@ - - + diff --git a/src/tooling/docs-assembler/AssembleContext.cs b/src/tooling/docs-assembler/AssembleContext.cs index 2440b80a6..5cbf78901 100644 --- a/src/tooling/docs-assembler/AssembleContext.cs +++ b/src/tooling/docs-assembler/AssembleContext.cs @@ -31,6 +31,8 @@ public class AssembleContext /// This property is used to determine if the site should be indexed by search engines public bool AllowIndexing { get; init; } + public bool IgnorePrivateRepositories { get; init; } + public PublishEnvironment Environment { get; } public AssembleContext( diff --git a/src/tooling/docs-assembler/AssembleSources.cs b/src/tooling/docs-assembler/AssembleSources.cs index f0aa51606..63d926a79 100644 --- a/src/tooling/docs-assembler/AssembleSources.cs +++ b/src/tooling/docs-assembler/AssembleSources.cs @@ -16,7 +16,6 @@ using Elastic.Markdown.IO.Navigation; using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using YamlDotNet.RepresentationModel; namespace Documentation.Assembler; @@ -39,6 +38,7 @@ public record TocConfigurationMapping public class AssembleSources { public AssembleContext AssembleContext { get; } + public FrozenDictionary AssembleSets { get; } public FrozenDictionary NavigationTocMappings { get; } @@ -154,7 +154,8 @@ static void ReadHistoryMappings(IDictionary> public static FrozenDictionary GetTocMappings(AssembleContext context) { var dictionary = new Dictionary(); - var reader = new YamlStreamReader(context.ConfigurationFileProvider.NavigationFile, context.Collector); + var file = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories); + var reader = new YamlStreamReader(file, context.Collector); var entries = new List>(); foreach (var entry in reader.Read()) { diff --git a/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs index c6d9f74ca..5d1f091f8 100644 --- a/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs +++ b/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs @@ -19,7 +19,11 @@ public override async Task Fetch(Cancel ctx) var linkReferences = new Dictionary(); var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); - var repositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]); + // We do want to always fetch cross-link data for all repositories. + // This is public information + var repositories = configuration.AvailableRepositories.Values + .Concat(configuration.PrivateRepositories.Values) + .ToList(); foreach (var repository in repositories) { diff --git a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs index 9ff91f494..645bdb3e9 100644 --- a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs +++ b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs @@ -41,10 +41,10 @@ public async Task Validate(Cancel ctx = default) ILinkIndexReader linkIndexReader = Aws3LinkIndexReader.CreateAnonymous(); var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader); var links = await fetcher.FetchLinkIndex(ctx); - var repositories = context.Configuration.ReferenceRepositories.Values.Concat([context.Configuration.Narrative]).ToList(); + var repositories = context.Configuration.AvailableRepositories; var reportPath = context.ConfigurationFileProvider.AssemblerFile; - foreach (var repository in repositories) + foreach (var repository in repositories.Values) { if (!links.Repositories.TryGetValue(repository.Name, out var registryMapping)) { diff --git a/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs b/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs index 36cd94fce..866527ba5 100644 --- a/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs +++ b/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs @@ -43,10 +43,7 @@ public NavigationPrefixChecker(ILoggerFactory logFactory, AssembleContext contex { _phantoms = GlobalNavigationFile.GetPhantomPrefixes(context); - _repositories = context.Configuration.ReferenceRepositories - .Values - .Concat([context.Configuration.Narrative]) - .Where(p => !p.Skip) + _repositories = context.Configuration.AvailableRepositories.Values .Select(r => r.Name) .ToImmutableHashSet(); diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs index ffe72d4b2..472650402 100644 --- a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -27,7 +27,7 @@ public GlobalNavigationFile(AssembleContext context, AssembleSources assembleSou { _context = context; _assembleSources = assembleSources; - NavigationFile = context.ConfigurationFileProvider.NavigationFile; + NavigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories); TableOfContents = Deserialize("toc"); Phantoms = Deserialize("phantoms"); ScopeDirectory = NavigationFile.Directory!; @@ -62,7 +62,8 @@ public static ImmutableHashSet GetPhantomPrefixes(AssembleContext context) private static ImmutableHashSet GetSourceUris(string key, AssembleContext context) { - var reader = new YamlStreamReader(context.ConfigurationFileProvider.NavigationFile, context.Collector); + var navigationFile = context.ConfigurationFileProvider.CreateNavigationFile(context.Configuration.PrivateRepositories); + var reader = new YamlStreamReader(navigationFile, context.Collector); var set = new HashSet(); foreach (var entry in reader.Read()) { @@ -149,10 +150,10 @@ public void EmitWarning(string message) => public void EmitError(string message) => _context.Collector.EmitWarning(NavigationFile, message); - private IReadOnlyCollection Deserialize(string key) { - var reader = new YamlStreamReader(NavigationFile, _context.Collector); + var navigationFile = _context.ConfigurationFileProvider.CreateNavigationFile(_context.Configuration.PrivateRepositories); + var reader = new YamlStreamReader(navigationFile, _context.Collector); try { foreach (var entry in reader.Read()) diff --git a/src/tooling/docs-assembler/Program.cs b/src/tooling/docs-assembler/Program.cs index bca2c65a7..24a8b9527 100644 --- a/src/tooling/docs-assembler/Program.cs +++ b/src/tooling/docs-assembler/Program.cs @@ -7,19 +7,22 @@ using ConsoleAppFramework; using Documentation.Assembler.Cli; using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.ServiceDefaults; using Elastic.Documentation.Tooling; using Elastic.Documentation.Tooling.Filters; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -await using var serviceProvider = DocumentationTooling.CreateServiceProvider(ref args, (s, p) => -{ - _ = s.AddSingleton(AssemblyConfiguration.Create(p)); -}); +var builder = Host.CreateApplicationBuilder() + .AddDocumentationServiceDefaults(ref args, (s, p) => + { + _ = s.AddSingleton(AssemblyConfiguration.Create(p)); + }) + .AddDocumentationToolingDefaults(); -ConsoleApp.ServiceProvider = serviceProvider; +var app = builder.ToConsoleAppBuilder(); -var app = ConsoleApp.Create(); app.UseFilter(); app.UseFilter(); app.UseFilter(); @@ -32,7 +35,7 @@ app.Add("deploy"); app.Add("legacy-docs"); -var githubActions = ConsoleApp.ServiceProvider.GetService(); +var githubActions = ConsoleApp.ServiceProvider!.GetService(); var command = githubActions?.GetInput("COMMAND"); if (!string.IsNullOrEmpty(command)) args = command.Split(' '); diff --git a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs index 64cc5770d..74521cf57 100644 --- a/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs +++ b/src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs @@ -25,14 +25,14 @@ public class AssemblerRepositorySourcer(ILoggerFactory logFactory, AssembleConte public CheckoutResult GetAll() { var fs = context.ReadFileSystem; - var repositories = Configuration.ReferenceRepositories.Values.Concat([Configuration.Narrative]); + var repositories = Configuration.AvailableRepositories; var checkouts = new List(); var linkRegistrySnapshotPath = Path.Combine(context.CheckoutDirectory.FullName, CheckoutResult.LinkRegistrySnapshotFileName); if (!fs.File.Exists(linkRegistrySnapshotPath)) throw new FileNotFoundException("Link-index snapshot not found. Run the clone-all command first.", linkRegistrySnapshotPath); var linkRegistrySnapshotStr = File.ReadAllText(linkRegistrySnapshotPath); var linkRegistry = LinkRegistry.Deserialize(linkRegistrySnapshotStr); - foreach (var repo in repositories) + foreach (var repo in repositories.Values) { var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(context.CheckoutDirectory.FullName, repo.Name)); IGitRepository gitFacade = new SingleCommitOptimizedGitRepository(context.Collector, checkoutFolder); @@ -68,12 +68,7 @@ public async Task CloneAll(bool fetchLatest, Cancel ctx = defaul ILinkIndexReader linkIndexReader = Aws3LinkIndexReader.CreateAnonymous(); var linkRegistry = await linkIndexReader.GetRegistry(ctx); - var repositories = new Dictionary(Configuration.ReferenceRepositories) - { - { NarrativeRepository.RepositoryName, Configuration.Narrative } - }; - - await Parallel.ForEachAsync(repositories, + await Parallel.ForEachAsync(Configuration.AvailableRepositories, new ParallelOptions { CancellationToken = ctx, diff --git a/src/tooling/docs-assembler/docs-assembler.csproj b/src/tooling/docs-assembler/docs-assembler.csproj index ccdd564d1..d7a387fc9 100644 --- a/src/tooling/docs-assembler/docs-assembler.csproj +++ b/src/tooling/docs-assembler/docs-assembler.csproj @@ -17,6 +17,7 @@ + diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs index bcca91319..2ab0c6289 100644 --- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs +++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs @@ -15,6 +15,7 @@ using Elastic.Documentation.Api.Infrastructure.Gcp; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Versions; +using Elastic.Documentation.ServiceDefaults; using Elastic.Documentation.Site.FileProviders; using Elastic.Documentation.Tooling; using Elastic.Markdown.IO; @@ -40,8 +41,9 @@ public DocumentationWebHost(ILoggerFactory logFactory, string? path, int port, I { _writeFileSystem = writeFs; var builder = WebApplication.CreateSlimBuilder(); + _ = builder.AddDocumentationServiceDefaults(); + builder.Services.AddElasticDocsApiUsecases("dev"); - DocumentationTooling.CreateServiceCollection(builder.Services, LogLevel.Information); _ = builder.Logging .AddFilter("Microsoft.AspNetCore.Hosting.Diagnostics", LogLevel.Error) .AddFilter("Microsoft.AspNetCore.StaticFiles.StaticFileMiddleware", LogLevel.Error) diff --git a/src/tooling/docs-builder/Http/StaticWebHost.cs b/src/tooling/docs-builder/Http/StaticWebHost.cs index 387c77548..dd63cb5df 100644 --- a/src/tooling/docs-builder/Http/StaticWebHost.cs +++ b/src/tooling/docs-builder/Http/StaticWebHost.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using Elastic.Documentation.Configuration; +using Elastic.Documentation.ServiceDefaults; using Elastic.Documentation.Tooling; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -14,7 +15,7 @@ namespace Documentation.Builder.Http; public class StaticWebHost { - private readonly WebApplication _webApplication; + public WebApplication WebApplication { get; } public StaticWebHost(int port) { @@ -24,7 +25,8 @@ public StaticWebHost(int port) { ContentRootPath = contentRoot }); - DocumentationTooling.CreateServiceCollection(builder.Services, LogLevel.Warning); + + _ = builder.AddDocumentationServiceDefaults(); _ = builder.Logging .AddFilter("Microsoft.AspNetCore.Hosting.Diagnostics", LogLevel.Error) @@ -32,23 +34,23 @@ public StaticWebHost(int port) .AddFilter("Microsoft.Hosting.Lifetime", LogLevel.Information); _ = builder.WebHost.UseUrls($"http://localhost:{port}"); - _webApplication = builder.Build(); + WebApplication = builder.Build(); SetUpRoutes(); } - public async Task RunAsync(Cancel ctx) => await _webApplication.RunAsync(ctx); + public async Task RunAsync(Cancel ctx) => await WebApplication.RunAsync(ctx); - public async Task StopAsync(Cancel ctx) => await _webApplication.StopAsync(ctx); + public async Task StopAsync(Cancel ctx) => await WebApplication.StopAsync(ctx); private void SetUpRoutes() { _ = - _webApplication + WebApplication .UseRouting(); - _ = _webApplication.MapGet("/", (Cancel _) => Results.Redirect("docs")); + _ = WebApplication.MapGet("/", (Cancel _) => Results.Redirect("docs")); - _ = _webApplication.MapGet("{**slug}", ServeDocumentationFile); + _ = WebApplication.MapGet("{**slug}", ServeDocumentationFile); } private async Task ServeDocumentationFile(string slug, Cancel _) diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 493064625..ddd76f905 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -3,16 +3,22 @@ // See the LICENSE file in the project root for more information using System.Diagnostics.CodeAnalysis; +using Actions.Core.Extensions; using ConsoleAppFramework; using Documentation.Builder.Cli; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.ServiceDefaults; using Elastic.Documentation.Tooling; using Elastic.Documentation.Tooling.Filters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -await using var serviceProvider = DocumentationTooling.CreateServiceProvider(ref args, (s, p) => { }); -ConsoleApp.ServiceProvider = serviceProvider; +var builder = Host.CreateApplicationBuilder() + .AddDocumentationServiceDefaults(ref args) + .AddDocumentationToolingDefaults(); -var app = ConsoleApp.Create(); +var app = builder.ToConsoleAppBuilder(); app.UseFilter(); app.UseFilter(); diff --git a/tests-integration/Directory.Build.props b/tests-integration/Directory.Build.props new file mode 100644 index 000000000..009f14389 --- /dev/null +++ b/tests-integration/Directory.Build.props @@ -0,0 +1,33 @@ + + + + + + false + false + true + CA1822 + + false + true + Exe + + + + + + + + + + + + + + + + + + + + diff --git a/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs b/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs new file mode 100644 index 000000000..8577ba5e7 --- /dev/null +++ b/tests-integration/Elastic.Assembler.IntegrationTests/AssembleFixture.cs @@ -0,0 +1,77 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using Elastic.Documentation.ServiceDefaults; +using FluentAssertions; +using InMemLogger; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +[assembly: CaptureConsole, AssemblyFixture(typeof(Elastic.Assembler.IntegrationTests.DocumentationFixture))] + +namespace Elastic.Assembler.IntegrationTests; + +public class DocumentationFixture : IAsyncLifetime +{ + public DistributedApplication DistributedApplication { get; private set; } = null!; + + public InMemoryLogger InMemoryLogger { get; private set; } = null!; + + /// + public async ValueTask InitializeAsync() + { + var builder = await DistributedApplicationTestingBuilder.CreateAsync( + ["--skip-private-repositories"], + (options, settings) => + { + options.DisableDashboard = true; + options.AllowUnsecuredTransport = true; + } + ); + _ = builder.Services.AddElasticDocumentationLogging(LogLevel.Information); + _ = builder.Services.AddLogging(c => c.AddXUnit()); + _ = builder.Services.AddLogging(c => c.AddInMemory()); + DistributedApplication = await builder.BuildAsync(); + InMemoryLogger = DistributedApplication.Services.GetService()!; + await DistributedApplication.StartAsync(); + } + + /// + public async ValueTask DisposeAsync() + { + await DistributedApplication.StopAsync(); + await DistributedApplication.DisposeAsync(); + GC.SuppressFinalize(this); + } +} + +public class ServeStaticTests(DocumentationFixture fixture, ITestOutputHelper output) : IAsyncLifetime +{ + [Fact] + public async Task AssertRequestToRootReturnsData() + { + _ = await fixture.DistributedApplication.ResourceNotifications + .WaitForResourceHealthyAsync("DocsBuilderServeStatic", cancellationToken: TestContext.Current.CancellationToken); + var client = fixture.DistributedApplication.CreateHttpClient("DocsBuilderServeStatic", "http"); + var root = await client.GetStringAsync("/", TestContext.Current.CancellationToken); + _ = root.Should().NotBeNullOrEmpty(); + } + + + /// + public ValueTask DisposeAsync() + { + GC.SuppressFinalize(this); + if (TestContext.Current.TestState?.Result is TestResult.Passed) + return default; + foreach (var resource in fixture.InMemoryLogger.RecordedLogs) + output.WriteLine(resource.Message); + return default; + } + + /// + public ValueTask InitializeAsync() => default; +} diff --git a/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj b/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj new file mode 100644 index 000000000..563a928ac --- /dev/null +++ b/tests-integration/Elastic.Assembler.IntegrationTests/Elastic.Assembler.IntegrationTests.csproj @@ -0,0 +1,21 @@ + + + + net9.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/tests-integration/Elastic.Documentation.Aspire/AppHost.cs b/tests-integration/Elastic.Documentation.Aspire/AppHost.cs new file mode 100644 index 000000000..fdb1848ad --- /dev/null +++ b/tests-integration/Elastic.Documentation.Aspire/AppHost.cs @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation; +using Microsoft.Extensions.Logging; + +var logLevel = LogLevel.Information; +GlobalCommandLine.Process(ref args, ref logLevel, out var skipPrivateRepositories); +var globalArguments = new List(); +if (skipPrivateRepositories) + globalArguments.Add("--skip-private-repositories"); + +if (logLevel != LogLevel.Information) +{ + globalArguments.Add("--log-level"); + globalArguments.Add(logLevel.ToString()); +} + +var builder = DistributedApplication.CreateBuilder(args); + +var cloneAll = builder.AddProject("DocsAssemblerCloneAll").WithArgs(["repo", "clone-all", .. globalArguments]); + +var buildAll = builder.AddProject("DocsAssemblerBuildAll").WithArgs(["repo", "build-all", .. globalArguments]) + .WaitForCompletion(cloneAll); + +var api = builder.AddProject("ApiLambda").WithArgs(globalArguments); + +var serveStatic = builder.AddProject("DocsBuilderServeStatic") + .WithHttpEndpoint(port: 4000, isProxied: false) + .WithArgs(["serve-static", .. globalArguments]) + .WithHttpHealthCheck("/", 200) + .WaitForCompletion(buildAll); + +//builder.AddElasticsearch("elasticsearch"); + +builder.Build().Run(); diff --git a/tests-integration/Elastic.Documentation.Aspire/Elastic.Documentation.Aspire.csproj b/tests-integration/Elastic.Documentation.Aspire/Elastic.Documentation.Aspire.csproj new file mode 100644 index 000000000..44282aa75 --- /dev/null +++ b/tests-integration/Elastic.Documentation.Aspire/Elastic.Documentation.Aspire.csproj @@ -0,0 +1,26 @@ + + + + + + Exe + net9.0 + enable + enable + 72f50f33-6fb9-4d08-bff3-39568fe370b3 + false + + + + + + + + + + + + + + + diff --git a/tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json b/tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json new file mode 100644 index 000000000..474d114c7 --- /dev/null +++ b/tests-integration/Elastic.Documentation.Aspire/Properties/launchSettings.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17166;http://localhost:15066", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21053", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22211" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15066", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19109", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20157" + } + } + } +} diff --git a/tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json b/tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json new file mode 100644 index 000000000..0c208ae91 --- /dev/null +++ b/tests-integration/Elastic.Documentation.Aspire/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/tests-integration/Elastic.Documentation.Aspire/appsettings.json b/tests-integration/Elastic.Documentation.Aspire/appsettings.json new file mode 100644 index 000000000..31c092aa4 --- /dev/null +++ b/tests-integration/Elastic.Documentation.Aspire/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Aspire.Hosting.Dcp": "Warning" + } + } +} diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props new file mode 100644 index 000000000..2339f7c53 --- /dev/null +++ b/tests/Directory.Build.props @@ -0,0 +1,31 @@ + + + + + + false + false + true + CA1822 + + false + true + Exe + + + + + + + + + + + + + + + + + + diff --git a/tests/Elastic.ApiExplorer.Tests/Elastic.ApiExplorer.Tests.csproj b/tests/Elastic.ApiExplorer.Tests/Elastic.ApiExplorer.Tests.csproj index c09d6cce9..35899012f 100644 --- a/tests/Elastic.ApiExplorer.Tests/Elastic.ApiExplorer.Tests.csproj +++ b/tests/Elastic.ApiExplorer.Tests/Elastic.ApiExplorer.Tests.csproj @@ -2,35 +2,10 @@ net9.0 - enable - enable - - false - true - Exe - - false - true - CA1822 - - - - - - - - - - - - - - - diff --git a/tests/Elastic.Documentation.LegacyDocs.Tests/Elastic.Documentation.LegacyDocs.Tests.csproj b/tests/Elastic.Documentation.LegacyDocs.Tests/Elastic.Documentation.LegacyDocs.Tests.csproj index 3b0324670..ae6ce021d 100644 --- a/tests/Elastic.Documentation.LegacyDocs.Tests/Elastic.Documentation.LegacyDocs.Tests.csproj +++ b/tests/Elastic.Documentation.LegacyDocs.Tests/Elastic.Documentation.LegacyDocs.Tests.csproj @@ -1,34 +1,9 @@  + + net9.0 + - - - net9.0 - enable - enable - - false - true - Library - - false - true - - - - - - - - - - - - - - - - - - - + + + diff --git a/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj b/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj index 5530dffce..01b287269 100644 --- a/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj +++ b/tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj @@ -2,28 +2,11 @@ net9.0 - enable - enable - - false - true - Exe - - false - true - - - - - - - - @@ -31,8 +14,4 @@ - - - - diff --git a/tests/authoring/authoring.fsproj b/tests/authoring/authoring.fsproj index 4bfb1f994..053238969 100644 --- a/tests/authoring/authoring.fsproj +++ b/tests/authoring/authoring.fsproj @@ -7,18 +7,9 @@ true Library - false - true - - - - - - - diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs index b232b9dc0..8504d7bd1 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/AssemblerConfigurationTests.cs @@ -10,6 +10,38 @@ namespace Documentation.Assembler.Tests; +public class PublicOnlyAssemblerConfigurationTests +{ + private DiagnosticsCollector Collector { get; } + private AssembleContext Context { get; } + private FileSystem FileSystem { get; } + private IDirectoryInfo CheckoutDirectory { get; set; } + public PublicOnlyAssemblerConfigurationTests() + { + FileSystem = new FileSystem(); + CheckoutDirectory = FileSystem.DirectoryInfo.New( + FileSystem.Path.Combine(Paths.GetSolutionDirectory()!.FullName, ".artifacts", "checkouts") + ); + Collector = new DiagnosticsCollector([]); + var configurationFileProvider = new ConfigurationFileProvider(FileSystem, skipPrivateRepositories: true); + var config = AssemblyConfiguration.Create(configurationFileProvider); + Context = new AssembleContext(config, configurationFileProvider, "dev", Collector, FileSystem, FileSystem, CheckoutDirectory.FullName, null); + } + + [Fact] + public void ReadsPrivateRepositories() + { + var config = Context.Configuration; + config.ReferenceRepositories.Should().NotBeEmpty().And.NotContainKey("cloud"); + config.PrivateRepositories.Should().NotBeEmpty().And.ContainKey("cloud"); + var cloud = config.PrivateRepositories["cloud"]; + cloud.Should().NotBeNull(); + cloud.GitReferenceCurrent.Should().NotBeNullOrEmpty() + .And.Be("master"); + } + +} + public class AssemblerConfigurationTests { private DiagnosticsCollector Collector { get; } @@ -76,5 +108,9 @@ public void ReadsVersions() var beats = config.ReferenceRepositories["beats"]; beats.GitReferenceCurrent.Should().NotBeNullOrEmpty() .And.NotBe("main"); + + var cloud = config.ReferenceRepositories["cloud"]; + cloud.GitReferenceCurrent.Should().NotBeNullOrEmpty() + .And.Be("master"); } } diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj b/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj index 817de1c4b..0fe87c358 100644 --- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj +++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/docs-assembler.Tests.csproj @@ -1,10 +1,6 @@ - - net9.0 - enable - enable false true @@ -17,21 +13,9 @@ - - - - - - - - - - - - - \ No newline at end of file +