Merge branch 'main' into crosslinks-in-toc-take-three

Author: theletterf

.editorconfig

@@ -246,7 +246,9 @@ dotnet_diagnostic.IDE0305.severity = none
 # CS8509 already warns
 dotnet_diagnostic.IDE0072.severity = none
 
-
+[src/api/Elastic.Documentation.Api.Lambda/**.cs]
+dotnet_diagnostic.IL3050.severity = none
+dotnet_diagnostic.IL2026.severity = none
 
 [DocumentationWebHost.cs]
 dotnet_diagnostic.IL3050.severity = none

.github/actions/bootstrap/action.yml

@@ -18,6 +18,12 @@ runs:
       run: |
         git fetch --prune --unshallow --tags
         git tag --list
+
+    - name: Git config
+      shell: bash
+      run: |
+        git config --global init.defaultBranch main
+
     - uses: actions/setup-dotnet@v4
       with:
         global-json-file: global.json
@@ -37,4 +43,3 @@ runs:
         cache: npm
         cache-dependency-path: src/Elastic.Documentation.Site/package-lock.json
         node-version-file: .nvmrc
-      

.github/workflows/build-api-lambda.yml

@@ -0,0 +1,44 @@
+---
+# This workflow is used to build the API lambda
+# lambda function bootstrap binary that can be deployed to AWS Lambda.
+name: Build API Lambda
+
+on:
+  workflow_dispatch: 
+  workflow_call: 
+    inputs:
+      ref:
+        required: false
+        type: string
+        default: ${{ github.ref }}
+        
+permissions: 
+  contents: read
+
+jobs: 
+  build:
+    runs-on: ubuntu-latest
+    env:
+      BINARY_PATH: .artifacts/Elastic.Documentation.Api.Lambda/release_linux-x64/bootstrap
+    steps:
+      - uses: actions/checkout@v4
+        with: 
+          ref: ${{ inputs.ref }}
+      - name: Amazon Linux 2023 build
+        run: |
+          docker build . -t api-lambda:latest -f src/api/Elastic.Documentation.Api.Lambda/Dockerfile
+      - name: Get bootstrap binary
+        run: |
+          docker cp $(docker create --name tc api-lambda:latest):/app/.artifacts/publish ./.artifacts && docker rm tc
+      - name: Inspect bootstrap binary
+        run: |
+          tree .artifacts
+          stat "${BINARY_PATH}"
+      - name: Archive artifact
+        id: upload-artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: api-lambda-binary
+          retention-days: 1
+          if-no-files-found: error
+          path: ${{ env.BINARY_PATH }}

.github/workflows/ci.yml

@@ -32,9 +32,12 @@ jobs:
       - name: Validate Content Sources
         run: dotnet run --project src/tooling/docs-assembler -c release -- content-source validate
 
-  build-lambda:
+  build-link-index-updater-lambda:
     uses: ./.github/workflows/build-link-index-updater-lambda.yml
-
+  
+  build-api-lambda:
+    uses: ./.github/workflows/build-api-lambda.yml
+  
   npm:
     runs-on: ubuntu-latest
     defaults: 
@@ -81,11 +84,36 @@ jobs:
         id: bootstrap
         uses: ./.github/actions/bootstrap
 
+      - name: Install Aspire workload
+        run: dotnet workload install aspire
+
       - name: Build
         run: dotnet run --project build -c release
 
       - name: Test
-        run: dotnet run --project build -c release -- test
+        run: dotnet run --project build -c release -- unit-test
 
       - name: Publish AOT
         run: dotnet run --project build -c release -- publishbinaries
+
+  integration:
+    if: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          tool-cache: false
+          dotnet: true
+
+      - name: Bootstrap Action Workspace
+        id: bootstrap
+        uses: ./.github/actions/bootstrap
+
+      - name: Install Aspire workload
+        run: dotnet workload install aspire
+
+      - name: Integration Tests
+        run: dotnet run --project build -c release -- integrate

.github/workflows/deploy-api-lambda.yml

@@ -0,0 +1,51 @@
+name: Deploy API Lambda
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch: 
+    
+permissions: 
+  contents: read
+
+jobs:
+  build:
+    permissions: 
+      contents: read
+    uses: ./.github/workflows/build-api-lambda.yml
+    
+  deploy:
+    permissions: 
+      contents: read
+      id-token: write
+    runs-on: ubuntu-latest
+    needs: build
+    environment: docs-api-edge
+    env:
+      ZIP_FILE: api-lambda.zip
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download bootstrap binary
+        uses: actions/download-artifact@v4
+        with:
+          name: api-lambda-binary
+          
+      - name: Download bootstrap binary
+        uses: actions/download-artifact@v4
+        with:
+          name: api-lambda-binary # Defined in build-api-lambda.yml
+
+      - name: Create zip
+        run: |
+          zip -j "${ZIP_FILE}" ./bootstrap
+
+      - uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
+        with:
+          role-to-assume: arn:aws:iam::197730964718:role/elastic-docs-v3-api-deployer-edge
+          aws-region: us-east-1
+
+      - name: Upload Lambda function
+        run: |
+          aws lambda update-function-code \
+            --function-name elastic-docs-v3-edge-api \
+            --zip-file "fileb://${ZIP_FILE}"

Directory.Packages.props

@@ -2,24 +2,36 @@
   <PropertyGroup>
     <!-- Enable central package management, https://learn.microsoft.com/en-us/nuget/consume-packages/Central-Package-Management -->
     <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+    <MinVerSkip Condition="'$(Configuration)' == 'Debug'">true</MinVerSkip>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
     <GlobalPackageReference Include="MinVer" Version="6.0.0" PrivateAssets="All" />
   </ItemGroup>
   <!-- AWS -->
   <ItemGroup>
+    <PackageVersion Include="Amazon.Lambda.AspNetCoreServer.Hosting" Version="1.9.0" />
     <PackageVersion Include="Amazon.Lambda.RuntimeSupport" Version="1.13.0" />
     <PackageVersion Include="Amazon.Lambda.Core" Version="2.5.1" />
     <PackageVersion Include="Amazon.Lambda.S3Events" Version="3.1.0" />
     <PackageVersion Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
     <PackageVersion Include="Amazon.Lambda.SQSEvents" Version="2.2.0" />
+    <PackageVersion Include="Aspire.Hosting" Version="9.4.0" />
+    <PackageVersion Include="Aspire.Hosting.Testing" Version="9.4.0" />
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
+    <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="FakeItEasy" Version="8.3.0" />
     <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.11.3" />
+    <PackageVersion Include="InMemoryLogger" Version="1.0.66" />
+    <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.6.0" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.4" />
     <PackageVersion Include="Microsoft.OpenApi" Version="2.0.0-preview9" />
     <PackageVersion Include="System.Text.Json" Version="9.0.5" />
+    <PackageVersion Include="TUnit" Version="0.25.21" />
+    <PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
   </ItemGroup>
   <!-- Build -->
   <ItemGroup>
@@ -37,12 +49,12 @@
     <PackageVersion Include="DotNet.Glob" Version="3.1.3" />
     <PackageVersion Include="Errata" Version="0.14.0" />
     <PackageVersion Include="Github.Actions.Core" Version="9.0.0" />
-    <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.4" />
-    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="9.0.4" />
+    <PackageVersion Include="Microsoft.Extensions.Logging" Version="9.0.7" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="9.0.7" />
     <PackageVersion Include="Markdig" Version="0.41.1" />
     <PackageVersion Include="NetEscapades.EnumGenerators" Version="1.0.0-beta12" PrivateAssets="all" ExcludeAssets="runtime" />
     <PackageVersion Include="Proc" Version="0.9.1" />
-    <PackageVersion Include="RazorSlices" Version="0.9.2" />
+    <PackageVersion Include="RazorSlices" Version="0.9.4" />
     <PackageVersion Include="Samboy063.Tomlet" Version="6.0.0" />
     <PackageVersion Include="Slugify.Core" Version="4.0.1" />
     <PackageVersion Include="SoftCircuits.IniFileParser" Version="2.7.0" />
@@ -55,6 +67,17 @@
   </ItemGroup>
   <!-- Test packages -->
   <ItemGroup>
+    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="9.7.0" />
+    <PackageVersion Include="Microsoft.Extensions.ServiceDiscovery" Version="9.4.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.12.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.12.0" />
+  </ItemGroup>
+  <!-- Test packages -->
+  <ItemGroup>
+    <PackageVersion Include="Aspire.Hosting.AppHost" Version="9.4.0" />
     <PackageVersion Include="AngleSharp.Diffing" Version="1.0.0" />
     <PackageVersion Include="DiffPlex" Version="1.7.2" />
     <PackageVersion Include="FluentAssertions" Version="7.2.0" />
@@ -70,4 +93,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

build/CommandLine.fs

@@ -9,13 +9,22 @@ open Microsoft.FSharp.Reflection
 open System
 open Bullseye
 
+type TestSuite = All | Unit | Integration
+    with
+    member this.SuitName =
+        match FSharpValue.GetUnionFields(this, typeof<TestSuite>) with
+        | case, _ -> case.Name.ToLowerInvariant()
+
 type Build =
     | [<CliPrefix(CliPrefix.None);SubCommand>] Clean
     | [<CliPrefix(CliPrefix.None);SubCommand>] Version
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] Compile
     | [<CliPrefix(CliPrefix.None);SubCommand>] Build
+
     | [<CliPrefix(CliPrefix.None);SubCommand>] Test
-    
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Unit_Test
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Integrate
+
     | [<CliPrefix(CliPrefix.None);SubCommand>] Format
     | [<CliPrefix(CliPrefix.None);SubCommand>] Watch
 
@@ -33,6 +42,7 @@ type Build =
     | [<Inherit;AltCommandLine("-s")>] Single_Target
     | [<Inherit>] Token of string 
     | [<Inherit;AltCommandLine("-c")>] Skip_Dirty_Check
+    | [<Inherit;EqualsAssignment>] Test_Suite of TestSuite
 with
     interface IArgParserTemplate with
         member this.Usage =
@@ -41,8 +51,11 @@ with
             | Clean -> "clean known output locations"
             | Version -> "print version information"
             | Build -> "Run build"
-            
-            | Test -> "runs a clean build and then runs all the tests "
+
+            | Unit_Test -> "alias to providing: test --test-suite=unit"
+            | Integrate -> "alias to providing: test --test-suite=integration"
+            | Test -> "runs a clean build and then runs all the tests unless --test-suite is provided"
+
             | Release -> "runs build, tests, and create and validates the packages shy of publishing them"
             | Publish -> "Publishes artifacts"
             | Format -> "runs dotnet format"
@@ -62,6 +75,7 @@ with
             | Single_Target -> "Runs the provided sub command without running their dependencies"
             | Token _ -> "Token to be used to authenticate with github"
             | Skip_Dirty_Check -> "Skip the clean checkout check that guards the release/publish targets"
+            | Test_Suite _ -> "Specify the test suite to run, defaults to all"
 
     member this.StepName =
         match FSharpValue.GetUnionFields(this, typeof<Build>) with
@@ -87,4 +101,4 @@ with
         let dependsOn = if singleTarget then [] else dependsOn
 
         let steps = dependsOn @ composedOf |> List.map _.StepName
-        Targets.Target(target.StepName, steps, Action(fun _ -> action parsed))
+        Targets.Target(target.StepName, steps, Action(fun _ -> action parsed))

build/Targets.fs

@@ -34,7 +34,7 @@ let private version _ =
 
 let private format _ = exec { run "dotnet" "format" "--verbosity" "quiet" }
 
-let private watch _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--no-hot-reload" "--" "serve" }
+let private watch _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "serve" }
 
 let private lint _ =
     match exec {
@@ -107,10 +107,17 @@ let private publishContainers _ =
     createImage "docs-builder"
     createImage "docs-assembler"
 
-let private runTests _ =
+let private runTests (testSuite: TestSuite) _ =
+    let testFilter =
+        match testSuite with
+        | All -> []
+        | Unit -> ["--filter"; "FullyQualifiedName~.Tests"]
+        | Integration -> ["--filter"; "FullyQualifiedName~.IntegrationTests"]
+
     exec {
         run "dotnet" (
             ["test"; "-c"; "release"; "--no-restore"; "--no-build"; "--logger"; "GitHubActions"]
+            @ testFilter
             @ ["--"; "RunConfiguration.CollectSourceInformation=true"]
         )
     }
@@ -132,8 +139,10 @@ let Setup (parsed:ParseResults<Build>) =
             Build.Cmd
                 [Clean; Lint; Compile] [] build
 
-        | Test -> Build.Cmd [Compile] [] runTests
-        
+        | Test -> Build.Cmd [Compile] [] <| runTests TestSuite.All
+        | Unit_Test -> Build.Cmd [Compile] [] <| runTests TestSuite.Unit
+        | Integrate -> Build.Cmd [Compile] [] <| runTests TestSuite.Integration
+
         | Release ->
             Build.Cmd 
                 [PristineCheck; Build]
@@ -159,6 +168,7 @@ let Setup (parsed:ParseResults<Build>) =
 
         // flags
         | Single_Target
+        | Test_Suite _
         | Token _
         | Skip_Dirty_Check -> Build.Ignore
 

config/assembler.yml

@@ -25,6 +25,7 @@ environments:
       enabled: false
     feature_flags:
       LAZY_LOAD_NAVIGATION: true
+      SEARCH_OR_ASK_AI: true
   dev:
     uri: http://localhost:4000
     content_source: current
@@ -94,6 +95,7 @@ references:
   elastic-otel-python:
   elastic-serverless-forwarder:
   integration-docs:
+    private: true
   integrations:
   logstash-docs-md:
   opentelemetry:
@@ -104,7 +106,12 @@ references:
 
   # @elastic/admin-docs
   cloud-on-k8s:
-  cloud: *master
+  cloud:
+    current: master
+    next: master
+    edge: master
+    private: true
+
   curator: *master
   ecctl: *master