Skip to content

Commit 36c4df9

Browse files
MpdreamzMpdreamz
committed
stage
1 parent 6eb6928 commit 36c4df9

File tree

9 files changed

+75
-82
lines changed

9 files changed

+75
-82
lines changed

‎Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
<PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
1717
<PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
1818
<PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
19-
<PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.8.0" />
19+
<PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.9.0" />
2020
</ItemGroup>
2121
<!-- Build -->
2222
<ItemGroup>

‎docs-builder.sln‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-filter", "assembl
9494
actions\assembler-filter\action.yml = actions\assembler-filter\action.yml
9595
EndProjectSection
9696
EndProject
97-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.Search", "src\Elastic.Documentation.Search\Elastic.Documentation.Search.csproj", "{052F70DE-CA5A-45F9-800B-E13CFEAE262C}"
98-
EndProject
9997
Global
10098
GlobalSection(SolutionConfigurationPlatforms) = preSolution
10199
Debug|Any CPU = Debug|Any CPU
@@ -157,10 +155,6 @@ Global
157155
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Debug|Any CPU.Build.0 = Debug|Any CPU
158156
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.ActiveCfg = Release|Any CPU
159157
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.Build.0 = Release|Any CPU
160-
{052F70DE-CA5A-45F9-800B-E13CFEAE262C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
161-
{052F70DE-CA5A-45F9-800B-E13CFEAE262C}.Debug|Any CPU.Build.0 = Debug|Any CPU
162-
{052F70DE-CA5A-45F9-800B-E13CFEAE262C}.Release|Any CPU.ActiveCfg = Release|Any CPU
163-
{052F70DE-CA5A-45F9-800B-E13CFEAE262C}.Release|Any CPU.Build.0 = Release|Any CPU
164158
EndGlobalSection
165159
GlobalSection(NestedProjects) = preSolution
166160
{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -184,6 +178,5 @@ Global
184178
{059E787F-85C1-43BE-9DD6-CE319E106383} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
185179
{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383}
186180
{FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
187-
{052F70DE-CA5A-45F9-800B-E13CFEAE262C} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
188181
EndGlobalSection
189182
EndGlobal

‎src/Elastic.Documentation.Search/IngestCollector.cs‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎src/Elastic.Documentation/Search/DocumentationDocument.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace Elastic.Documentation.Search;
6+
7+
public record DocumentationDocument
8+
{
9+
public string? Title { get; set; }
10+
}
11+

‎src/Elastic.Documentation/Serialization/SourceGenerationContext.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
using System.Text.Json.Serialization;
66
using Elastic.Documentation.Links;
7+
using Elastic.Documentation.Search;
78
using Elastic.Documentation.State;
89

910
namespace Elastic.Documentation.Serialization;
@@ -16,4 +17,5 @@ namespace Elastic.Documentation.Serialization;
1617
[JsonSerializable(typeof(GitCheckoutInformation))]
1718
[JsonSerializable(typeof(LinkReferenceRegistry))]
1819
[JsonSerializable(typeof(LinkRegistryEntry))]
20+
[JsonSerializable(typeof(DocumentationDocument))]
1921
public sealed partial class SourceGenerationContext : JsonSerializerContext;

‎src/Elastic.Markdown/DocumentationGenerator.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
103103

104104
var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState();
105105

106-
// clear output directory if force is true but never for assembler builds since these build multiple times to the output.
106+
// clear the output directory if force is true but never for assembler builds since these build multiple times to the output.
107107
if (Context is { AssemblerBuild: false, Force: true }
108108
// clear the output directory if force is false but generation state is null, except for assembler builds.
109109
|| (Context is { AssemblerBuild: false, Force: false } && generationState == null))

‎src/tooling/docs-assembler/Building/AssemblerBuilder.cs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
3535

3636
var redirects = new Dictionary<string, string>();
3737

38+
var esExporter =
39+
Environment.GetEnvironmentVariable("ELASTIC_API_KEY") is { } apiKey &&
40+
Environment.GetEnvironmentVariable("ELASTIC_URL") is { } url
41+
? new ElasticsearchMarkdownExporter(logger, url, apiKey)
42+
: null;
43+
IMarkdownExporter[] markdownExporters = esExporter is null ? [] : [esExporter];
44+
3845
foreach (var (_, set) in assembleSets)
3946
{
4047
var checkout = set.Checkout;
@@ -46,7 +53,7 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
4653

4754
try
4855
{
49-
var result = await BuildAsync(set, ctx);
56+
var result = await BuildAsync(set, markdownExporters, ctx);
5057
CollectRedirects(redirects, result.Redirects, checkout.Repository.Name, set.DocumentationSet.LinkResolver);
5158
}
5259
catch (Exception e) when (e.Message.Contains("Can not locate docset.yml file in"))
@@ -60,6 +67,9 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
6067
}
6168
}
6269

70+
if (esExporter is not null)
71+
await esExporter.WaitForDrain();
72+
6373
await context.Collector.StopAsync(ctx);
6474
}
6575

@@ -92,13 +102,8 @@ string Resolve(string relativeMarkdownPath)
92102
}
93103
}
94104

95-
private async Task<GenerationResult> BuildAsync(AssemblerDocumentationSet set, Cancel ctx)
105+
private async Task<GenerationResult> BuildAsync(AssemblerDocumentationSet set, IMarkdownExporter[]? markdownExporters, Cancel ctx)
96106
{
97-
IMarkdownExporter[]? markdownExporters =
98-
Environment.GetEnvironmentVariable("ELASTIC_API_KEY") is { } apiKey &&
99-
Environment.GetEnvironmentVariable("ELASTIC_URL") is { } url
100-
? [new ElasticsearchMarkdownExporter(logger, url, apiKey)]
101-
: null;
102107
var generator = new DocumentationGenerator(
103108
set.DocumentationSet,
104109
logger, HtmlWriter,

‎src/tooling/docs-assembler/Indexing/ElasticsearchMarkdownExporter.cs‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,62 @@
33
// See the LICENSE file in the project root for more information
44

55
using Elastic.Documentation.Search;
6+
using Elastic.Documentation.Serialization;
7+
using Elastic.Ingest.Elasticsearch.Indices;
68
using Elastic.Markdown.Exporters;
79
using Elastic.Markdown.IO;
10+
using Elastic.Transport;
11+
using Elastic.Transport.Products.Elasticsearch;
812
using Microsoft.Extensions.Logging;
913

1014
namespace Documentation.Assembler.Indexing;
1115

12-
public class ElasticsearchMarkdownExporter(ILoggerFactory logFactory, string url, string apiKey) : IMarkdownExporter, IDisposable
16+
public class ElasticsearchMarkdownExporter : IMarkdownExporter, IDisposable
1317
{
14-
private readonly IngestCollector _ingestCollector = new(logFactory, url, apiKey);
18+
private readonly IndexChannel<DocumentationDocument> _channel;
19+
private readonly ILogger<ElasticsearchMarkdownExporter> _logger;
20+
21+
public ElasticsearchMarkdownExporter(ILoggerFactory logFactory, string url, string apiKey)
22+
{
23+
_logger = logFactory.CreateLogger<ElasticsearchMarkdownExporter>();
24+
var configuration = new ElasticsearchConfiguration(new Uri(url), new ApiKey(apiKey))
25+
{
26+
//Uncomment to see the requests with Fiddler
27+
//ProxyAddress = "http://localhost:8866"
28+
};
29+
var transport = new DistributedTransport(configuration);
30+
var options = new IndexChannelOptions<DocumentationDocument>(transport)
31+
{
32+
SerializerContext = SourceGenerationContext.Default,
33+
IndexFormat = "documentation",
34+
ExportExceptionCallback = e => _logger.LogError(e, "Failed to export document"),
35+
ServerRejectionCallback = items => _logger.LogInformation("Server rejection: {Rejection}", items.First().Item2)
36+
};
37+
_channel = new IndexChannel<DocumentationDocument>(options);
38+
}
39+
40+
public async Task WaitForDrain()
41+
{
42+
_logger.LogInformation("Elasticsearch export: waiting for in flight exports");
43+
var drained = await _channel.WaitForDrainAsync();
44+
if (!drained)
45+
_logger.LogError("Elasticsearch export: failed to complete indexing in a timely fashion while shutting down");
46+
}
47+
48+
private async ValueTask<bool> TryWrite(DocumentationDocument document, Cancel ctx = default)
49+
{
50+
if (_channel.TryWrite(document))
51+
return true;
52+
53+
if (await _channel.WaitToWriteAsync(ctx))
54+
return _channel.TryWrite(document);
55+
return false;
56+
}
1557

1658
public void Dispose()
1759
{
18-
_ingestCollector.Dispose();
60+
_channel.Complete();
61+
_channel.Dispose();
1962
GC.SuppressFinalize(this);
2063
}
2164

@@ -25,6 +68,6 @@ public async ValueTask<bool> Export(MarkdownFile file)
2568
{
2669
Title = file.Title,
2770
};
28-
return await _ingestCollector.TryWrite(doc);
71+
return await TryWrite(doc);
2972
}
3073
}

‎src/tooling/docs-assembler/docs-assembler.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@
1919
<PackageReference Include="AWSSDK.S3"/>
2020
<PackageReference Include="ConsoleAppFramework.Abstractions"/>
2121
<PackageReference Include="ConsoleAppFramework" />
22+
<PackageReference Include="Elastic.Ingest.Elasticsearch" />
2223
<PackageReference Include="Proc" />
2324
<PackageReference Include="YamlDotNet" />
2425
<PackageReference Include="NetEscapades.EnumGenerators" />
2526
<PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" />
2627
</ItemGroup>
2728

2829
<ItemGroup>
29-
<ProjectReference Include="..\..\Elastic.Documentation.Search\Elastic.Documentation.Search.csproj" />
3030
<ProjectReference Include="..\..\Elastic.Documentation\Elastic.Documentation.csproj" />
3131
<ProjectReference Include="..\..\Elastic.Markdown\Elastic.Markdown.csproj" />
3232
<ProjectReference Include="..\Elastic.Documentation.Tooling\Elastic.Documentation.Tooling.csproj" />

0 commit comments

Comments
 (0)