Skip to content

Commit 6e1a7be

Browse files
MpdreamzMpdreamz
committed
Add exporter options to assembler command line
1 parent 8e4b8d3 commit 6e1a7be

File tree

17 files changed

+344
-148
lines changed

17 files changed

+344
-148
lines changed

‎Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
<PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
1717
<PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
1818
<PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
19-
<PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.9.0" />
19+
<PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.11.1" />
2020
</ItemGroup>
2121
<!-- Build -->
2222
<ItemGroup>

‎src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,18 @@ public static void EmitWarning(this IDiagnosticsCollector collector, IFileInfo f
3535

3636
public static void EmitHint(this IDiagnosticsCollector collector, IFileInfo file, string message) =>
3737
collector.EmitHint(file.FullName, message);
38+
39+
/// Emit an error not associated with a file
40+
public static void EmitGlobalError(this IDiagnosticsCollector collector, string message, Exception? e = null) =>
41+
collector.EmitError(string.Empty, message, e);
42+
43+
/// Emit a warning not associated with a file
44+
public static void EmitGlobalWarning(this IDiagnosticsCollector collector, string message) =>
45+
collector.EmitWarning(string.Empty, message);
46+
47+
/// Emit a hint not associated with a file
48+
public static void EmitGlobalHint(this IDiagnosticsCollector collector, string message) =>
49+
collector.EmitHint(string.Empty, message);
3850
}
3951

4052

‎src/Elastic.Documentation/Search/DocumentationDocument.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,28 @@
22
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information
44

5+
using System.Text.Json.Serialization;
6+
57
namespace Elastic.Documentation.Search;
68

79
public record DocumentationDocument
810
{
11+
[JsonPropertyName("title")]
912
public string? Title { get; set; }
13+
14+
[JsonPropertyName("body")]
15+
public string? Body { get; set; }
16+
17+
[JsonPropertyName("abstract")]
18+
public string? Abstract { get; set; }
19+
20+
[JsonPropertyName("headings")]
21+
public string[] Headings { get; set; } = [];
22+
23+
[JsonPropertyName("links")]
24+
public string[] Links { get; set; } = [];
25+
26+
[JsonPropertyName("url")]
27+
public string? Url { get; set; }
1028
}
1129

‎src/Elastic.Markdown/DocumentationGenerator.cs‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
namespace Elastic.Markdown;
2020

21+
/// Used primarily for testing, do not use in production paths since it might keep references alive to long
2122
public interface IConversionCollector
2223
{
2324
void Collect(MarkdownFile file, MarkdownDocument document, string html);
@@ -212,7 +213,7 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
212213
}
213214
}
214215

215-
private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, Cancel token)
216+
private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, Cancel ctx)
216217
{
217218
if (!Context.Force)
218219
{
@@ -222,17 +223,28 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
222223
return;
223224
}
224225

225-
if (file is MarkdownFile markdown)
226-
{
227-
foreach (var exporter in _markdownExporters)
228-
_ = await exporter.Export(markdown);
229-
}
230-
231226
_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
232-
//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
233227
var outputFile = OutputFile(file.RelativePath);
234228
if (outputFile is not null)
235-
await _documentationFileExporter.ProcessFile(Context, file, outputFile, HtmlWriter, _conversionCollector, token);
229+
{
230+
var context = new ProcessingFileContext
231+
{
232+
BuildContext = Context,
233+
OutputFile = outputFile,
234+
ConversionCollector = _conversionCollector,
235+
File = file,
236+
HtmlWriter = HtmlWriter
237+
};
238+
await _documentationFileExporter.ProcessFile(context, ctx);
239+
if (file is MarkdownFile markdown)
240+
{
241+
foreach (var exporter in _markdownExporters)
242+
{
243+
var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx);
244+
_ = await exporter.ExportAsync(new MarkdownExportContext { Document = document, File = markdown }, ctx);
245+
}
246+
}
247+
}
236248
}
237249

238250
private IFileInfo? OutputFile(string relativePath)

‎src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,27 @@
55
using System.IO.Abstractions;
66
using Elastic.Markdown.IO;
77
using Elastic.Markdown.Slices;
8+
using Markdig.Syntax;
89

910
namespace Elastic.Markdown.Exporters;
1011

12+
public class ProcessingFileContext
13+
{
14+
public required BuildContext BuildContext { get; init; }
15+
public required DocumentationFile File { get; init; }
16+
public required IFileInfo OutputFile { get; init; }
17+
public required HtmlWriter HtmlWriter { get; init; }
18+
public required IConversionCollector? ConversionCollector { get; init; }
19+
20+
public MarkdownDocument? MarkdownDocument { get; set; }
21+
}
22+
1123
public interface IDocumentationFileExporter
1224
{
13-
/// Used in documentation state to ensure we break the build cache if a different exporter is chosen
25+
/// Used in the documentation state to ensure we break the build cache if a different exporter is chosen
1426
string Name { get; }
1527

16-
Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter, IConversionCollector? conversionCollector,
17-
Cancel token);
28+
ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx);
1829

1930
Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx);
2031
}
@@ -23,16 +34,14 @@ public abstract class DocumentationFileExporterBase(IFileSystem readFileSystem,
2334
{
2435
public abstract string Name { get; }
2536

26-
public abstract Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter,
27-
IConversionCollector? conversionCollector,
28-
Cancel token);
37+
public abstract ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx);
2938

3039
protected async Task CopyFileFsAware(DocumentationFile file, IFileInfo outputFile, Cancel ctx)
3140
{
3241
// fast path, normal case.
3342
if (readFileSystem == writeFileSystem)
3443
readFileSystem.File.Copy(file.SourceFile.FullName, outputFile.FullName, true);
35-
//slower when we are mocking the write filesystem
44+
//slower when we are mocking the write-filesystem
3645
else
3746
{
3847
var bytes = await file.SourceFile.FileSystem.File.ReadAllBytesAsync(file.SourceFile.FullName, ctx);
@@ -49,26 +58,20 @@ public async Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStre
4958
}
5059
}
5160

52-
public class DocumentationFileExporter(
53-
IFileSystem readFileSystem,
54-
IFileSystem writeFileSystem
55-
) : DocumentationFileExporterBase(readFileSystem, writeFileSystem)
61+
public class DocumentationFileExporter(IFileSystem readFileSystem, IFileSystem writeFileSystem)
62+
: DocumentationFileExporterBase(readFileSystem, writeFileSystem)
5663
{
57-
public override string Name { get; } = nameof(DocumentationFileExporter);
64+
public override string Name => nameof(DocumentationFileExporter);
5865

59-
public override async Task ProcessFile(BuildContext context, DocumentationFile file,
60-
IFileInfo outputFile,
61-
HtmlWriter htmlWriter,
62-
IConversionCollector? conversionCollector,
63-
Cancel token)
66+
public override async ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx)
6467
{
65-
if (file is MarkdownFile markdown)
66-
await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, token);
68+
if (context.File is MarkdownFile markdown)
69+
context.MarkdownDocument = await context.HtmlWriter.WriteAsync(context.OutputFile, markdown, context.ConversionCollector, ctx);
6770
else
6871
{
69-
if (outputFile.Directory is { Exists: false })
70-
outputFile.Directory.Create();
71-
await CopyFileFsAware(file, outputFile, token);
72+
if (context.OutputFile.Directory is { Exists: false })
73+
context.OutputFile.Directory.Create();
74+
await CopyFileFsAware(context.File, context.OutputFile, ctx);
7275
}
7376
}
7477
}

‎src/Elastic.Markdown/Exporters/IMarkdownExporter.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,20 @@
33
// See the LICENSE file in the project root for more information
44

55
using Elastic.Markdown.IO;
6+
using Markdig.Syntax;
67

78
namespace Elastic.Markdown.Exporters;
89

10+
public class MarkdownExportContext
11+
{
12+
public required MarkdownDocument Document { get; init; }
13+
public required MarkdownFile File { get; init; }
14+
public string? LLMText { get; set; }
15+
}
16+
917
public interface IMarkdownExporter
1018
{
11-
ValueTask<bool> Export(MarkdownFile file);
19+
ValueTask StartAsync(Cancel ctx = default);
20+
ValueTask StopAsync(Cancel ctx = default);
21+
ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel ctx);
1222
}

‎src/Elastic.Markdown/Exporters/NoopDocumentationFileExporter.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,15 @@
33
// See the LICENSE file in the project root for more information
44

55
using System.IO.Abstractions;
6-
using Elastic.Markdown.IO;
7-
using Elastic.Markdown.Slices;
86

97
namespace Elastic.Markdown.Exporters;
108

119
public class NoopDocumentationFileExporter : IDocumentationFileExporter
1210
{
1311
public string Name { get; } = nameof(NoopDocumentationFileExporter);
1412

15-
public Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter,
16-
IConversionCollector? conversionCollector, Cancel token) =>
17-
Task.CompletedTask;
13+
public ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx) =>
14+
ValueTask.CompletedTask;
1815

1916
public Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx) => Task.CompletedTask;
2017
}

‎src/Elastic.Markdown/Extensions/DetectionRules/RuleDocumentationFileExporter.cs‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,24 @@ public class RuleDocumentationFileExporter(IFileSystem readFileSystem, IFileSyst
1414
{
1515
public override string Name { get; } = nameof(RuleDocumentationFileExporter);
1616

17-
public override async Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter,
18-
IConversionCollector? conversionCollector, Cancel token)
17+
public override async ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx)
1918
{
20-
if (file is DetectionRuleFile df)
21-
await htmlWriter.WriteAsync(DetectionRuleFile.OutputPath(outputFile, context), df, conversionCollector, token);
22-
else if (file is MarkdownFile markdown)
23-
await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, token);
24-
else
19+
var htmlWriter = context.HtmlWriter;
20+
var outputFile = context.OutputFile;
21+
var conversionCollector = context.ConversionCollector;
22+
switch (context.File)
2523
{
26-
if (outputFile.Directory is { Exists: false })
27-
outputFile.Directory.Create();
28-
await CopyFileFsAware(file, outputFile, token);
24+
case DetectionRuleFile df:
25+
context.MarkdownDocument = await htmlWriter.WriteAsync(DetectionRuleFile.OutputPath(outputFile, context.BuildContext), df, conversionCollector, ctx);
26+
break;
27+
case MarkdownFile markdown:
28+
context.MarkdownDocument = await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, ctx);
29+
break;
30+
default:
31+
if (outputFile.Directory is { Exists: false })
32+
outputFile.Directory.Create();
33+
await CopyFileFsAware(context.File, outputFile, ctx);
34+
break;
2935
}
3036
}
3137
}

‎src/Elastic.Markdown/IO/MarkdownFile.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
using Elastic.Markdown.Slices;
1818
using Markdig;
1919
using Markdig.Extensions.Yaml;
20+
using Markdig.Renderers.Roundtrip;
2021
using Markdig.Syntax;
2122

2223
namespace Elastic.Markdown.IO;
@@ -186,6 +187,17 @@ public async Task<MarkdownDocument> ParseFullAsync(Cancel ctx)
186187
return document;
187188
}
188189

190+
public static string ToLLMText(MarkdownDocument document)
191+
{
192+
using var sw = new StringWriter();
193+
var rr = new RoundtripRenderer(sw);
194+
rr.Write(document);
195+
var outputMarkdown = sw.ToString();
196+
197+
return outputMarkdown;
198+
199+
}
200+
189201
private IReadOnlyDictionary<string, string> GetSubstitutions()
190202
{
191203
var globalSubstitutions = _globalSubstitutions;

‎src/Elastic.Markdown/Myst/MarkdownParser.cs‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,21 +31,11 @@ public class MarkdownParser(BuildContext build, IParserResolvers resolvers)
3131
private BuildContext Build { get; } = build;
3232
private IParserResolvers Resolvers { get; } = resolvers;
3333

34-
public Task<MarkdownDocument> MinimalParseAsync(IFileInfo path, Cancel ctx)
35-
{
36-
var state = new ParserState(Build)
37-
{
38-
MarkdownSourcePath = path,
39-
YamlFrontMatter = null,
40-
DocumentationFileLookup = Resolvers.DocumentationFileLookup,
41-
CrossLinkResolver = Resolvers.CrossLinkResolver,
42-
SkipValidation = true
43-
};
44-
var context = new ParserContext(state);
45-
return ParseAsync(path, context, MinimalPipeline, ctx);
46-
}
34+
public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx) => ParseFromFile(path, matter, Pipeline, ctx);
4735

48-
public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx)
36+
public Task<MarkdownDocument> MinimalParseAsync(IFileInfo path, Cancel ctx) => ParseFromFile(path, null, MinimalPipeline, ctx);
37+
38+
private Task<MarkdownDocument> ParseFromFile(IFileInfo path, YamlFrontMatter? matter, MarkdownPipeline pipeline, Cancel ctx)
4939
{
5040
var state = new ParserState(Build)
5141
{
@@ -55,7 +45,7 @@ public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter
5545
CrossLinkResolver = Resolvers.CrossLinkResolver
5646
};
5747
var context = new ParserContext(state);
58-
return ParseAsync(path, context, Pipeline, ctx);
48+
return ParseAsync(path, context, pipeline, ctx);
5949
}
6050

6151
public Task<MarkdownDocument> ParseSnippetAsync(IFileInfo path, IFileInfo parentPath, YamlFrontMatter? matter, Cancel ctx)

0 commit comments

Comments
 (0)