Skip to content

Commit cf9e9ec

Browse files
SamRemisSamRemis
committed
Updates based on PR feedback
1 parent eb0f27b commit cf9e9ec

File tree

11 files changed

+93
-62
lines changed

11 files changed

+93
-62
lines changed

‎codegen/aws/core/build.gradle.kts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,4 @@ extra["moduleName"] = "software.amazon.smithy.python.aws.codegen"
1212
dependencies {
1313
implementation(project(":core"))
1414
implementation(libs.smithy.aws.traits)
15-
implementation("org.jsoup:jsoup:1.19.1")
1615
}

‎codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsRstDocFileGenerator.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ public List<? extends CodeInterceptor<? extends CodeSection, PythonWriter>> inte
2323
GenerationContext context
2424
) {
2525
return List.of(
26+
// We generate custom RST files for each member that we want to have
27+
// its own page. This gives us much more fine-grained control of
28+
// what gets generated than just using automodule or autoclass on
29+
// the client would alone.
2630
new OperationGenerationInterceptor(context),
2731
new StructureGenerationInterceptor(context),
2832
new ErrorGenerationInterceptor(context),

‎codegen/aws/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,4 @@ software.amazon.smithy.python.aws.codegen.AwsProtocolsIntegration
88
software.amazon.smithy.python.aws.codegen.AwsServiceIdIntegration
99
software.amazon.smithy.python.aws.codegen.AwsUserAgentIntegration
1010
software.amazon.smithy.python.aws.codegen.AwsStandardRegionalEndpointsIntegration
11-
software.amazon.smithy.python.aws.codegen.AwsDocConverter
1211
software.amazon.smithy.python.aws.codegen.AwsRstDocFileGenerator

‎codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/AwsDocConverterTest.java‎ renamed to ‎codegen/aws/core/src/test/java/software/amazon/smithy/python/aws/codegen/MarkdownToRstDocConverterTest.java‎

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,101 +8,102 @@
88

99
import org.junit.jupiter.api.BeforeEach;
1010
import org.junit.jupiter.api.Test;
11+
import software.amazon.smithy.python.codegen.writer.MarkdownToRstDocConverter;
1112

12-
public class AwsDocConverterTest {
13+
public class MarkdownToRstDocConverterTest {
1314

14-
private AwsDocConverter awsDocConverter;
15+
private MarkdownToRstDocConverter markdownToRstDocConverter;
1516

1617
@BeforeEach
1718
public void setUp() {
18-
awsDocConverter = new AwsDocConverter();
19+
markdownToRstDocConverter = new MarkdownToRstDocConverter();
1920
}
2021

2122
@Test
22-
public void testConvertHtmlToRstWithTitleAndParagraph() {
23+
public void testConvertCommonmarkToRstWithTitleAndParagraph() {
2324
String html = "<html><body><h1>Title</h1><p>Paragraph</p></body></html>";
2425
String expected = "\n\nTitle\n=====\nParagraph\n";
25-
String result = awsDocConverter.convertHtmlToRst(html);
26+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
2627
assertEquals(expected, result);
2728
}
2829

2930
@Test
30-
public void testConvertHtmlToRstWithImportantNote() {
31+
public void testConvertCommonmarkToRstWithImportantNote() {
3132
String html = "<html><body><important>Important note</important></body></html>";
3233
String expected = "\n\n.. important::\n Important note\n";
33-
String result = awsDocConverter.convertHtmlToRst(html);
34+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
3435
assertEquals(expected, result);
3536
}
3637

3738
@Test
38-
public void testConvertHtmlToRstWithList() {
39+
public void testConvertCommonmarkToRstWithList() {
3940
String html = "<html><body><ul><li>Item 1</li><li>Item 2</li></ul></body></html>";
4041
String expected = "\n\n* Item 1\n\n* Item 2\n\n";
41-
String result = awsDocConverter.convertHtmlToRst(html);
42+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
4243
assertEquals(expected, result);
4344
}
4445

4546
@Test
46-
public void testConvertHtmlToRstWithMixedElements() {
47+
public void testConvertCommonmarkToRstWithMixedElements() {
4748
String html = "<html><body><h1>Title</h1><p>Paragraph</p><ul><li>Item 1</li><li>Item 2</li></ul></body></html>";
4849
String expected = "\n\nTitle\n=====\nParagraph\n\n* Item 1\n\n* Item 2\n\n";
49-
String result = awsDocConverter.convertHtmlToRst(html);
50+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
5051
assertEquals(expected, result);
5152
}
5253

5354
@Test
54-
public void testConvertHtmlToRstWithNestedElements() {
55+
public void testConvertCommonmarkToRstWithNestedElements() {
5556
String html = "<html><body><h1>Title</h1><p>Paragraph with <strong>bold</strong> text</p></body></html>";
5657
String expected = "\n\nTitle\n=====\nParagraph with **bold** text\n";
57-
String result = awsDocConverter.convertHtmlToRst(html);
58+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
5859
assertEquals(expected, result);
5960
}
6061

6162
@Test
62-
public void testConvertHtmlToRstWithAnchorTag() {
63+
public void testConvertCommonmarkToRstWithAnchorTag() {
6364
String html = "<html><body><a href='https://example.com'>Link</a></body></html>";
6465
String expected = "\n`Link <https://example.com>`_";
65-
String result = awsDocConverter.convertHtmlToRst(html);
66+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
6667
assertEquals(expected, result);
6768
}
6869

6970
@Test
70-
public void testConvertHtmlToRstWithBoldTag() {
71+
public void testConvertCommonmarkToRstWithBoldTag() {
7172
String html = "<html><body><b>Bold text</b></body></html>";
7273
String expected = "\n**Bold text**";
73-
String result = awsDocConverter.convertHtmlToRst(html);
74+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
7475
assertEquals(expected, result);
7576
}
7677

7778
@Test
78-
public void testConvertHtmlToRstWithItalicTag() {
79+
public void testConvertCommonmarkToRstWithItalicTag() {
7980
String html = "<html><body><i>Italic text</i></body></html>";
8081
String expected = "\n*Italic text*";
81-
String result = awsDocConverter.convertHtmlToRst(html);
82+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
8283
assertEquals(expected, result);
8384
}
8485

8586
@Test
86-
public void testConvertHtmlToRstWithCodeTag() {
87+
public void testConvertCommonmarkToRstWithCodeTag() {
8788
String html = "<html><body><code>code snippet</code></body></html>";
8889
String expected = "\n``code snippet``";
89-
String result = awsDocConverter.convertHtmlToRst(html);
90+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
9091
assertEquals(expected, result);
9192
}
9293

9394
@Test
94-
public void testConvertHtmlToRstWithNoteTag() {
95+
public void testConvertCommonmarkToRstWithNoteTag() {
9596
String html = "<html><body><note>Note text</note></body></html>";
9697
String expected = "\n\n.. note::\n Note text\n";
97-
String result = awsDocConverter.convertHtmlToRst(html);
98+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
9899
assertEquals(expected, result);
99100
}
100101

101102
@Test
102-
public void testConvertHtmlToRstWithNestedList() {
103+
public void testConvertCommonmarkToRstWithNestedList() {
103104
String html = "<html><body><ul><li>Item 1<ul><li>Subitem 1</li></ul></li><li>Item 2</li></ul></body></html>";
104105
String expected = "\n\n* Item 1\n * Subitem 1\n\n* Item 2\n\n";
105-
String result = awsDocConverter.convertHtmlToRst(html);
106+
String result = markdownToRstDocConverter.convertCommonmarkToRst(html);
106107
assertEquals(expected, result);
107108
}
108109
}

‎codegen/core/build.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,6 @@ dependencies {
1515
implementation(libs.smithy.protocol.test.traits)
1616
// We have this because we're using RestJson1 as a 'generic' protocol.
1717
implementation(libs.smithy.aws.traits)
18+
implementation(libs.jsoup)
19+
implementation(libs.commonmark)
1820
}

‎codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/SetupGenerator.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import java.util.function.Function;
1515
import java.util.stream.Collectors;
1616
import java.util.stream.Stream;
17+
18+
import software.amazon.smithy.aws.traits.ServiceTrait;
1719
import software.amazon.smithy.codegen.core.CodegenException;
1820
import software.amazon.smithy.codegen.core.SymbolDependency;
1921
import software.amazon.smithy.codegen.core.WriterDelegator;
@@ -270,6 +272,8 @@ private static void writeDocsSkeleton(
270272
PythonSettings settings,
271273
GenerationContext context
272274
) {
275+
//TODO Add a configurable flag to disable the generation of the sphinx files
276+
//TODO Add a configuration that will allow users to select a sphinx theme
273277
context.writerDelegator().useFileWriter("pyproject.toml", "", writer -> {
274278
writer.addDependency(SmithyPythonDependency.SPHINX);
275279
writer.addDependency(SmithyPythonDependency.SPHINX_PYDATA_THEME);
@@ -291,14 +295,18 @@ private static void writeConf(
291295
PythonSettings settings,
292296
GenerationContext context
293297
) {
298+
var service = context.model().expectShape(settings.service());
294299
String version = settings.moduleVersion();
300+
String project = service.getTrait(TitleTrait.class)
301+
.map(StringTrait::getValue)
302+
.orElse(service.getTrait(ServiceTrait.class).get().getSdkId());
295303
context.writerDelegator().useFileWriter("docs/conf.py", "", writer -> {
296304
writer.write("""
297305
import os
298306
import sys
299307
sys.path.insert(0, os.path.abspath('..'))
300308
301-
project = 'AWS SDK for Python'
309+
project = '$L'
302310
author = 'Boto'
303311
release = '$L'
304312
@@ -322,7 +330,7 @@ private static void writeConf(
322330
}
323331
324332
autodoc_typehints = 'both'
325-
""", version);
333+
""", project, version);
326334
});
327335
}
328336

‎codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -317,8 +317,7 @@ private boolean hasDocs() {
317317
private void writeMemberDocs(MemberShape member) {
318318
member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> {
319319
String memberName = symbolProvider.toMemberName(member);
320-
String docs = writer.formatDocs(String.format(":param %s: %s", memberName, trait.getValue()))
321-
.replace("\n", "\n ");
320+
String docs = writer.formatDocs(String.format(":param %s: %s", memberName, trait.getValue()));
322321
writer.write(docs);
323322
});
324323
}

‎codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/UnionGenerator.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ raise NotImplementedError()
133133
writer.popState();
134134

135135
writer.pushState(new UnionSection(shape, parentName, memberNames));
136+
// We need to use the old union syntax until we either migrate away from
137+
// Sphinx or Sphinx fixes the issue upstream: https://github.com/sphinx-doc/sphinx/issues/10785
136138
writer.write("$L = Union[$L]\n", parentName, String.join(" | ", memberNames));
137139
shape.getTrait(DocumentationTrait.class).ifPresent(trait -> writer.writeDocs(trait.getValue()));
138140
writer.popState();

‎codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsDocConverter.java‎ renamed to ‎codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/MarkdownToRstDocConverter.java‎

Lines changed: 39 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,53 +2,62 @@
22
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
33
* SPDX-License-Identifier: Apache-2.0
44
*/
5-
package software.amazon.smithy.python.aws.codegen;
5+
package software.amazon.smithy.python.codegen.writer;
66

77
import org.jsoup.Jsoup;
88
import org.jsoup.nodes.Document;
99
import org.jsoup.nodes.Element;
1010
import org.jsoup.nodes.Node;
1111
import org.jsoup.nodes.TextNode;
1212
import org.jsoup.select.NodeVisitor;
13-
import software.amazon.smithy.model.Model;
14-
import software.amazon.smithy.model.shapes.Shape;
15-
import software.amazon.smithy.model.traits.DocumentationTrait;
16-
import software.amazon.smithy.model.transform.ModelTransformer;
17-
import software.amazon.smithy.python.codegen.PythonSettings;
18-
import software.amazon.smithy.python.codegen.integrations.PythonIntegration;
13+
import software.amazon.smithy.utils.SetUtils;
1914
import software.amazon.smithy.utils.SmithyInternalApi;
15+
import org.commonmark.node.BlockQuote;
16+
import org.commonmark.node.FencedCodeBlock;
17+
import org.commonmark.node.Heading;
18+
import org.commonmark.node.HtmlBlock;
19+
import org.commonmark.node.ListBlock;
20+
import org.commonmark.node.ThematicBreak;
21+
import org.commonmark.parser.Parser;
22+
import org.commonmark.renderer.html.HtmlRenderer;
23+
24+
import static org.jsoup.nodes.Document.OutputSettings.Syntax.html;
2025

2126
/**
22-
* Add a runtime plugin to convert the HTML docs that are provided by services into
23-
* RST
27+
* Add a runtime plugin to convert the HTML docs that are provided by services into RST
2428
*/
2529
@SmithyInternalApi
26-
public class AwsDocConverter implements PythonIntegration {
27-
@Override
28-
public Model preprocessModel(Model model, PythonSettings settings) {
29-
return ModelTransformer.create().mapShapes(model, shape -> {
30-
if (shape.hasTrait(DocumentationTrait.class)) {
31-
DocumentationTrait docTrait = shape.getTrait(DocumentationTrait.class).get();
32-
String html = docTrait.getValue();
33-
String rst = convertHtmlToRst(html);
34-
DocumentationTrait newDocTrait = new DocumentationTrait(rst);
35-
return Shape.shapeToBuilder(shape)
36-
.addTrait(newDocTrait)
37-
.build();
38-
} else {
39-
return shape;
40-
}
41-
});
30+
public class MarkdownToRstDocConverter {
31+
private static final Parser MARKDOWN_PARSER = Parser.builder()
32+
.enabledBlockTypes(SetUtils.of(
33+
Heading.class, HtmlBlock.class, ThematicBreak.class, FencedCodeBlock.class,
34+
BlockQuote.class, ListBlock.class))
35+
.build();
36+
37+
// Singleton instance
38+
private static final MarkdownToRstDocConverter DOC_CONVERTER = new MarkdownToRstDocConverter();
39+
40+
// Private constructor to prevent instantiation
41+
private MarkdownToRstDocConverter() {
42+
// Constructor
4243
}
4344

44-
String convertHtmlToRst(String html) {
45-
Document document = Jsoup.parse(html);
45+
public static MarkdownToRstDocConverter getInstance() {
46+
return DOC_CONVERTER;
47+
}
48+
49+
50+
public String convertCommonmarkToRst(String commonmark) {
51+
String html =
52+
HtmlRenderer.builder().escapeHtml(false).build().render(MARKDOWN_PARSER.parse(commonmark));
53+
Document document = Jsoup.parse(commonmark);
4654
RstNodeVisitor visitor = new RstNodeVisitor();
4755
document.body().traverse(visitor);
4856
return "\n" + visitor;
4957
}
5058

5159
private static class RstNodeVisitor implements NodeVisitor {
60+
//TODO migrate away from StringBuilder to use a SimpleCodeWriter
5261
private final StringBuilder sb = new StringBuilder();
5362
private boolean inList = false;
5463
private int listDepth = 0;
@@ -127,7 +136,8 @@ public void tail(Node node, int depth) {
127136
sb.append("``");
128137
break;
129138
case "important":
130-
case "note", "p":
139+
case "note":
140+
case "p":
131141
sb.append("\n");
132142
break;
133143
case "ul":
@@ -159,4 +169,4 @@ public String toString() {
159169
return sb.toString();
160170
}
161171
}
162-
}
172+
}

‎codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonWriter.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
public final class PythonWriter extends SymbolWriter<PythonWriter, ImportDeclarations> {
4040

4141
private static final Logger LOGGER = Logger.getLogger(PythonWriter.class.getName());
42+
private static final MarkdownToRstDocConverter DOC_CONVERTER = MarkdownToRstDocConverter.getInstance();
4243

4344
private final String fullPackageName;
4445
private final String commentStart;
@@ -152,8 +153,8 @@ public PythonWriter writeDocs(String docs) {
152153
* @return Formatted documentation.
153154
*/
154155
public String formatDocs(String docs) {
155-
String rstDocs = wrapRST(docs).toString();
156-
return rstDocs.replace("$", "$$");
156+
String rstDocs = DOC_CONVERTER.convertCommonmarkToRst(docs);
157+
return wrapRST(rstDocs).toString().replace("$", "$$");
157158
}
158159

159160
public static String wrapRST(String text) {

0 commit comments

Comments
 (0)