Replace Pegdown with Commonmark (#366)

Pegdown is no longer maintained, which will cause problems when upgrading build
pipeline to Java 17. Also it's nice to use an implemenntation of Markdown that
is compatible with GH previews and IDE plugins -- commonmark seems to do quite
well.

Three differences had to be handled in this PR though:
 * code snippets are now delimited by backticks since indented snippets behaved
   differently in Pegdown and Commonmark.
 * Commonmark does not support typography replacements (-- to en-dash, --- to 
   em-dash, quotes) out of the box, HTML entities can be used instead.
 * no support for special definition list MD extension (only used by
   DevGuideI18nMessages) -- rewritten to HTML tags

Fixes #332

Author: zbynek

pom.xml

@@ -131,11 +131,18 @@
     </dependency>
 
     <dependency>
-      <groupId>org.pegdown</groupId>
-      <artifactId>pegdown</artifactId>
-      <version>1.4.2</version>
+      <groupId>org.commonmark</groupId>
+      <artifactId>commonmark</artifactId>
+      <!-- last version supporting Java 8 -->
+      <version>0.21.0</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.commonmark</groupId>
+      <artifactId>commonmark-ext-gfm-tables</artifactId>
+      <!-- last version supporting Java 8 -->
+      <version>0.21.0</version>
+    </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>

src/main/java/com/google/gwt/site/markdown/MDHelper.java

@@ -15,8 +15,6 @@
 
 import java.io.File;
 
-import org.pegdown.PegDownProcessor;
-
 import com.google.gwt.site.markdown.fs.FileSystemTraverser;
 import com.google.gwt.site.markdown.fs.MDParent;
 import com.google.gwt.site.markdown.toc.TocCreator;
@@ -107,7 +105,7 @@ public MDHelper create() throws MDHelperException {
 
     // read template TOC if parameter is provided
     if (templateTocFile != null) {
-      templateToc = new PegDownProcessor().markdownToHtml(readFile(templateTocFile));
+      templateToc = MDTranslater.markDownToHtml(readFile(templateTocFile));
     }
 
     created = true;

src/main/java/com/google/gwt/site/markdown/MDTranslater.java

@@ -17,23 +17,21 @@
 import com.google.gwt.site.markdown.fs.MDParent;
 import com.google.gwt.site.markdown.toc.TocCreator;
 
-import org.apache.commons.io.FileUtils;
-import org.pegdown.Extensions;
-import org.pegdown.PegDownProcessor;
+import org.commonmark.Extension;
+import org.commonmark.ext.gfm.tables.TablesExtension;
+import org.commonmark.node.*;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.html.HtmlRenderer;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.FileSystem;
-import java.nio.file.FileSystems;
+import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 public class MDTranslater {
-  private static final int PEG_DOWN_FLAGS = Extensions.SMARTYPANTS | Extensions.AUTOLINKS |
-      Extensions.FENCED_CODE_BLOCKS | Extensions.TABLES | Extensions.DEFINITIONS;
   private static final String SEPARATOR = File.separator;
 
-  private PegDownProcessor pegDownProcessor = new PegDownProcessor(PEG_DOWN_FLAGS, Long
-      .MAX_VALUE);
 
   private final TocCreator tocCreator;
 
@@ -63,7 +61,7 @@ private void renderTree(MDNode node, MDParent root) throws TranslaterException {
 
     } else {
       String markDown = getNodeContent(node.getPath());
-      String htmlMarkDown = pegDownProcessor.markdownToHtml(markDown);
+      String htmlMarkDown = markDownToHtml(markDown);
 
       String toc = tocCreator.createTocForNode(root, node);
 
@@ -86,6 +84,15 @@ private void renderTree(MDNode node, MDParent root) throws TranslaterException {
 
   }
 
+  protected static String markDownToHtml(String markDown) {
+    Set<Extension> extensions = Collections.singleton(TablesExtension.create());
+    Parser parser = Parser.builder()
+        .extensions(extensions).build();
+    Node document = parser.parse(markDown);
+    HtmlRenderer renderer = HtmlRenderer.builder().extensions(extensions).build();
+    return renderer.render(document);
+  }
+
   private String getEditUrl(String path) {
     // TODO you should support more than one template
     if (path.endsWith("markdown/index.md")) {

src/main/java/com/google/gwt/site/markdown/Util.java

@@ -19,13 +19,14 @@
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 
 public class Util {
   public static String getStringFromFile(File file) throws IOException {
     FileInputStream fileInputStream = null;
     try {
       fileInputStream = new FileInputStream(file);
-      return IOUtils.toString(fileInputStream, "UTF-8");
+      return IOUtils.toString(fileInputStream, StandardCharsets.UTF_8);
     } finally {
       IOUtils.closeQuietly(fileInputStream);
     }
@@ -35,7 +36,7 @@ public static void writeStringToFile(File file, String content) throws IOExcepti
     FileOutputStream fileOutputStream = null;
     try {
       fileOutputStream = new FileOutputStream(file);
-      IOUtils.write(content, fileOutputStream);
+      IOUtils.write(content, fileOutputStream, StandardCharsets.UTF_8);
     } finally {
       IOUtils.closeQuietly(fileOutputStream);
     }

src/main/java/com/google/gwt/site/markdown/toc/TocFromMdCreator.java

@@ -16,10 +16,9 @@
 import com.google.gwt.site.markdown.fs.MDNode;
 import com.google.gwt.site.markdown.fs.MDParent;
 
+import java.util.Arrays;
 import java.util.List;
 
-import org.parboiled.common.StringUtils;
-
 public class TocFromMdCreator implements TocCreator {
 
   public String createTocForNode(MDParent root, MDNode node) {
@@ -49,17 +48,17 @@ private void render(MDNode node, StringBuffer buffer, MDNode tocNode) {
     }
 
     // Use 4 spaces to indent <li>'s, so as we have room for indenting <ul>'s
-    String margin = StringUtils.repeat(' ', 4 * node.getDepth());
+    String margin = spaces(4 * node.getDepth());
 
     if (node.isFolder()) {
       MDParent mdParent = node.asFolder();
 
       if (node.getDepth() != 0) {
-        buffer.append(margin + "<li class='folder'>");
+        buffer.append(margin).append("<li class='folder'>");
         buffer.append("<a href='#'>");
         buffer.append(node.getDisplayName());
         buffer.append("</a>\n");
-        buffer.append(margin + "  <ul>\n");
+        buffer.append(margin).append("  <ul>\n");
       }
 
       List<MDNode> children = mdParent.getChildren();
@@ -68,8 +67,8 @@ private void render(MDNode node, StringBuffer buffer, MDNode tocNode) {
       }
 
       if (node.getDepth() != 0) {
-        buffer.append(margin + "  </ul>\n");
-        buffer.append(margin + "</li>\n");
+        buffer.append(margin).append("  </ul>\n");
+        buffer.append(margin).append("</li>\n");
       }
     } else {
       StringBuffer relativeUrl = new StringBuffer();
@@ -85,11 +84,19 @@ private void render(MDNode node, StringBuffer buffer, MDNode tocNode) {
 
       relativeUrl.append(node.getRelativePath());
 
-      buffer.append(margin + "<li class='file'>");
+      buffer.append(margin).append("<li class='file'>");
       // TODO escape HTML
-      buffer.append("<a href='" + relativeUrl.toString() + "' ahref='" + absoluteUrl.toString()
-          + "' title='" + node.getDescription() + "'>" + node.getDisplayName() + "</a>");
+      buffer.append("<a href='").append(relativeUrl)
+          .append("' ahref='").append(absoluteUrl)
+          .append("' title='").append(node.getDescription()).append("'>")
+          .append(node.getDisplayName()).append("</a>");
       buffer.append("</li>\n");
     }
   }
+
+  private String spaces(int count) {
+    final byte[] spaceBytes = new byte[count];
+    Arrays.fill(spaceBytes, (byte) ' ');
+    return new String(spaceBytes);
+  }
 }

src/main/markdown/articles/dom_events_memory_leaks_and_you.md

@@ -9,7 +9,7 @@ You may ask yourself, "Why do I have to use bitfields to sink DOM events?", and
 
 If you're creating a widget from scratch (using DOM elements directly, as opposed to simply creating a composite widget), the setup for event handling generally looks something like this:
 
-```
+```java
 class MyWidget extends Widget {
   public MyWidget() {
     setElement(DOM.createDiv());
@@ -35,7 +35,7 @@ The upshot of all this is that in some browsers, any reference cycle that involv
 
 Imagine the following (raw JavaScript) example:
 
-```
+```javascript
 function makeWidget() {
   var widget = {};
   widget.someVariable = "foo";
@@ -48,7 +48,7 @@ function makeWidget() {
 
 Now, I'm not suggesting that you'd really build a JavaScript library quite this way, but it serves to illustrate the point. The reference cycle created here is:
 
-```
+```text
 widget -> elem(native) -> closure -> widget
 
 ```
@@ -68,7 +68,7 @@ How do we enforce this? Each widget has a single "root" element. Whenever the wi
 
 Which brings is back to that odd bitfield used in the sinkEvents() method. If you look at the implementation of [DOM.sinkEvents()](/javadoc/latest/com/google/gwt/user/client/DOM.html#sinkEvents-com.google.gwt.user.client.Element-int-), you'll see that it does something like this:
 
-```
+```javascript
 elem.onclick = (bits & 0x00001) ? $wnd.__dispatchEvent : null;
 
 ```

src/main/markdown/articles/dynamic_host_page.md

@@ -20,7 +20,7 @@ service in the onModuleLoad() method of your EntryPoint class to check
 if the user is logged in. This initiates a GWT-RPC request as soon as
 the GWT module loads.
 
-```
+```java
 public void onModuleLoad() {
   // loginService is a GWT-RPC service that checks if the user is logged in
   loginService.checkLoggedIn(new AsyncCallback<Boolean> {
@@ -41,7 +41,7 @@ Let's examine everything that happens here if the user isn't logged in:
 1.  Your app is requested and your GWT host page (YourModule.html) is downloaded
 2.  <var>module</var>.nocache.js is requested by the page and is downloaded
 3.  <var>MD5</var>.cache.html is selected based on the browser and is downloaded
-4.  Your module loads and makes a GWT-RPC call to check if the user is logged in -- since they're not, they are redirected to the login page
+4.  Your module loads and makes a GWT-RPC call to check if the user is logged in &mdash; since they're not, they are redirected to the login page
 
 That's up to **four** server requests (depending on what is cached)
 just to send your user to the login page. And step 3 consists of
@@ -70,7 +70,7 @@ for custom authentication schemes and the ability to vary the content of
 the host page based on the user. Here's an example of a simple host page
 written as a servlet:
 
-```
+```java
 public class GwtHostingServlet extends HttpServlet {
 
  @Override
@@ -99,7 +99,7 @@ The following example uses the App Engine
 user is logged in. Even if you're not using App Engine, you can imagine
 how the code would look slightly different in your servlet environment.
 
-```
+```java
 // In GwtHostingServlet's doGet() method...
 PrintWriter writer = resp.getWriter();
 writer.append("<html><head>");
@@ -133,7 +133,7 @@ GWT module, then immediately making another request to get this data. A
 more efficient way is to write the initial data as a Javascript variable
 into the host page itself.
 
-```
+```java
 // In GwtHostingServlet's doGet() method...
 writer.append("<html><head>");
 writer.append("<script type=\"text/javascript\" src=\"sample/sample.nocache.js\"></script>");
@@ -157,7 +157,7 @@ writer.append("</head><body>Hello, world!</body></html>");
 
 Now your GWT code can access the data using JSNI, like so:
 
-```
+```java
 public native String getEmail() /*-{
   return $wnd.info['email'];
 }-*/;
@@ -166,7 +166,7 @@ public native String getEmail() /*-{
 Alternatively, you can take advantage of GWT's
 [Dictionary](/javadoc/latest/com/google/gwt/i18n/client/Dictionary.html) class:
 
-```
+```java
 public void onModuleLoad() {
   // Looks for a JS variable called "info" in the global scope
   Dictionary info = Dictionary.getDictionary("info");
@@ -182,7 +182,7 @@ worthwhile to consider using a templating language like JSP to make your
 code more readable. Here's our example as a JSP page instead of a
 servlet:
 
-```
+```jsp
 <!-- gwt-hosting.jsp -->
 <html>
  <head>
@@ -212,7 +212,7 @@ servlet:
 
 You can make this JSP page your welcome file by specifying it in your web.xml file:
 
-```
+```xml
 <welcome-file-list>
   <welcome-file>gwt-hosting.jsp</welcome-file>
 </welcome-file-list>

src/main/markdown/articles/elemental.md

@@ -33,7 +33,7 @@ Elemental uses Java interfaces to hide most of the generated overlay types it us
 can be obtained by the `Window` or `Document` interfaces. Here is a simple example to play a sound using
 the Web Audio API.
 
-```
+```java
 package com.myapp;
 import elemental.client.*;
 import elemental.dom.*;