Fix doc attribute parsing to properly handle block comments

The existing parsing logic is flawed, as it concatenates consecutive
lines, e.g. this:

/**
This is the first paragraph.
*/

...becomes this:

/**
 * # Heading This is the first paragraph.
*/

Lists are also effected.

Author: sd-collins

src/bindgen/utilities.rs

@@ -425,13 +425,38 @@ impl SynAttributeHelpers for [syn::Attribute] {
 }
 
 fn split_doc_attr(input: &str) -> Vec<String> {
+    if !input.contains('\n') {
+        // This is a special case for single-line doc comments, which normally already contain a leading space
+        // if it is desired.
+        return vec![input.to_owned()];
+    }
+
+    // Calculate the common leading whitespace across all non-empty lines, so we can trim it from all lines while
+    // preserving relative indentation. This is important for items nested (esp. in modules) where the doc comment
+    // is usually indented to the same level as the item, leaving whitespace at the beginning of each line.
+    // We want to trim that, but preserve relative indentation.
+    // Note: we assume you aren't using mixed tabs and spaces, but that is probably safe to assume for rust code
+    // which is usually indented with spaces.
+    let common_indent = input
+        .trim()
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| line.chars().take_while(|c| c.is_whitespace()).count())
+        .min()
+        .unwrap_or(0);
+
     input
-        // Convert two newline (indicate "new paragraph") into two line break.
-        .replace("\n\n", "  \n  \n")
-        // Convert newline after two spaces (indicate "line break") into line break.
-        .split("  \n")
-        // Convert single newline (indicate hard-wrapped) into space.
-        .map(|s| s.replace('\n', " "))
-        .map(|s| s.trim_end().to_string())
+        // Trim leading and trailing whitespace (first and last lines are usually empty)
+        .trim()
+        .lines()
+        // Add a leading space to non-empty lines to prevent misinterpreting leading symbols and
+        // mirror the behaviour of single-line doc comments, which already have a leading space.
+        .map(|s| {
+            if s.trim().is_empty() {
+                String::new()
+            } else {
+                format!(" {}", s.chars().skip(common_indent).collect::<String>())
+            }
+        })
         .collect()
 }

tests/expectations-symbols/documentation_block.c.sym

@@ -0,0 +1,5 @@
+{
+root;
+block_function;
+FOO;
+};

tests/expectations/documentation_attr.c

@@ -8,13 +8,14 @@
  *like this one with a new line character at its end
  *and this one as well. So they are in the same paragraph
  *
- *Line ends with one new line should not break
+ *We treat empty doc comments as empty lines, so they break to the next paragraph.
  *
- *Line ends with two spaces and a new line
- *should break to next line
+ * Newlines are preserved with leading spaces added
+ * to prettify and avoid misinterpreting leading symbols.
+ *like headings and lists.
  *
- *Line ends with two new lines
+ * Line ends with two new lines
  *
- *Should break to next paragraph
+ * Should break to next paragraph
  */
 void root(void);

tests/expectations/documentation_attr.compat.c

@@ -12,14 +12,15 @@ extern "C" {
  *like this one with a new line character at its end
  *and this one as well. So they are in the same paragraph
  *
- *Line ends with one new line should not break
+ *We treat empty doc comments as empty lines, so they break to the next paragraph.
  *
- *Line ends with two spaces and a new line
- *should break to next line
+ * Newlines are preserved with leading spaces added
+ * to prettify and avoid misinterpreting leading symbols.
+ *like headings and lists.
  *
- *Line ends with two new lines
+ * Line ends with two new lines
  *
- *Should break to next paragraph
+ * Should break to next paragraph
  */
 void root(void);
 

tests/expectations/documentation_attr.cpp

@@ -10,14 +10,15 @@ extern "C" {
 ///like this one with a new line character at its end
 ///and this one as well. So they are in the same paragraph
 ///
-///Line ends with one new line should not break
+///We treat empty doc comments as empty lines, so they break to the next paragraph.
 ///
-///Line ends with two spaces and a new line
-///should break to next line
+/// Newlines are preserved with leading spaces added
+/// to prettify and avoid misinterpreting leading symbols.
+///like headings and lists.
 ///
-///Line ends with two new lines
+/// Line ends with two new lines
 ///
-///Should break to next paragraph
+/// Should break to next paragraph
 void root();
 
 }  // extern "C"

tests/expectations/documentation_attr.pyx

@@ -10,12 +10,13 @@ cdef extern from *:
   #like this one with a new line character at its end
   #and this one as well. So they are in the same paragraph
   #
-  #Line ends with one new line should not break
+  #We treat empty doc comments as empty lines, so they break to the next paragraph.
   #
-  #Line ends with two spaces and a new line
-  #should break to next line
+  # Newlines are preserved with leading spaces added
+  # to prettify and avoid misinterpreting leading symbols.
+  #like headings and lists.
   #
-  #Line ends with two new lines
+  # Line ends with two new lines
   #
-  #Should break to next paragraph
+  # Should break to next paragraph
   void root();

tests/expectations/documentation_block.c

@@ -0,0 +1,48 @@
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/**
+ * Some docs.
+ */
+extern const uint32_t FOO;
+
+/**
+ * The root of all evil.
+ *
+ * But at least it contains some more documentation as someone would expect
+ * from a simple test case like this.
+ *
+ * # Hint
+ * Always ensure that everything is properly documented, even if you feel lazy.
+ * **Sometimes** it is also helpful to include some markdown formatting.
+ *
+ * ////////////////////////////////////////////////////////////////////////////
+ *
+ * Attention:
+ *
+ *    This is an indentation test.
+ *    The indentation should be preserved in the generated documentation.
+ *
+ * ...and here is my shopping list to check that we do not mess with line breaks and indentation:
+ * - Bread
+ *    - Brown
+ *    - White
+ * - Milk
+ * - Eggs
+ */
+void root(void);
+
+/**
+ * In this block, we're testing indentation handling.
+ *     Since all of these lines are equally indented, we want to discard the common leading whitespace,
+ *        but preserve the relative indentation and line breaks.
+ *
+ *        Including between paragraphs,
+ *
+ *     - And
+ *       - within
+ *       - Lists
+ */
+void block_function(void);

tests/expectations/documentation_block.compat.c

@@ -0,0 +1,56 @@
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+
+/**
+ * Some docs.
+ */
+extern const uint32_t FOO;
+
+/**
+ * The root of all evil.
+ *
+ * But at least it contains some more documentation as someone would expect
+ * from a simple test case like this.
+ *
+ * # Hint
+ * Always ensure that everything is properly documented, even if you feel lazy.
+ * **Sometimes** it is also helpful to include some markdown formatting.
+ *
+ * ////////////////////////////////////////////////////////////////////////////
+ *
+ * Attention:
+ *
+ *    This is an indentation test.
+ *    The indentation should be preserved in the generated documentation.
+ *
+ * ...and here is my shopping list to check that we do not mess with line breaks and indentation:
+ * - Bread
+ *    - Brown
+ *    - White
+ * - Milk
+ * - Eggs
+ */
+void root(void);
+
+/**
+ * In this block, we're testing indentation handling.
+ *     Since all of these lines are equally indented, we want to discard the common leading whitespace,
+ *        but preserve the relative indentation and line breaks.
+ *
+ *        Including between paragraphs,
+ *
+ *     - And
+ *       - within
+ *       - Lists
+ */
+void block_function(void);
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus

tests/expectations/documentation_block.cpp

@@ -0,0 +1,47 @@
+#include <cstdarg>
+#include <cstdint>
+#include <cstdlib>
+#include <ostream>
+#include <new>
+
+extern "C" {
+
+/// Some docs.
+extern const uint32_t FOO;
+
+/// The root of all evil.
+///
+/// But at least it contains some more documentation as someone would expect
+/// from a simple test case like this.
+///
+/// # Hint
+/// Always ensure that everything is properly documented, even if you feel lazy.
+/// **Sometimes** it is also helpful to include some markdown formatting.
+///
+/// ////////////////////////////////////////////////////////////////////////////
+///
+/// Attention:
+///
+///    This is an indentation test.
+///    The indentation should be preserved in the generated documentation.
+///
+/// ...and here is my shopping list to check that we do not mess with line breaks and indentation:
+/// - Bread
+///    - Brown
+///    - White
+/// - Milk
+/// - Eggs
+void root();
+
+/// In this block, we're testing indentation handling.
+///     Since all of these lines are equally indented, we want to discard the common leading whitespace,
+///        but preserve the relative indentation and line breaks.
+///
+///        Including between paragraphs,
+///
+///     - And
+///       - within
+///       - Lists
+void block_function();
+
+}  // extern "C"

tests/expectations/documentation_block.pyx

@@ -0,0 +1,45 @@
+from libc.stdint cimport int8_t, int16_t, int32_t, int64_t, intptr_t
+from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, uintptr_t
+cdef extern from *:
+  ctypedef bint bool
+  ctypedef struct va_list
+
+cdef extern from *:
+
+  # Some docs.
+  extern const uint32_t FOO;
+
+  # The root of all evil.
+  #
+  # But at least it contains some more documentation as someone would expect
+  # from a simple test case like this.
+  #
+  # # Hint
+  # Always ensure that everything is properly documented, even if you feel lazy.
+  # **Sometimes** it is also helpful to include some markdown formatting.
+  #
+  # ////////////////////////////////////////////////////////////////////////////
+  #
+  # Attention:
+  #
+  #    This is an indentation test.
+  #    The indentation should be preserved in the generated documentation.
+  #
+  # ...and here is my shopping list to check that we do not mess with line breaks and indentation:
+  # - Bread
+  #    - Brown
+  #    - White
+  # - Milk
+  # - Eggs
+  void root();
+
+  # In this block, we're testing indentation handling.
+  #     Since all of these lines are equally indented, we want to discard the common leading whitespace,
+  #        but preserve the relative indentation and line breaks.
+  #
+  #        Including between paragraphs,
+  #
+  #     - And
+  #       - within
+  #       - Lists
+  void block_function();