issue doxygen#11621 Markdown code handling ` gets confused by unterminated ` within (introduced by 1.11)

Fixed for escaped backtick

Author: doxygen

doc/autolink.dox

@@ -18,7 +18,7 @@
 
   \tableofcontents{html,latex}
 
-  Most documentation systems have special 'see also' sections where links
+  Most documentation systems have special `see also' sections where links
   to other pieces of documentation can be inserted.
   Although Doxygen also has a command to start such a section (See section
   \ref cmdsa "\\sa"), it does allow you to put these kind of links anywhere in the

doc/features.dox

@@ -101,7 +101,7 @@ Although Doxygen can now be used in any project written in a language that is
 supported by Doxygen, initially it was specifically designed to be used for projects 
 that make use of Qt Software's 
 <A HREF="https://doc.qt.io">Qt toolkit</A>. I have tried to 
-make Doxygen 'Qt-compatible'. That is: Doxygen can read the documentation contained in 
+make Doxygen `Qt-compatible'. That is: Doxygen can read the documentation contained in 
 the Qt source code and create a class browser that looks quite similar to the 
 one that is generated by Qt Software. Doxygen understands the C++ extensions 
 used by Qt such as signals and slots and many of the markup commands used in the Qt sources.

src/commentcnv.l

@@ -616,6 +616,9 @@ SLASHopt [/]*
                                      yyextra->verbatimLine=yyextra->lineNr;
                                      BEGIN(VerbatimCode);
   				   }
+<CComment,ReadLine,IncludeFile>"\\`" {
+                                     copyToOutput(yyscanner,yytext,yyleng);
+                                   }
 <CComment,ReadLine,IncludeFile>"```" { // skip ``` if not at the start of the line
                                      copyToOutput(yyscanner,yytext,yyleng);
                                    }
@@ -740,7 +743,7 @@ SLASHopt [/]*
 				     }
   				   }
 <VerbatimCode>("```"[`]*|"~~~"[~]*) { /* end of markdown code block */
-                                     if (yyextra->blockName=="`" || yyextra->blockName=="``") // ``` inside single quote block
+                                     if (yytext[0]=='`' && (yyextra->blockName=="`" || yyextra->blockName=="``")) // ``` inside single quote block
                                      {
                                        // copy the end block marker
                                        copyToOutput(yyscanner,yytext,yyleng);
@@ -757,6 +760,14 @@ SLASHopt [/]*
                                        }
                                      }
                                    }
+<VerbatimCode>"'"                  {
+                                     copyToOutput(yyscanner,yytext,yyleng);
+                                     if (yyextra->blockName=="`") // support for `text'
+                                     {
+                                       yyextra->inVerbatim=false;
+                                       BEGIN(yyextra->lastCommentContext);
+                                     }
+                                   }
 <VerbatimCode>"`"{1,2}             {
                                      copyToOutput(yyscanner,yytext,yyleng);
                                      if (yytext==yyextra->blockName)
@@ -765,7 +776,7 @@ SLASHopt [/]*
                                        BEGIN(yyextra->lastCommentContext);
                                      }
                                    }
-<VerbatimCode>{CMD}("enddot"|"endcode"|"endmsc"|"enduml")/("{")? { /* end of verbatim block */
+<VerbatimCode>{CMD}("enddot"|"endcode"|"endmsc"|"enduml")/("{")? { // end of verbatim block
                                      copyToOutput(yyscanner,yytext,yyleng);
 				     if (&yytext[1]==yyextra->blockName)
 				     {
@@ -796,10 +807,10 @@ SLASHopt [/]*
                                        }
                                      }
   				   }
-<Verbatim,VerbatimCode>[^`~@\/\-\\\n{}]* { /* any character not a backslash or new line or } */
+<Verbatim,VerbatimCode>[^`'~@\/\-\\\n{}]* { /* any character not a backslash or new line or } */
                                      copyToOutput(yyscanner,yytext,yyleng);
                                    }
-<Verbatim,VerbatimCode>[^`~@\/\-\\\n{}]+/\n { /* premature end of comment block */
+<Verbatim,VerbatimCode>[^`'~@\/\-\\\n{}]+/\n { /* premature end of comment block */
                                      if (yyextra->lastCommentContext==ReadLine)
                                      {
                                        yyextra->inVerbatim=false;

src/markdown.cpp

@@ -1626,9 +1626,9 @@ int Markdown::Private::processLink(const std::string_view data,size_t offset)
 }
 
 /** `` ` `` parsing a code span (assuming codespan != 0) */
-int Markdown::Private::processCodeSpan(std::string_view data,size_t)
+int Markdown::Private::processCodeSpan(std::string_view data,size_t offset)
 {
-  AUTO_TRACE("data='{}'",Trace::trunc(data));
+  AUTO_TRACE("data='{}' offset={}",Trace::trunc(data),offset);
   const size_t size = data.size();
 
   /* counting the number of backticks in the delimiter */
@@ -1650,7 +1650,11 @@ int Markdown::Private::processCodeSpan(std::string_view data,size_t)
     else if (data[end]=='\n')
     {
       // consecutive newlines
-      if (pc == '\n') return 0;
+      if (pc == '\n')
+      {
+        AUTO_TRACE_EXIT("new paragraph");
+        return 0;
+      }
       pc = '\n';
       i = 0;
     }
@@ -1659,6 +1663,7 @@ int Markdown::Private::processCodeSpan(std::string_view data,size_t)
       out+="‘";
       out+=data.substr(nb,end-nb);
       out+="’";
+      AUTO_TRACE_EXIT("quoted end={}",end+1);
       return static_cast<int>(end+1);
     }
     else
@@ -1669,8 +1674,18 @@ int Markdown::Private::processCodeSpan(std::string_view data,size_t)
   }
   if (i < nb && end >= size)
   {
+    AUTO_TRACE_EXIT("no matching delimiter nb={} i={}",nb,i);
+    if (nb>=3) // found ``` that is not at the start of the line, keep it as-is.
+    {
+      out+=data.substr(0,nb);
+      return nb;
+    }
     return 0;  // no matching delimiter
   }
+  while (end<size && data[end]=='`') // do greedy match in case we have more end backticks.
+  {
+    end++;
+  }
 
   //printf("found code span '%s'\n",qPrint(QCString(data+f_begin).left(f_end-f_begin)));
 
@@ -1682,7 +1697,7 @@ int Markdown::Private::processCodeSpan(std::string_view data,size_t)
     out+=escapeSpecialChars(codeFragment);
     out+="</tt>";
   }
-  AUTO_TRACE_EXIT("result={}",end);
+  AUTO_TRACE_EXIT("result={} nb={}",end,nb);
   return static_cast<int>(end);
 }
 

src/scanner.l

@@ -7190,6 +7190,9 @@ NONLopt [^\n]*
                                           BEGIN(DocCopyBlock);
                                         }
 <DocBlock>"```"                         { // skip ``` if not at the start of a line
+                                          yyextra->docBlock << "```";
+                                        }
+<DocBlock>"\\`"                         { // skip escaped backtick
                                           yyextra->docBlock << yytext;
                                         }
 <DocBlock>"`"{1,2}                      {
@@ -7209,7 +7212,7 @@ NONLopt [^\n]*
                                             REJECT;
                                           }
                                         }
-<DocBlock>[^@*~`\/\\\n]+                { // any character that isn't special
+<DocBlock>[^@*~`'\/\\\n]+               { // any character that isn't special
                                           yyextra->docBlock << yytext;
                                         }
 <DocBlock>\n                            { // newline
@@ -7323,7 +7326,20 @@ NONLopt [^\n]*
                                           yyextra->docBlock << pat;
                                         }
 <DocCopyBlock>"```"                     { // backtick + end marker
-                                          if (yyextra->docBlockName=="``" && endVerbatimBlock(yyscanner,"``",2))
+                                          if (yyextra->docBlockName=="``") 
+                                          {
+                                            yyextra->docBlock << "`";
+                                            endVerbatimBlock(yyscanner,"``",2);
+                                            yyextra->docBlock << "``";
+                                            BEGIN(DocBlock);
+                                          }
+                                          else
+                                          {
+                                            yyextra->docBlock << yytext;
+                                          }
+                                        }
+<DocCopyBlock>"'"/[^a-z_A-Z0-9-]        {
+                                          if (endVerbatimBlock(yyscanner,"`",1))
                                           {
                                             BEGIN(DocBlock);
                                           }
@@ -7336,7 +7352,7 @@ NONLopt [^\n]*
                                           }
                                           yyextra->docBlock << yytext;
                                         }
-<DocCopyBlock>[^\<@/\*\]`~"\$\\\n]+     { // any character that is not special
+<DocCopyBlock>[^\<@/\*\]`'~"\$\\\n]+    { // any character that is not special
                                           yyextra->docBlock << yytext;
                                         }
 <DocCopyBlock>\"                        {

testing/110/indexpage.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<doxygen xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="compound.xsd" version="" xml:lang="en-US">
+  <compounddef id="indexpage" kind="page">
+    <compoundname>index</compoundname>
+    <title>My Project</title>
+    <briefdescription>
+    </briefdescription>
+    <detaileddescription>
+      <para>Text with <computeroutput>single</computeroutput> backtick. <linebreak/>
+ Text with <lsquo/>mixed<rsquo/> backtick. <linebreak/>
+ Text <computeroutput>it's cool</computeroutput>. with embedded ' <linebreak/>
+ Text with <computeroutput>double</computeroutput> backtick. <linebreak/>
+ Text with <computeroutput>double and `single` form</computeroutput> of backticks. <linebreak/>
+ Text with <computeroutput>double backtick `ending`</computeroutput> in triple backtick. <linebreak/>
+ Text with `escaped` backticks. <linebreak/>
+ Text with triple ``` backtick not at the start of the line. <linebreak/>
+ <programlisting><codeline><highlight class="normal">Text<sp/>inside<sp/>triple<sp/>backticks</highlight></codeline><codeline><highlight class="normal">Nested<sp/>`single`<sp/>backtick</highlight></codeline><codeline><highlight class="normal">Nested<sp/>``</highlight><highlight class="keywordtype">double</highlight><highlight class="normal">``<sp/>backtick</highlight></codeline></programlisting> More text after triple backtick section. </para>
+    </detaileddescription>
+    <location file="110_backticks.dox"/>
+  </compounddef>
+</doxygen>

testing/110_backticks.dox

@@ -0,0 +1,18 @@
+// objective: test various forms of backtick handling
+// check: indexpage.xml
+/** @mainpage
+Text with `single` backtick.  
+Text with `mixed' backtick.  
+Text `it's cool`. with embedded '  
+Text with ``double`` backtick.  
+Text with ``double and `single` form`` of backticks.  
+Text with ``double backtick `ending``` in triple backtick.  
+Text with \`escaped\` backticks.  
+Text with triple ``` backtick not at the start of the line.  
+```
+Text inside triple backticks
+Nested `single` backtick
+Nested ``double`` backtick
+```
+More text after triple backtick section.
+*/