Fixed decoration of contents of style nodes in HTML. Consolidated code that handles nested languages in the process.

Author: mikesamuel

CHANGES.html

@@ -70,6 +70,9 @@ <h2>6 Jan 2009</h2>
         <a
         href="http://code.google.com/p/google-code-prettify/issues/detail?id=58"
        >issue 58</a>.
+      <li>Changed HTML lexer to use the same embedded source mechanism as the
+        wiki language handler, and changed to use the registered
+        CSS handler for STYLE element content.
     </ul>
   </body>
 </html>

README.html

@@ -154,7 +154,7 @@ <h3>How do I put line numbers in my code?</h3>
     <div class="footer">
 <!-- Created: Tue Oct  3 17:51:56 PDT 2006 -->
 <!-- hhmts start -->
-Last modified: Fri Jul  4 20:49:30 PDT 2008
+Last modified: Wed Jan  7 13:25:42 PST 2009
 <!-- hhmts end -->
     </div>
   </body>

src/lang-wiki.js

@@ -38,7 +38,7 @@ PR.registerLangHandler(
          [PR.PR_LITERAL,     /^(?:[A-Z][a-z][a-z0-9]+[A-Z][a-z][a-zA-Z0-9]+)\b/
           ],
          // A preformatted block in an unknown language
-         ['lang-',           /^\{\{\{([\s\S]*?)\}\}\}/],
+         ['lang-',           /^\{\{\{([\s\S]+?)\}\}\}/],
          // A block of source code in an unknown language
          ['lang-',           /^`([^\r\n`]+)`/],
          // An inline URL.

src/prettify.js

@@ -209,6 +209,11 @@ function _pr_isIE6() {
     * literal in a syntactically legal javascript program, and I've removed the
     * "in" keyword since it's not a keyword in many languages, and might be used
     * as a count of inches.
+    *
+    * <p>The link a above does not accurately describe EcmaScript rules since
+    * it fails to distinguish between (a=++/b/i) and (a++/b/i) but it works
+    * very well in practice.
+    *
     * @private
     */
   var REGEXP_PRECEDER_PATTERN = function () {
@@ -241,8 +246,7 @@ function _pr_isIE6() {
       // CAVEAT: this does not properly handle the case where a regular
       // expression immediately follows another since a regular expression may
       // have flags for case-sensitivity and the like.  Having regexp tokens
-      // adjacent is not
-      // valid in any language I'm aware of, so I'm punting.
+      // adjacent is not valid in any language I'm aware of, so I'm punting.
       // TODO: maybe style special characters inside a regexp as punctuation.
     }();
 
@@ -467,8 +471,9 @@ function _pr_isIE6() {
               // tag.
               var name = match.match(pr_tagNameRe)[2];
               var depth = 1;
+              var j;
               end_tag_loop:
-              for (var j = i + 1; j < n; ++j) {
+              for (j = i + 1; j < n; ++j) {
                 var name2 = matches[j].match(pr_tagNameRe);
                 if (name2 && name2[2] === name) {
                   if (name2[1] === '/') {
@@ -509,6 +514,23 @@ function _pr_isIE6() {
         .match(/[cC][lL][aA][sS][sS]=\"[^\"]*\bnocode\b/);
   }
 
+  /**
+   * Apply the given language handler to sourceCode and add the resulting
+   * decorations to out.
+   * @param {number} offset the index of sourceCode within the chunk of source
+   *    whose decorations are already present on out.
+   */
+  function appendDecorations(offset, sourceCode, langHandler, out) {
+    if (!sourceCode) { return; }
+    var decorations = langHandler.call({}, sourceCode);
+    if (offset) {
+      for (var i = decorations.length; (i -= 2) >= 0;) {
+        decorations[i] += offset;
+      }
+    }
+    out.push.apply(out, decorations);
+  }
+
   /** Given triples of [style, pattern, context] returns a lexing function,
     * The lexing function interprets the patterns to find token boundaries and
     * returns a decoration list of the form
@@ -526,6 +548,17 @@ function _pr_isIE6() {
     * E.g., if style is 'lang-lisp', and group 1 contains the text
     * '(hello (world))', then that portion of the token will be passed to the
     * registered lisp handler for formatting.
+    * The text before and after group 1 will be restyled using this decorator
+    * so decorators should take care that this doesn't result in infinite
+    * recursion.  For example, the HTML lexer rule for SCRIPT elements looks
+    * something like ['lang-js', /<[s]cript>(.+?)<\/script>/].  This may match
+    * '<script>foo()<\/script>', which would cause the current decorator to
+    * be called with '<script>' which would not match the same rule since
+    * group 1 must not be empty, so it would be instead styled as PR_TAG by
+    * the generic tag rule.  The handler registered for the 'js' extension would
+    * then be called with 'foo()', and finally, the current decorator would
+    * be called with '<\/script>' which would not match the original rule and
+    * so the generic tag rule would identify it as a tag.
     *
     * Pattern must only match prefixes, and if it matches a prefix and context
     * is null or matches the last non-comment token parsed, then that match is
@@ -564,7 +597,7 @@ function _pr_isIE6() {
     var nPatterns = fallthroughStylePatterns.length;
     var notWs = /\S/;
 
-    return function (sourceCode, opt_basePos) {
+    return function decorate(sourceCode, opt_basePos) {
       opt_basePos = opt_basePos || 0;
       var decorations = [opt_basePos, PR_PLAIN];
       var lastToken = '';
@@ -603,31 +636,36 @@ function _pr_isIE6() {
           }
         }
 
-        if (!match || !match[1] || 'lang-' !== style.substring(0, 5)) {
+        var isEmbedded = 'lang-' === style.substring(0, 5);
+        if (isEmbedded && !(match && match[1])) {
+          isEmbedded = false;
+          style = PR_SOURCE;
+        }
+        if (!isEmbedded) {
           decorations.push(opt_basePos + pos, style);
         } else {  // Treat group 1 as an embedded block of source code.
-          var lang = style.substring(5);
           var embeddedSource = match[1];
           var embeddedSourceStart = token.indexOf(embeddedSource);
           var embeddedSourceEnd = embeddedSourceStart + embeddedSource.length;
-          if (embeddedSourceStart) {
-            decorations.push(opt_basePos + pos, PR_SOURCE);
-          }
+          var lang = style.substring(5);
           if (!langHandlerRegistry.hasOwnProperty(lang)) {
             lang = /^\s*</.test(embeddedSource)
                 ? 'default-markup'
-                : 'default-code'
-          }
-          var delegate = langHandlerRegistry[lang];
-          var embeddedOffset = opt_basePos + pos + embeddedSourceStart;
-          var embeddedDecorations = delegate.call({}, embeddedSource);
-          for (var i = 0, n = embeddedDecorations.length; i < n; i += 2) {
-            decorations.push(embeddedOffset + embeddedDecorations[i],
-                             embeddedDecorations[i + 1]);
-          }
-          if (embeddedSourceEnd < token.length) {
-            decorations.push(opt_basePos + pos + embeddedSourceEnd, PR_SOURCE);
+                : 'default-code';
           }
+          var size = decorations.length - 10;
+          appendDecorations(
+              opt_basePos + pos,
+              token.substring(0, embeddedSourceStart),
+              decorate, decorations);
+          appendDecorations(
+              opt_basePos + pos + embeddedSourceStart,
+              token.substring(embeddedSourceStart, embeddedSourceEnd),
+              langHandlerRegistry[lang], decorations);
+          appendDecorations(
+              opt_basePos + pos + embeddedSourceEnd,
+              token.substring(embeddedSourceEnd),
+              decorate, decorations);
         }
         pos += token.length;
         tail = tail.substring(token.length);
@@ -638,14 +676,18 @@ function _pr_isIE6() {
   }
 
   var PR_MARKUP_LEXER = createSimpleLexer([], [
-      [PR_PLAIN,       /^[^<]+/, null],
+      [PR_PLAIN,       /^[^<?]+/, null],
       [PR_DECLARATION, /^<!\w[^>]*(?:>|$)/, null],
       [PR_COMMENT,     /^<!--[\s\S]*?(?:-->|$)/, null],
-      [PR_SOURCE,      /^<\?[\s\S]*?(?:\?>|$)/, null],
-      [PR_SOURCE,      /^<%[\s\S]*?(?:%>|$)/, null],
-      [PR_SOURCE,
-       // Tags whose content is not escaped, and which contain source code.
-       /^<(script|style|xmp)\b[^>]*>[\s\S]*?<\/\1\b[^>]*>/i, null],
+       // Unescaped content in an unknown language
+      ['lang-',        /^<\?([\s\S]+?)(?:\?>|$)/, null],
+      ['lang-',        /^<%([\s\S]+?)(?:%>|$)/, null],
+      [PR_PUNCTUATION, /^(?:<[%?]|[%?]>)/, null],
+      ['lang-',        /^<xmp\b[^>]*>([\s\S]+?)<\/xmp\b[^>]*>/i, null],
+      // Unescaped content in javascript.  (Or possibly vbscript).
+      ['lang-js',      /^<script\b[^>]*>([\s\S]+?)<\/script\b[^>]*>/i, null],
+      // Contains unescaped stylesheet content
+      ['lang-css',     /^<style\b[^>]*>([\s\S]+?)<\/style\b[^>]*>/i, null],
       [PR_TAG,         /^<\/?\w[^<>]*>/, null]
       ]);
   // Splits any of the source|style|xmp entries above into a start tag,
@@ -856,29 +898,6 @@ function _pr_isIE6() {
         regexLiterals: true
       });
 
-  /** identify regions of markup that are really source code, and recursivley
-    * lex them.
-    * @private
-    */
-  function splitSourceNodes(source, decorations) {
-    for (var i = 0; i < decorations.length; i += 2) {
-      var style = decorations[i + 1];
-      if (style === PR_SOURCE) {
-        // Recurse using the non-markup lexer
-        var start, end;
-        start = decorations[i];
-        end = i + 2 < decorations.length ? decorations[i + 2] : source.length;
-        var subDecorations = decorateSource(source.substring(start, end));
-        for (var j = 0, m = subDecorations.length; j < m; j += 2) {
-          subDecorations[j] += start;
-        }
-        spliceArrayInto(subDecorations, decorations, i, 2);
-        i += subDecorations.length - 2;
-      }
-    }
-    return decorations;
-  }
-
   /** identify attribute values that really contain source code and recursively
     * lex them.
     * @private
@@ -969,7 +988,6 @@ function _pr_isIE6() {
     //            (PR_TAG, PR_PLAIN, PR_SOURCE, NAME, VALUE, null)
     var decorations = tokenizeMarkup(sourceCode);
     decorations = splitTagAttributes(sourceCode, decorations);
-    decorations = splitSourceNodes(sourceCode, decorations);
     decorations = splitSourceAttributes(sourceCode, decorations);
     return decorations;
   }
@@ -1189,7 +1207,7 @@ function _pr_isIE6() {
         document.getElementsByTagName('xmp') ];
     var elements = [];
     for (var i = 0; i < codeSegments.length; ++i) {
-      for (var j = 0; j < codeSegments[i].length; ++j) {
+      for (var j = 0, n = codeSegments[i].length; j < n; ++j) {
         elements.push(codeSegments[i][j]);
       }
     }

tests/prettify_test.html

@@ -497,6 +497,7 @@ <h1>HTML</h1>
 &lt;html&gt;
   &lt;head>
     &lt;title&gt;Fibonacci number&lt;/title&gt;
+    &lt;style&gt;&lt;!-- BODY { text-decoration: blink } --&gt;&lt;/style&gt;
   &lt;/head&gt;
   &lt;body>
     &lt;noscript&gt;
@@ -1533,14 +1534,20 @@ <h1>CSS w/ language specified</h1>
   html: (
       '`PUN&lt;`END`TAGhtml`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; `END`PUN&lt;`END`TAGhead`END`PUN&gt;`END`PLN<br>' +
-      '&nbsp; &nbsp; `END`PUN&lt;`END`TAGtitle`END`PUN&gt;`END`PLNFibonacci number`END' +
+      '&nbsp; &nbsp; `END`PUN&lt;`END`TAGtitle`END`PUN&gt;`END' +
+          '`PLNFibonacci number`END' +
           '`PUN&lt;/`END`TAGtitle`END`PUN&gt;`END`PLN<br>' +
+      '&nbsp; &nbsp; `END`PUN&lt;`END`TAGstyle`END`PUN&gt;`END`COM&lt;!--`END' +
+          '`PLN BODY `END`PUN{`END`PLN `END`KWDtext-decoration`END`PUN:`END' +
+          '`PLN blink `END`PUN}`END`PLN `END`COM--&gt;`END`PUN&lt;/`END' +
+          '`TAGstyle`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; `END`PUN&lt;/`END`TAGhead`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; `END`PUN&lt;`END`TAGbody`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; &nbsp; `END`PUN&lt;`END`TAGnoscript`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; &nbsp; &nbsp; `END`PUN&lt;`END`TAGdl`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; &nbsp; &nbsp; &nbsp; `END`PUN&lt;`END`TAGdt`END`PUN&gt;`END' +
-          '`PLNFibonacci numbers`END`PUN&lt;/`END`TAGdt`END`PUN&gt;`END`PLN<br>' +
+          '`PLNFibonacci numbers`END`PUN&lt;/`END`TAGdt`END`PUN&gt;`END' +
+          '`PLN<br>' +
       '&nbsp; &nbsp; &nbsp; &nbsp; `END`PUN&lt;`END`TAGdd`END`PUN&gt;`END`PLN1`END' +
           '`PUN&lt;/`END`TAGdd`END`PUN&gt;`END`PLN<br>' +
       '&nbsp; &nbsp; &nbsp; &nbsp; `END`PUN&lt;`END`TAGdd`END`PUN&gt;`END`PLN1`END' +
@@ -2307,31 +2314,31 @@ <h1>CSS w/ language specified</h1>
            '`STRhttp://www.google.com/?q=WikiSyntax+site:code.google.com`END' +
            '`PLN `END`LITWikiSyntax`END`PUN]`END`PLN<br>' +
       '<br>' +
-      'Lorem Ipsum `END`SRC``END`KWDwhile`END`PLN `END`PUN(`END`LIT1`END' +
+      'Lorem Ipsum ``END`KWDwhile`END`PLN `END`PUN(`END`LIT1`END' +
            '`PUN)`END`PLN `END`KWDprint`END`PUN(`END`STR"blah blah"`END' +
-           '`PUN);`END`SRC``END`PLN<br>' +
+           '`PUN);`END`PLN`<br>' +
       '<br>' +
       '&nbsp; &nbsp;`END`PUN*`END`PLN Bullet<br>' +
       '&nbsp; &nbsp;`END`PUN*`END`PLN Points<br>' +
       '&nbsp; &nbsp; &nbsp; `END`PUN*`END`PLN `END`LITNestedBullet`END' +
            '`PLN<br>' +
       '<br>' +
       '`END`PUN==`END`LITDroningOnAndOn`END`PUN==`END`PLN<br>' +
-      '`END`SRC{{{`END`PLN<br>' +
+      '{{{<br>' +
       '&nbsp; `END`COM// Some EmbeddedSourceCode`END`PLN<br>' +
       '&nbsp; `END`KWDvoid`END`PLN main`END`PUN()`END`PLN `END`PUN{`END' +
            '`PLN<br>' +
       '&nbsp; &nbsp; `END`TYPPrint`END`PUN(`END`STR\'hello world\'`END' +
            '`PUN);`END`PLN<br>' +
       '&nbsp; `END`PUN}`END`PLN<br>' +
-      '`END`SRC}}}`END`PLN<br>' +
+      '}}}<br>' +
       '<br>' +
-      '`END`SRC{{{`END`PLN<br>' +
+      '{{{<br>' +
       '&nbsp; `END`COM&lt;!-- Embedded XML --&gt;`END`PLN<br>' +
       '&nbsp; `END`PUN&lt;`END`TAGfoo`END`PLN `END`ATNbar`END`PUN=`END' +
            '`ATV"baz"`END`PUN&gt;&lt;`END`TAGboo`END`PLN `END' +
            '`PUN/&gt;&lt;`END`TAGfoo`END`PUN&gt;`END`PLN<br>' +
-      '`END`SRC}}}`END'
+      '}}}`END'
       ),
   css: (
       '`COM&lt;!--`END`PLN<br>' +
@@ -2342,8 +2349,8 @@ <h1>CSS w/ language specified</h1>
           '`PUN);`END`PLN<br>' +
       '<br>' +
       'HTML `END`PUN{`END`PLN `END`KWDcontent-before`END`PUN:`END`PLN `END' +
-          '`STR\'hello\20\'`END`PUN;`END`PLN `END`KWDcontent-after`END' +
-          '`PUN:`END`PLN `END`STR\'w\6f rld\'`END`PUN;`END`PLN<br>' +
+          '`STR\'hello\\20\'`END`PUN;`END`PLN `END`KWDcontent-after`END' +
+          '`PUN:`END`PLN `END`STR\'w\\6f rld\'`END`PUN;`END`PLN<br>' +
       '&nbsp; &nbsp; &nbsp; &nbsp;`END`KWD-moz-spiff`END`PUN:`END`PLN `END' +
           '`KWDinherit`END`PLN `END`KWD!important`END`PLN `END`PUN}`END' +
           '`PLN<br>' +
@@ -2457,8 +2464,9 @@ <h1>CSS w/ language specified</h1>
       htmlOut.push('<h2>' + html(lang) + ' OK<\/h1>');
     }
   }
-  htmlOut.push('<h2>' + (failures ? (failures + ' test(s) failed') : 'Pass') +
-               '<\/h2>');
+  var summary = (failures ? (failures + ' test(s) failed') : 'Tests Passed');
+  htmlOut.push('<h2>' + summary + '<\/h2>');
+  document.title += ' \u2014 ' + summary;
   document.getElementById('errorReport').innerHTML =
       htmlOut.join('').replace(/&lt;br&gt;/g, '&lt;br&gt;\n');
 }