[dylan mode] Improve and add tests

 * Support nested block comments.
 * Improve number and operator parsing.
 * Improve string parsing.
 * Style brackets correctly.
 * Improve #-words.

Author: waywardmonkeys

mode/dylan/dylan.js

@@ -169,15 +169,16 @@ CodeMirror.defineMode("dylan", function(_config) {
       } else if (stream.eat("/")) {
         stream.skipToEnd();
         return "comment";
-      } else {
-        stream.skipTo(" ");
-        return "operator";
       }
+      stream.backUp(1);
     }
     // Decimal
-    else if (/\d/.test(ch)) {
-      stream.match(/^\d*(?:\.\d*)?(?:e[+\-]?\d+)?/);
-      return "number";
+    else if (/[+\-\d\.]/.test(ch)) {
+      if (stream.match(/^[+-]?[0-9]*\.[0-9]*([esdx][+-]?[0-9]+)?/i) ||
+          stream.match(/^[+-]?[0-9]+([esdx][+-]?[0-9]+)/i) ||
+          stream.match(/^[+-]?\d+/)) {
+        return "number";
+      }
     }
     // Hash
     else if (ch == "#") {
@@ -186,7 +187,7 @@ CodeMirror.defineMode("dylan", function(_config) {
       ch = stream.peek();
       if (ch == '"') {
         stream.next();
-        return chain(stream, state, tokenString('"', "string-2"));
+        return chain(stream, state, tokenString('"', "string"));
       }
       // Binary number
       else if (ch == "b") {
@@ -206,11 +207,51 @@ CodeMirror.defineMode("dylan", function(_config) {
         stream.eatWhile(/[0-7]/);
         return "number";
       }
+      // Token concatenation in macros
+      else if (ch == '#') {
+        stream.next();
+        return "punctuation";
+      }
+      // Sequence literals
+      else if ((ch == '[') || (ch == '(')) {
+        stream.next();
+        return "bracket";
       // Hash symbol
-      else {
+      } else if (stream.match(/f|t|all-keys|include|key|next|rest/i)) {
+        return "atom";
+      } else {
         stream.eatWhile(/[-a-zA-Z]/);
-        return "keyword";
+        return "error";
+      }
+    } else if (ch == "~") {
+      stream.next();
+      ch = stream.peek();
+      if (ch == "=") {
+        stream.next();
+        ch = stream.peek();
+        if (ch == "=") {
+          stream.next();
+          return "operator";
+        }
+        return "operator";
       }
+      return "operator";
+    } else if (ch == ":") {
+      stream.next();
+      ch = stream.peek();
+      if (ch == "=") {
+        stream.next();
+        return "operator";
+      } else if (ch == ":") {
+        stream.next();
+        return "punctuation";
+      }
+    } else if ("[](){}".indexOf(ch) != -1) {
+      stream.next();
+      return "bracket";
+    } else if (".,".indexOf(ch) != -1) {
+      stream.next();
+      return "punctuation";
     } else if (stream.match("end")) {
       return "keyword";
     }
@@ -223,6 +264,10 @@ CodeMirror.defineMode("dylan", function(_config) {
           return patternStyles[name];
       }
     }
+    if (/[+\-*\/^=<>&|]/.test(ch)) {
+      stream.next();
+      return "operator";
+    }
     if (stream.match("define")) {
       return "def";
     } else {
@@ -240,29 +285,37 @@ CodeMirror.defineMode("dylan", function(_config) {
   }
 
   function tokenComment(stream, state) {
-    var maybeEnd = false,
-    ch;
+    var maybeEnd = false, maybeNested = false, nestedCount = 0, ch;
     while ((ch = stream.next())) {
       if (ch == "/" && maybeEnd) {
-        state.tokenize = tokenBase;
-        break;
+        if (nestedCount > 0) {
+          nestedCount--;
+        } else {
+          state.tokenize = tokenBase;
+          break;
+        }
+      } else if (ch == "*" && maybeNested) {
+        nestedCount++;
       }
       maybeEnd = (ch == "*");
+      maybeNested = (ch == "/");
     }
     return "comment";
   }
 
   function tokenString(quote, style) {
     return function(stream, state) {
-      var next, end = false;
+      var escaped = false, next, end = false;
       while ((next = stream.next()) != null) {
-        if (next == quote) {
+        if (next == quote && !escaped) {
           end = true;
           break;
         }
+        escaped = !escaped && next == "\\";
       }
-      if (end)
+      if (end || !escaped) {
         state.tokenize = tokenBase;
+      }
       return style;
     };
   }

mode/dylan/test.js

@@ -0,0 +1,88 @@
+// CodeMirror, copyright (c) by Marijn Haverbeke and others
+// Distributed under an MIT license: http://codemirror.net/LICENSE
+
+(function() {
+  var mode = CodeMirror.getMode({indentUnit: 2}, "dylan");
+  function MT(name) { test.mode(name, mode, Array.prototype.slice.call(arguments, 1)); }
+
+  MT('comments',
+     '[comment // This is a line comment]',
+     '[comment /* This is a block comment */]',
+     '[comment /* This is a multi]',
+     '[comment line comment]',
+     '[comment */]',
+     '[comment /* And this is a /*]',
+     '[comment /* nested */ comment */]');
+
+  MT('unary_operators',
+     '[operator -][variable a]',
+     '[operator -] [variable a]',
+     '[operator ~][variable a]',
+     '[operator ~] [variable a]');
+
+  MT('binary_operators',
+     '[variable a] [operator +] [variable b]',
+     '[variable a] [operator -] [variable b]',
+     '[variable a] [operator *] [variable b]',
+     '[variable a] [operator /] [variable b]',
+     '[variable a] [operator ^] [variable b]',
+     '[variable a] [operator =] [variable b]',
+     '[variable a] [operator ==] [variable b]',
+     '[variable a] [operator ~=] [variable b]',
+     '[variable a] [operator ~==] [variable b]',
+     '[variable a] [operator <] [variable b]',
+     '[variable a] [operator <=] [variable b]',
+     '[variable a] [operator >] [variable b]',
+     '[variable a] [operator >=] [variable b]',
+     '[variable a] [operator &] [variable b]',
+     '[variable a] [operator |] [variable b]',
+     '[variable a] [operator :=] [variable b]');
+
+  MT('integers',
+     '[number 1]',
+     '[number 123]',
+     '[number -123]',
+     '[number +456]',
+     '[number #b010]',
+     '[number #o073]',
+     '[number #xabcDEF123]');
+
+  MT('floats',
+     '[number .3]',
+     '[number -1.]',
+     '[number -2.335]',
+     '[number +3.78d1]',
+     '[number 3.78s-1]',
+     '[number -3.32e+5]');
+
+  MT('characters_and_strings',
+     "[string 'a']",
+     "[string '\\\\'']",
+     '[string ""]',
+     '[string "a"]',
+     '[string "abc def"]',
+     '[string "More escaped characters: \\\\\\\\ \\\\a \\\\b \\\\e \\\\f \\\\n \\\\r \\\\t \\\\0 ..."]');
+
+  MT('brackets',
+     '[bracket #[[]]]',
+     '[bracket #()]',
+     '[bracket #(][number 1][bracket )]',
+     '[bracket [[][number 1][punctuation ,] [number 3][bracket ]]]',
+     '[bracket ()]',
+     '[bracket {}]',
+     '[keyword if] [bracket (][variable foo][bracket )]',
+     '[bracket (][number 1][bracket )]',
+     '[bracket [[][number 1][bracket ]]]');
+
+  MT('hash_words',
+     '[punctuation ##]',
+     '[atom #f]', '[atom #F]',
+     '[atom #t]', '[atom #T]',
+     '[atom #all-keys]',
+     '[atom #include]',
+     '[atom #key]',
+     '[atom #next]',
+     '[atom #rest]',
+     '[string #"foo"]',
+     '[error #invalid]');
+})();

test/index.html

@@ -40,6 +40,7 @@
 <script src="../keymap/vim.js"></script>
 <script src="../mode/rust/rust.js"></script>
 <script src="../mode/mscgen/mscgen.js"></script>
+<script src="../mode/dylan/dylan.js"></script>
 
 <style type="text/css">
   .ok {color: #090;}
@@ -123,6 +124,7 @@ <h2>Test Suite</h2>
     <script src="../mode/mscgen/mscgen_test.js"></script>
     <script src="../mode/mscgen/xu_test.js"></script>
     <script src="../mode/mscgen/msgenny_test.js"></script>
+    <script src="../mode/dylan/test.js"></script>
     <script src="../addon/mode/multiplex_test.js"></script>
     <script src="emacs_test.js"></script>
     <script src="sql-hint-test.js"></script>