[soy mode] tokenize operators in expressions

- tokenize arithmetic and equality operators as "operator"
- added tests to validate this

Author: Christopher Wallis

mode/soy/soy.js

@@ -271,6 +271,10 @@
               stream.match(/-?([0-9]*[.])?[0-9]+/)) {
               return "atom";
             }
+            if (stream.match(/(\||[+\-*\/%]|[=!][=]|[<>][=]?)/)) {
+              // Tokenize filter, binary, and equality operators.
+              return "operator";
+            }
             if (match = stream.match(/^\$([\w]+)/)) {
               return ref(state.variables, match[1]);
             }

mode/soy/test.js

@@ -8,7 +8,7 @@
   // Test of small keywords and words containing them.
   MT('keywords-test',
      '[keyword {] [keyword as] worrying [keyword and] notorious [keyword as]',
-     '    the Fandor-alias assassin, [keyword or]',
+     '    the Fandor[operator -]alias assassin, [keyword or]',
      '    Corcand cannot fit [keyword in] [keyword }]');
 
   MT('let-test',
@@ -33,6 +33,20 @@
      '[keyword {namespace] [variable my.namespace.templates] ' +
          '[attribute requirecss]=[string "my.namespace"][keyword }]');
 
+  MT('operators-test',
+     '[keyword {] [atom 1] [operator ==] [atom 1] [keyword }]',
+     '[keyword {] [atom 1] [operator !=] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator +] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator -] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator *] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator /] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator %] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator <=] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator >=] [atom 2] [keyword }]',
+     '[keyword {] [atom 3] [operator >] [atom 2] [keyword }]',
+     '[keyword {] [atom 2] [operator >] [atom 3] [keyword }]',
+     '[keyword {] [variable-2&error $variable] [operator |] safeHtml [keyword }]')
+
   MT('primitive-test',
      '[keyword {] [atom true] [keyword }]',
      '[keyword {] [atom false] [keyword }]',