Merge pull request #4 from nkovacs/better-function-param-parsing

Improve function param parsing

Author: eemeli

README.md

@@ -4,13 +4,23 @@ A [PEG.js] parser for [ICU MessageFormat] strings – part of [messageformat.js]
 Outputs an AST defined by [parser.pegjs].
 
 The generated parser function takes two parameters, first the string to be
-parsed, and a second optional parameter `options`, an object containing arrays
+parsed, and a second optional parameter `options`, an object.
+
+The options object contains arrays
 of keywords for `cardinal` and `ordinal` rules for the current locale – these
 are used to validate plural and selectordinal keys. If `options` or its fields
 are missing or set to false, the full set of valid [Unicode CLDR] keys is used:
 `'zero', 'one', 'two', 'few', 'many', 'other'`. To disable this check, pass in
 an empty array.
 
+The `options` object also supports a setting that makes the parser
+follow the ICU MessageFormat spec more closely: `strictFunctionParams`.
+
+By default, function parameters are split on commas and trimmed,
+so the parameters in `{x,fn,   a,   b   }` are parsed as `['a','b']`.
+Setting `strictFunctionParams` to true will result in a params array
+with a single element: `['   a,   b   ']`.
+
 [ICU MessageFormat]: https://messageformat.github.io/guide/
 [messageformat.js]: https://messageformat.github.io/
 [parser.pegjs]: ./parser.pegjs

parser.pegjs

@@ -1,4 +1,4 @@
-start = token* 
+start = token*
 
 token
   = argument / select / plural / function
@@ -37,7 +37,7 @@ plural = '{' _ arg:id _ ',' _ type:('plural'/'selectordinal') _ ',' _ offset:off
     };
   }
 
-function = '{' _ arg:id _ ',' _ key:id _ params:functionParams* '}' {
+function = '{' _ arg:id _ ',' _ key:id _ params:functionParams '}' {
     return {
       type: 'function',
       arg: arg,
@@ -48,6 +48,10 @@ function = '{' _ arg:id _ ',' _ key:id _ params:functionParams* '}' {
 
 id = $([0-9a-zA-Z$_][^ \t\n\r,.+={}]*)
 
+paramDefault = str:paramcharsDefault+ { return str.join(''); }
+
+paramStrict = str:paramcharsStrict+ { return str.join(''); }
+
 selectCase = _ key:id _ tokens:caseTokens { return { key: key, tokens: tokens }; }
 
 pluralCase = _ key:pluralKey _ tokens:caseTokens { return { key: key, tokens: tokens }; }
@@ -60,7 +64,25 @@ pluralKey
   = id
   / '=' d:digits { return d; }
 
-functionParams = _ ',' _ p:id _ { return p; }
+functionParams
+  = p:functionParamsDefault* ! { return options.strictFunctionParams; } { return p; }
+  / p:functionParamsStrict* & { return options.strictFunctionParams; } { return p; }
+
+functionParamsStrict = _ ',' p:paramStrict { return p; }
+
+functionParamsDefault = _ ',' _ p:paramDefault _ { return p.replace(/^[ \t\n\r]*|[ \t\n\r]*$/g, ''); }
+
+doubleapos = "''" { return "'"; }
+
+inapos = doubleapos / str:[^']+ { return str.join(''); }
+
+quotedCurly
+  = "'{"str:inapos*"'" { return '\u007B'+str.join(''); }
+  / "'}"str:inapos*"'" { return '\u007D'+str.join(''); }
+
+quotedFunctionParams
+  = quotedCurly
+  / "'"
 
 char
   = [^{}#\\\0-\x08\x0e-\x1f\x7f]
@@ -72,6 +94,18 @@ char
       return String.fromCharCode(parseInt('0x' + h1 + h2 + h3 + h4));
     }
 
+paramcharsCommon
+  = doubleapos
+  / quotedFunctionParams
+
+paramcharsDefault
+  = paramcharsCommon
+  / str:[^',}]+ { return str.join(''); }
+
+paramcharsStrict
+  = paramcharsCommon
+  / str:[^'}]+ { return str.join(''); }
+
 digits = $([0-9]+)
 
 hexDigit = [0-9a-fA-F]

test.js

@@ -239,6 +239,61 @@ describe("Ordinals", function() {
   });
 
 });
+describe("Functions", function() {
+  it("should accept no parameters", function() {
+    expect(parse('{var,date}')[0].type).to.eql('function');
+    expect(parse('{var,date}')[0].key).to.eql('date');
+    expect(parse('{var,date}')[0].params).to.be.empty;
+  })
+
+  it("should accept parameters", function() {
+    expect(parse('{var,date,long}')[0].type).to.eql('function');
+    expect(parse('{var,date,long}')[0].key).to.eql('date');
+    expect(parse('{var,date,long}')[0].params[0]).to.eql('long');
+    expect(parse('{var,date,long,short}')[0].params[0]).to.eql('long');
+    expect(parse('{var,date,long,short}')[0].params[1]).to.eql('short');
+  })
+
+  it("should accept parameters with whitespace", function() {
+    expect(parse('{var,date,y-M-d HH:mm:ss zzzz}')[0].type).to.eql('function');
+    expect(parse('{var,date,y-M-d HH:mm:ss zzzz}')[0].key).to.eql('date');
+    expect(parse('{var,date,y-M-d HH:mm:ss zzzz}')[0].params[0]).to.eql('y-M-d HH:mm:ss zzzz');
+    // This is not how ICU works. ICU does not trim whitespace,
+    // but messageformat-parse must trim it to maintain backwards compatibility.
+    expect(parse('{var,date,   y-M-d HH:mm:ss zzzz    }')[0].params[0]).to.eql('y-M-d HH:mm:ss zzzz');
+    // This is how ICU works.
+    expect(parse('{var,date,   y-M-d HH:mm:ss zzzz    }', { strictFunctionParams: true })[0].params[0]).to.eql('   y-M-d HH:mm:ss zzzz    ');
+  })
+
+  it("should accept parameters with special characters", function() {
+    expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz}")[0].type).to.eql('function');
+    expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz}")[0].key).to.eql('date');
+    expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d {,} ' HH:mm:ss zzzz");
+    expect(parse("{var,date,y-M-d '{,}' '' HH:mm:ss zzzz'}'}")[0].params[0]).to.eql("y-M-d {,} ' HH:mm:ss zzzz}");
+    expect(parse("{var,date,y-M-d # HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d # HH:mm:ss zzzz");
+    expect(parse("{var,date,y-M-d '#' HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d '#' HH:mm:ss zzzz");
+    // This is not how ICU works.
+    expect(parse("{var,date,y-M-d, HH:mm:ss zzzz}")[0].params[0]).to.eql("y-M-d");
+    expect(parse("{var,date,y-M-d, HH:mm:ss zzzz}")[0].params[1]).to.eql("HH:mm:ss zzzz");
+    // This is how ICU works, but this only allows a single argStyle parameter.
+    expect(parse("{var,date,y-M-d, HH:mm:ss zzzz}", { strictFunctionParams: true })[0].params[0]).to.eql("y-M-d, HH:mm:ss zzzz");
+  })
+
+  it("should be gracious with whitespace", function() {
+    var firstRes = JSON.stringify(parse('{var, date, long, short}'));
+    expect(JSON.stringify(parse('{ var, date, long, short }'))).to.eql(firstRes);
+    expect(JSON.stringify(parse('{var,date,long,short}'))).to.eql(firstRes);
+    expect(JSON.stringify(parse('{\nvar,   \ndate,\n   long\n\n,\n short\n\n\n}'))).to.eql(firstRes);
+    expect(JSON.stringify(parse('{\tvar\t,\t\t\r date\t\n, \tlong\n,    short\t\n\n\n\n}'))).to.eql(firstRes);
+
+    // This is not how ICU works. ICU does not trim whitespace.
+    firstRes = JSON.stringify(parse('{var, date, y-M-d HH:mm:ss zzzz}'));
+    expect(JSON.stringify(parse('{ var, date, y-M-d HH:mm:ss zzzz }'))).to.eql(firstRes);
+    expect(JSON.stringify(parse('{var,date,y-M-d HH:mm:ss zzzz}'))).to.eql(firstRes);
+    expect(JSON.stringify(parse('{\nvar,   \ndate,\n  \n\n\n y-M-d HH:mm:ss zzzz\n\n\n}'))).to.eql(firstRes);
+    expect(JSON.stringify(parse('{\tvar\t,\t\t\r date\t\n, \t\ny-M-d HH:mm:ss zzzz\t\n\n\n\n}'))).to.eql(firstRes);
+  });
+});
 describe("Nested/Recursive blocks", function() {
 
   it("should allow a select statement inside of a select statement", function() {