Merge pull request #6 from alerque/format-ast

Massage PEG grammar's AST to match the Fluent reference implementation

Author: alerque

README.md

@@ -19,6 +19,20 @@ As of yet this does nothing *usable* (see [lua alternatives](#alternatives)). I'
 
 **Update 2019-09-14**: I've completed a PEG grammar based parser for the entire 1.0 Fluent file format spec. All the pieces are there, but it's only partially tested. It at least parses a few basic types of entries. The AST it returns is straight out of *luaebnf* and probably needs massaging to match the reference ones (via capture groups?), then it needs testing against the upstream fixtures.
 
+**Udate 2019-09-24**: The AST returned by the PEG grammar has been massaged to be usable for some basic cases. A basic Lua API is starting to take shape, modeled most closely to the Python implementation. It is possible to load almost any FTL file, and possible to format any messages that are plain strings (no parameters, attributes, functions, or other jazz yet). Note the usage is *off* because there is no locale handling yet no it's only usable with separate instances per locale. Also `add_messages()` likely only works once, so cram your whole FTL resource in there for now.
+
+## Usage
+
+```lua
+local FluentBundle = require("fluent")
+
+local en = FluentBundle("en-US")
+
+en:add_messages("foo = bar")
+
+en:format("foo")
+```
+
 ## Alternative(s)
 
 If you need something that works in Lua *now*, have a look at the already mature `i18n.lua` project ([Github](https://github.com/kikito/i18n.lua) / [LuaRocks](https://luarocks.org/modules/kikito/i18n)). It implements many of the same features this project will, just without the interoperability with other Fluent based tools. The Lua API it provides is quite nice, but your localization data needs to be provided in Lua tables instead of FTL files. While Fluent has quite a few more tricks up its sleeve the *i18n* module already has working interpolation, pluralization, locale fallbacks, and more.  And it works now, today.

fluent/init.lua

@@ -1,30 +1,31 @@
 -- External dependencies
 local class = require("pl.class")
+local tablex = require("pl.tablex")
 
 -- Internal modules
 local FluentSyntax = require("fluent.syntax")
 
-
-local FluentMessages = class({
-  })
-
 local FluentBundle = class({
     locale = nil,
+    locales = {},
+    syntax = FluentSyntax(),
 
     _init = function (self, locale)
       self.locale = locale
-      self.messages = FluentMessages()
-      self.syntax = FluentSyntax()
     end,
 
     add_messages = function (self, input)
-      for k, v in input:gmatch("(%w+) = (%w+)") do
-        self.messages[k] = v
-      end
+      if type(input) == "string" then input = { input } end
+      -- TODO: add way to add two resources together, then reduce instead of unpacking this
+      local resource = tablex.imap(function (v) return self.syntax:parsestring(v) end, input)
+      self.locales[self.locale] = resource[1]
     end,
 
-    format = function (self, key)
-      return self.messages[key]
+    format = function (self, identifier, parameters)
+      local resource = self.locales[self.locale]
+      local message = resource:get_message(identifier)
+      -- local message = resource[identifier]
+      return message:format(parameters)
     end
   })
 

fluent/messages.lua

@@ -0,0 +1,8 @@
+-- External dependencies
+local class = require("pl.class")
+
+local FluentMessages = class({
+  })
+
+return FluentMessages
+

fluent/parser.lua

@@ -0,0 +1,100 @@
+-- External dependencies
+local class = require("pl.class")
+local epnf = require("epnf")
+
+local nulleof = "NULL\000"
+local eol = function () return "\n" end
+
+-- UTF8 code points up to four-byte encodings
+local function f1 (s)
+  return string.byte(s)
+end
+local function f2 (s)
+  local c1, c2 = string.byte(s, 1, 2)
+  return c1 * 64 + c2 - 12416
+end
+local function f3 (s)
+  local c1, c2, c3 = string.byte(s, 1, 3)
+  return (c1 * 64 + c2) * 64 + c3 - 925824
+end
+local function f4 (s)
+  local c1, c2, c3, c4 = string.byte(s, 1, 4)
+  return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168
+end
+local cont = "\128\191"
+
+-- luacheck: push ignore
+local ftl_eof = epnf.define(function (_ENV)
+  eol_eof = 1^0 * P(nulleof) * -1
+  START("eol_eof")
+end)
+
+local ftl_grammar = epnf.define(function (_ENV)
+  local blank_inline = P" "^1
+  local line_end = P"\r\n" / eol + P"\n" + P(nulleof)
+  blank_block = C((blank_inline^-1 * line_end)^1); local blank_block = V"blank_block"
+  local blank = (blank_inline + line_end)^1
+  local digits = R"09"^1
+  local special_text_char = P"{" + P"}"
+  local any_char = R("\0\127") / f1 + R("\194\223") * R(cont) / f2 + R("\224\239") * R(cont) * R(cont) / f3 + R("\240\244") * R(cont) * R(cont) * R(cont) / f4
+  local text_char = any_char - special_text_char - line_end
+  local special_quoted_char = P'"' + P"\\"
+  local special_escape = P"\\" * special_quoted_char
+  local unicode_escape = (P"\\u" * P(4) * R("09", "af", "AF")^4) + (P"\\u" * P(6) * R("09", "af", "AF")^6)
+  local quoted_char = (any_char - special_quoted_char - line_end) + special_escape + unicode_escape
+  local indented_char = text_char - P"{" - P"*" - P"."
+  Identifier = Cg(R("az", "AZ") * (R("az", "AZ", "09") + P"_" + P"-")^0, "name")
+  local variant_list = V"Variant"^0 * V"DefaultVariant" * V"Variant" * line_end
+  Variant = line_end * blank^-1 * V"VariantKey" * blank_inline^-1 * V"Pattern"
+  DefaultVariant = line_end * blank^-1 * P"*" * V"VariantKey" * blank_inline^-1 * V"Pattern"
+  VariantKey = P"[" * blank^-1 * (V"NumberLiteral" + V"Identifier") * blank^-1 * P"]"
+  NumberLiteral = P"-"^-1 * digits * (P"." * digits)^-1
+  local inline_placeable = P"{" * blank^-1 * (V"SelectExpression" + V"InlineExpression") * blank^-1 * P"}"
+  local block_placeable = blank_block * blank_inline^-1 * inline_placeable
+  local inline_text = text_char^1
+  local block_text = blank_block * blank_inline * indented_char * inline_text^-1
+  StringLiteral = P'"' * quoted_char^0 * P'"'
+  FunctionReference = V"Identifier" * V"CallArguments"
+  MessageReference = V"Identifier" * V"AttributeAccessor"^-1
+  TermReference = P"-" * V"Identifier" * V"AttributeAccessor"^-1 * V"CallArguments"^-1
+  VariableReference = P"$" * V"Identifier"
+  AttributeAccessor = P"." * V"Identifier"
+  NamedArgument = V"Identifier" * blank^-1 * P":" * blank^-1 * (V"StringLiteral" + V"NumberLiteral")
+  Argument = V"NamedArgument" + V"InlineExpression"
+  local argument_list = (V"Argument" * blank^-1 * P"," * blank^-1)^0 * V"Argument"^-1
+  CallArguments = blank^-1 * P"(" * blank^-1 * argument_list * blank^-1 * P")"
+  SelectExpression = V"InlineExpression" * blank^-1 * P"->" * blank_inline^-1 * variant_list
+  InlineExpression = V"StringLiteral" + V"NumberLiteral" + V"FunctionReference" + V"MessageReference" + V"TermReference" + V"VariableReference" + inline_placeable
+  PatternElement = Cg(C(inline_text + block_text + inline_placeable + block_placeable), "value")
+  Pattern = V"PatternElement"^1
+  Attribute = line_end * blank^-1 * P"." * V"Identifier" * blank_inline^-1 * "=" * blank_inline^-1 * V"Pattern"
+  local junk_line =  (1-line_end)^0 * (P"\n" + P(nulleof))
+  Junk = Cg(junk_line * (junk_line - P"#" - P"-" - R("az","AZ"))^0, "content")
+  local comment_char = any_char - line_end
+  CommentLine = Cg(P"###" + P"##" + P"#", "sigil") * (" " * Cg(C(comment_char^0), "content"))^-1 * line_end
+  Term = P"-" * V"Identifier" * blank_inline^-1 * "=" * blank_inline^-1 * V"Pattern" * V"Attribute"^0
+  Message = V"Identifier" * blank_inline^-1 * P"=" * blank_inline^-1 * ((V"Pattern" * V"Attribute"^0) + V"Attribute"^1)
+  Entry = (V"Message" * line_end) + (V"Term" * line_end) + V"CommentLine"
+  Resource = (V"Entry" + blank_block + V"Junk")^0 * (P(nulleof) + EOF"unparsable input")
+  START("Resource")
+end)
+-- luacheck: pop
+
+-- TODO: if this doesn't need any state information make in a function not a class
+local FluentParser = class({
+    _init = function (self, input)
+      return type(input) == "string" and self:parsestring(input) or error("unknown input type")
+    end,
+
+    addtrailingnewine = function(input)
+      local hasnulleof = epnf.parsestring(ftl_eof, input)
+      return type(hasnulleof) == "nil" and input..nulleof or input
+    end,
+
+    parsestring = function (self, input)
+      input = self.addtrailingnewine(input)
+      return epnf.parsestring(ftl_grammar, input)
+    end
+  })
+
+return FluentParser