Merge pull request #10 from alerque/strip

Post-process element(s) to remove common indents and leading/trailing whitespace. Also adds support for formatting various types including parameter substitution for VariableReferences.

Author: alerque

fluent/parser.lua

@@ -68,7 +68,7 @@ local ftl_grammar = epnf.define(function (_ENV)
   PatternElement = Cg(C(inline_text + block_text), "value") + Cg(inline_placeable + block_placeable, "expression")
   Pattern = V"PatternElement"^1
   Attribute = line_end * blank^-1 * P"." * V"Identifier" * blank_inline^-1 * "=" * blank_inline^-1 * V"Pattern"
-  local junk_line = (1-line_end)^0 * (P"\n" + P(nulleof))
+  local junk_line = (1-P"\n"-P(nulleof))^0 * (P"\n" + P(nulleof))
   Junk = Cg(junk_line * (junk_line - P"#" - P"-" - R("az","AZ"))^0, "content")
   local comment_char = any_char - line_end
   CommentLine = Cg(P"###" + P"##" + P"#", "sigil") * (" " * Cg(C(comment_char^0), "content"))^-1 * line_end

fluent/resource.lua

@@ -4,7 +4,6 @@ local tablex = require("pl.tablex")
 
 local node_types = {}
 local node_to_type
-local dedent
 
 local FluentNode = class({
     discardable = false,
@@ -33,7 +32,13 @@ local FluentNode = class({
         self.value = node
       else
         if not self.elements then self.elements = {} end
-        table.insert(self.elements, node)
+        if #self.elements >= 1 then
+          if not self.elements[#self.elements]:append(node) then
+            table.insert(self.elements, node)
+          end
+        else
+          table.insert(self.elements, node)
+        end
       end
     end,
 
@@ -100,36 +105,60 @@ node_types.Pattern = class({
     _base = FluentNode,
     _init = function (self, node)
       self:super(node)
-      -- TODO: merge sequential mergables in elements
-      -- TODO: move dedent to here after merge?
+      self:dedent()
+    end,
+    dedent = function (self)
+      local mindent = function(node)
+        local indents = {}
+        if type(node.value) == "string" then
+          for indent in string.gmatch(node.value, "\n *%S") do
+            table.insert(indents, #indent-2)
+          end
+        end
+        return tablex.reduce(math.min, indents) or 0
+      end
+      local striplen = tablex.reduce(math.min, tablex.imap(mindent, self.elements)) or 0
+      local i, strippref = 1, "\n"
+      while i <= striplen do
+        strippref = strippref .. " "
+        i = i + 1
+      end
+      local strip = function(node, key, len)
+        if type(node.value) == "string" then
+          local value = string.gsub(node.value, "\r\n", "\n")
+          if len >= 1 then
+            value = string.gsub(value, strippref, "\n\n")
+          end
+          -- local function next_is_text
+          value = key == 1 and string.gsub(value, "^[\n ]+", "") or value
+          value = key == #self.elements and string.gsub(value, "[\n ]+$", "") or value
+          self.elements[key].value = value
+        end
+      end
+      tablex.foreachi(self.elements, strip, striplen)
     end,
     format = function (self, parameters)
-      local value = #self.elements >= 2 and dedent() or self.elements[1].value
-      -- Todo parse elements and actually format a value
+      local function evaluate (node) return node:format(parameters) end
+      local value = table.concat(tablex.map(evaluate, self.elements))
       return value, parameters
     end
   })
---   local lasttype = "none"
---   for key, value in ipairs(stuff) do
---     if lasttype == value.id then
---       ast.elements[#ast.elements] = ast.elements[#ast.elements].value .. value.value
---     else
---       table.insert(ast.elements, self(value))
---       lasttype = value.id
---     end
---   end
---   for key, value in ipairs(ast.elements) do
---     if key == "value" then
---       ast.elements[key] = dedent(value)
---     end
---   end
 
 node_types.TextElement = class({
     appendable = true,
     _base = FluentNode,
     _init = function (self, node)
       node.id = "TextElement"
       self:super(node)
+    end,
+    __add = function (self, node)
+      if self:is_a(node:is_a()) and self.appendable and node.appendable then
+        self.value = (self.value or "") .. "\n" .. (node.value or "")
+        return self
+      end
+    end,
+    format = function (self)
+      return self.value
     end
   })
 
@@ -142,6 +171,9 @@ node_types.Placeable = class({
       if node.expression then
         self.expression = node_to_type(node.expression[1])
       end
+    end,
+    format = function (self, parameters)
+      return self.expression:format(parameters)
     end
   })
 
@@ -157,20 +189,39 @@ node_types.StringLiteral = class({
     _base = FluentNode,
     _init = function (self, node)
       self:super(node)
+    end,
+    format = function (self)
+      return self.value
     end
   })
 
 node_types.NumberLiteral = class({
     _base = FluentNode,
     _init = function (self, node)
       self:super(node)
+    end,
+    format = function (self)
+      return self.value
     end
   })
 
 node_types.VariableReference = class({
     _base = FluentNode,
     _init = function (self, node)
       self:super(node)
+    end,
+    format = function (self, parameters)
+      return parameters[self.id.name]
+    end
+  })
+
+node_types.MessageReference = class({
+    _base = FluentNode,
+    _init = function (self, node)
+      self:super(node)
+    end,
+    format = function (self)
+      return self.value
     end
   })
 
@@ -235,25 +286,6 @@ node_to_type = function (node)
   end
 end
 
-dedent = function (content)
-  local min
-  for indent in string.gmatch(content, "\n *%S") do
-    min = min and math.min(min, #indent) or #indent
-  end
-  local common = function(shortest)
-    local i = 0
-    local s = ""
-    while i < shortest do
-      s = s .. " "
-      i = i + 1
-    end
-    return s
-  end
-  local sp = common(min-2)
-  local rep = string.gsub(content, "\n"..sp, "\n")
-  return rep
-end
-
 local FluentResource = class({
     type = "Resource",
     index = {},

spec/fixtures_spec.lua

@@ -41,7 +41,7 @@ describe('upstream reference fixture', function ()
         -- or fname:match("/messages.ftl$")
         -- or fname:match("/mixed_entries.ftl$")
         -- or fname:match("/multiline_values.ftl$")
-        -- or fname:match("/numbers.ftl$")
+        or fname:match("/numbers.ftl$")
         or fname:match("/obsolete.ftl$")
         -- or fname:match("/placeables.ftl$")
         -- or fname:match("/reference_expressions.ftl$")
@@ -53,7 +53,7 @@ describe('upstream reference fixture', function ()
         -- or fname:match("/terms.ftl$")
         or fname:match("/variables.ftl$")
         -- or fname:match("/variant_keys.ftl$")
-        -- or fname:match("/whitespace_in_value.ftl$")
+        or fname:match("/whitespace_in_value.ftl$")
         or fname:match("/zero_length.ftl$")
       ) then
       describe(object, function ()
@@ -119,9 +119,9 @@ describe('upstream structure fixture', function ()
         -- or fname:match("/message_with_empty_multiline_pattern.ftl")
         -- or fname:match("/message_with_empty_pattern.ftl")
         or fname:match("/multiline-comment.ftl")
-        -- or fname:match("/multiline_pattern.ftl")
+        or fname:match("/multiline_pattern.ftl")
         or fname:match("/multiline_string.ftl")
-        -- or fname:match("/multiline_with_non_empty_first_line.ftl")
+        or fname:match("/multiline_with_non_empty_first_line.ftl")
         -- or fname:match("/multiline_with_placeables.ftl")
         -- or fname:match("/non_id_attribute_name.ftl")
         -- or fname:match("/placeable_at_eol.ftl")
@@ -153,8 +153,8 @@ describe('upstream structure fixture', function ()
         -- or fname:match("/variant_with_empty_pattern.ftl")
         -- or fname:match("/variant_with_leading_space_in_name.ftl")
         or fname:match("/variant_with_symbol_with_space.ftl")
-        -- or fname:match("/whitespace_leading.ftl")
-        -- or fname:match("/whitespace_trailing.ftl")
+        or fname:match("/whitespace_leading.ftl")
+        or fname:match("/whitespace_trailing.ftl")
       ) then
       describe(object, function ()
         local ftl = filetostring(fname)

spec/fluent_spec.lua

@@ -28,6 +28,18 @@ describe('fluent.bundle', function ()
     assert.same("baz", en:format("bar"))
   end)
 
+  it('should parse and format literals', function ()
+    local en = FluentBundle("en-US")
+    en:add_messages('foo = bar {"baz"} quz {-3.14}')
+    assert.same("bar baz quz -3.14", en:format("foo"))
+  end)
+
+  it('should parse and format a variable substitution', function ()
+    local en = FluentBundle("en-US")
+    en:add_messages('foo = bar { $baz }')
+    assert.same("bar qux", en:format("foo", { baz = "qux" }))
+  end)
+
   it('should keep locale instances separate', function ()
     local en = FluentBundle("en-US")
     local tr = FluentBundle("tr-TR")