Merge branch 'document-protocol-parsing', remote-tracking branches 'eevee/expr-cache' and 'eevee/more-faster'

Author: habnabit

ometa/builder.py

@@ -32,6 +32,7 @@ def __init__(self, tree, grammarText):
         self.tree = tree
         self.grammarText = grammarText
         self.gensymCounter = 0
+        self.compiledExprCache = None
 
 
     def _generate(self, out, expr, retrn=False, debugname=None):
@@ -109,9 +110,19 @@ def compilePythonExpr(self, out, expr, debugname=None):
             ast.literal_eval(expr)
             return self._expr(out, 'python', '(' + expr + '), None', debugname)
         except ValueError:
-            return self._expr(out, 'python',
-                              'eval(%r, self.globals, _locals), None' % (expr,),
-                              debugname)
+            if self.compiledExprCache is None:
+                return self._expr(out, 'python',
+                                'eval(%r, self.globals, _locals), None' % (expr,),
+                                debugname)
+            else:
+                if expr in self.compiledExprCache:
+                    sym = self.compiledExprCache[expr]
+                else:
+                    sym = self.compiledExprCache[expr] = self._gensym('expr')
+
+                return self._expr(out, 'python',
+                                'eval(self.%s, self.globals, _locals), None' % (sym,),
+                                debugname)
 
     def _convertArgs(self, out, rawArgs, debugname):
         return [self._generateNode(out, x, debugname) for x in rawArgs]
@@ -325,6 +336,7 @@ def generate_Rule(self, prevOut, name, expr, debugname=None):
 
     def generate_Grammar(self, out, name, takesTreeInput, rules,
                          debugname=None):
+        self.compiledExprCache = {}
         self.takesTreeInput = takesTreeInput.tag.name == 'true'
         out.writeln("def createParserClass(GrammarBase, ruleGlobals):")
         funcOut = out.indent()
@@ -338,6 +350,8 @@ def generate_Grammar(self, out, name, takesTreeInput, rules,
             out.writeln("")
         if self.takesTreeInput:
             out.writeln("tree = %s" % self.takesTreeInput)
+        for expr, sym in self.compiledExprCache.items():
+            out.writeln("%s = compile(%r, '<string>', 'eval')" % (sym, expr))
         funcOut.writeln(
             "if %s.globals is not None:" % (name.data,))
         out.writeln("%s.globals = %s.globals.copy()" % (name.data,
@@ -347,6 +361,7 @@ def generate_Grammar(self, out, name, takesTreeInput, rules,
             "else:")
         out.writeln("%s.globals = ruleGlobals" % (name.data,))
         funcOut.writeln("return " + name.data)
+        self.compiledExprCache = None
 
 
 class _Term2PythonAction(object):

ometa/runtime.py

@@ -2,9 +2,7 @@
 """
 Code needed to run a grammar after it has been compiled.
 """
-import string
 import time
-import operator
 from textwrap import dedent
 from terml.nodes import coerceToTerm, Term, termMaker as t
 from ometa.builder import moduleFromGrammar, writePython
@@ -16,17 +14,10 @@ class ParseError(Exception):
     ?Redo from start
     """
 
-    @property
-    def position(self):
-        return self.args[0]
-
-
-    @property
-    def error(self):
-        return self.args[1]
-
     def __init__(self, input, position, message, trail=None):
         Exception.__init__(self, position, message)
+        self.position = position
+        self.error = message or []
         self.input = input
         self.trail = trail or []
 
@@ -36,8 +27,17 @@ def __eq__(self, other):
             return (self.position, self.error) == (other.position, other.error)
 
 
+    def mergeWith(self, other):
+        """
+        Merges in another error's error and trail.
+        """
+        self.error = list(set(self.error + other.error))
+        self.args = (self.position, self.error)
+        self.trail = other.trail or self.trail or []
+
+
     def formatReason(self):
-        if self.error is None:
+        if not self.error:
             return "Syntax error"
         if len(self.error) == 1:
             if self.error[0][0] == 'message':
@@ -129,19 +129,26 @@ def joinErrors(errors):
     """
     Return the error from the branch that matched the most of the input.
     """
-    errors.sort(reverse=True, key=operator.itemgetter(0))
+    if len(errors) == 1:
+        return errors[0]
+
+    highestPos = -1
     results = set()
-    pos = errors[0].position
     trail = None
+
     for err in errors:
-        if pos == err.position:
-            e, trail = err.error, (err.trail or trail)
-            if e is not None:
-                for item in e:
-                        results.add(item)
+        pos = err.position
+        if pos < highestPos:
+            continue
+        elif pos > highestPos:
+            highestPos = pos
+            trail = err.trail or None
+            results = set(err.error)
         else:
-            break
-    return ParseError(errors[0].input,  pos, list(results) or None, trail)
+            trail = err.trail or trail
+            results.update(err.error)
+
+    return ParseError(errors[0].input, highestPos, list(results), trail)
 
 
 class character(str):
@@ -389,10 +396,12 @@ def __init__(self, input, globals=None, name='<string>', tree=False,
 
     def considerError(self, error, typ=None):
         if error:
-            if error[0] > self.currentError[0]:
+            newPos = error.position
+            curPos = self.currentError.position
+            if newPos > curPos:
                 self.currentError = error
-            elif error[0] == self.currentError[0]:
-                self.currentError = joinErrors([error, self.currentError])
+            elif newPos == curPos:
+                self.currentError.mergeWith(error)
 
 
     def _trace(self, src, span, inputPos):
@@ -590,7 +599,7 @@ def _not(self, fn):
         m = self.input
         try:
             fn()
-        except ParseError, e:
+        except ParseError:
             self.input = m
             return True, self.input.nullError()
         else:
@@ -606,9 +615,9 @@ def eatWhitespace(self):
                 c, e = self.input.head()
             except EOFError, e:
                 break
-            t = self.input.tail()
+            tl = self.input.tail()
             if c.isspace():
-                self.input = t
+                self.input = tl
             else:
                 break
         return True, e
@@ -843,7 +852,7 @@ def parseGrammar(self, name):
         self.name = name
         res, err = self.apply("grammar")
         try:
-            x = self.input.head()
+            self.input.head()
         except EOFError:
             pass
         else: