Merge request for the updates since the original python3

Author: bjonnh

.travis.yml

@@ -6,7 +6,7 @@ python:
   - "3.3"
 
 install:
-  - "pip install pytest Twisted coveralls . --use-mirrors"
+  - "pip install pytest Twisted coveralls pytz . --use-mirrors"
 script:
   - "coverage run $(which trial) test_parsley"
   - "coverage run -a $(which trial) ometa"

examples/iso8601.py

@@ -0,0 +1,41 @@
+import datetime
+import pytz
+
+from parsley import makeGrammar
+
+# See www.ietf.org/rfc/rfc3339.txt
+
+_iso_8601_definition = r"""
+year = <digit{4}>:Y -> int(Y)
+month = <digit{2}>:m -> int(m)
+day = <digit{2}>:d -> int(d)
+
+hour = <digit{2}>:H -> int(H)
+minute = <digit{2}>:M -> int(M)
+second = <digit{2}>:S -> int(S)
+fraction = '.' <digit+>:frac -> int(float('0.' + frac) * 10 ** 6)
+
+sign = ('-' -> -1) | ('+' -> 1)
+numeric_offset = sign:s hour:h ':' minute:m -> FixedOffset(s * (h * 60 + m))
+utc = 'Z' -> UTC
+offset = utc | numeric_offset
+
+naive_time = hour:h ':' minute:m ':' second:s (fraction | -> 0):ms
+             -> time(h, m, s, ms)
+time = naive_time:t offset:o -> t.replace(tzinfo=o)
+date = year:y '-' month:m '-' day:d -> date(y, m, d)
+
+datetime = date:d 'T' time:t -> datetime.combine(d, t)
+"""
+
+
+DateTimeParser = makeGrammar(
+    _iso_8601_definition,
+    {
+        'FixedOffset': pytz.FixedOffset,
+        'date': datetime.date,
+        'time': datetime.time,
+        'datetime': datetime.datetime,
+        'UTC': pytz.UTC,
+    },
+)

examples/test_iso8601.py

@@ -0,0 +1,62 @@
+import datetime
+import unittest
+
+try:
+    import pytz
+    from iso8601 import DateTimeParser
+except ImportError:
+    skip = 'pytz is not installed or usable'
+else:
+    skip = None
+
+
+class TestDatetimeParsing(unittest.TestCase):
+    if skip is not None:
+        skip = skip
+
+    def test_date(self):
+        self.assertEqual(
+            datetime.date(2001, 12, 25),
+            DateTimeParser('2001-12-25').date())
+
+    def test_naive_time(self):
+        self.assertEqual(
+            datetime.time(13, 59, 43),
+            DateTimeParser('13:59:43').naive_time())
+
+    def test_fractional_naive_time(self):
+        self.assertEqual(
+            datetime.time(13, 59, 43, 880000),
+            DateTimeParser('13:59:43.88').naive_time())
+
+    def test_utc_time(self):
+        self.assertEqual(
+            datetime.time(13, 59, 43, tzinfo=pytz.UTC),
+            DateTimeParser('13:59:43Z').time())
+
+    def test_fractional_utc_time(self):
+        self.assertEqual(
+            datetime.time(13, 59, 43, 880000, tzinfo=pytz.UTC),
+            DateTimeParser('13:59:43.88Z').time())
+
+    def test_timezone_time(self):
+        self.assertEqual(
+            datetime.time(13, 59, 43, tzinfo=pytz.FixedOffset(60)),
+            DateTimeParser('13:59:43+01:00').time())
+
+    def test_fractional_timezone_time(self):
+        self.assertEqual(
+            datetime.time(13, 59, 43, 770000, tzinfo=pytz.FixedOffset(60)),
+            DateTimeParser('13:59:43.77+01:00').time())
+
+    def test_numeric_offset(self):
+        get_offset = lambda x: DateTimeParser(x).numeric_offset()
+        self.assertEqual(pytz.FixedOffset(0), get_offset('+00:00'))
+        self.assertEqual(pytz.FixedOffset(90), get_offset('+01:30'))
+        self.assertEqual(pytz.FixedOffset(-150), get_offset('-02:30'))
+
+    def test_datetime(self):
+        self.assertEqual(
+            datetime.datetime(
+                2001, 12, 25, 13, 59, 43, 770000, tzinfo=pytz.UTC),
+            DateTimeParser('2001-12-25T13:59:43.77Z').datetime())

ometa/interp.py

@@ -1,14 +1,4 @@
 import string
-
-try:
-    xrange
-except NameError:
-    xrange = range
-try:
-    basestring
-except NameError:
-    basestring = str
-
 from ometa.runtime import (InputStream, ParseError, EOFError, ArgInput,
                            joinErrors, expected, LeftRecursion)
 
@@ -30,13 +20,13 @@ class TrampolinedGrammarInterpreter(object):
     An interpreter for OMeta grammars that processes input
     incrementally.
     """
-    def __init__(self, grammar, ruleName, callback=None, globals=None):
+    def __init__(self, grammar, rule, callback=None, globals=None):
         self.grammar = grammar
         self.position = 0
         self.callback = callback
         self.globals = globals or {}
         self.rules = decomposeGrammar(grammar)
-        self.next_ = self.apply(ruleName, None, ())
+        self.next = self.setNext(rule)
         self._localsStack = []
         self.currentResult = None
         self.input = InputStream([], 0)
@@ -55,7 +45,7 @@ def receive(self, buf):
             raise ValueError("Can't feed a parser that's been ended.")
         self.input.data.extend(buf)
         x = None
-        for x in self.next_:
+        for x in self.next:
             if x is _feed_me:
                 return x
         if self.callback:
@@ -72,12 +62,18 @@ def end(self):
             return
         self.ended = True
         x = None
-        for x in self.next_:
+        for x in self.next:
             pass
         if self.callback:
             self.callback(*x)
 
 
+    def setNext(self, rule):
+        if not isinstance(rule, tuple):
+            rule = (rule, )
+        return self.apply(rule[0], None, rule[1:])
+
+
     ## Implementation note: each method, instead of being a function
     ## returning a value, is a generator that will yield '_feed_me' an
     ## arbitrary number of times, then finally yield the value of the
@@ -92,12 +88,8 @@ def _apply(self, rule, ruleName, args):
         @param args: A sequence of arguments to it.
         """
         if args:
-            if basestring is str:
-                attrname = '__code__'
-            else:
-                attrname = 'func_code'
-            if ((not getattr(rule, attrname, None))
-                 or getattr(rule, attrname).co_argcount - 1 != len(args)):
+            if ((not getattr(rule, 'func_code', None))
+                 or rule.func_code.co_argcount - 1 != len(args)):
                 for arg in args[::-1]:
                     self.input = ArgInput(arg, self.input)
                 g = rule()
@@ -162,10 +154,15 @@ def apply(self, ruleName, codeName, args):
         Invoke a rule, optionally with arguments.
         """
         argvals = []
-        for a in args:
-            for x in self._eval(a):
-                if x is _feed_me: yield x
-            argvals.append(x[0])
+        # we tell whether a rule is a manually set one by the codeName
+        # if it's None, then we think it's set by setNext
+        if codeName is None:
+            argvals = args
+        else:
+            for a in args:
+                for x in self._eval(a):
+                    if x is _feed_me: yield x
+                argvals.append(x[0])
         _locals = {}
         self._localsStack.append(_locals)
         try:
@@ -263,7 +260,7 @@ def parse_Or(self, expr):
                 self.currentError = joinErrors(errors)
                 yield x
                 return
-            except ParseError as err:
+            except ParseError, err:
                 errors.append(err)
                 self.input = i
         raise self.err(joinErrors(errors))
@@ -282,8 +279,7 @@ def parse_Many(self, expr, ans=None):
                     if x is _feed_me: yield x
                 ans.append(x[0])
                 self.currentError = x[1]
-            except ParseError as e:
-                err = e
+            except ParseError, err:
                 self.input = m
                 break
         yield ans, err
@@ -336,8 +332,7 @@ def parse_Repeat(self, min, max, expr):
                         if x is _feed_me: yield x
                     v, e = x
                     ans.append(v)
-                except ParseError as err:
-                    e = err
+                except ParseError, e:
                     self.input = m
                     break
         yield ans, e
@@ -384,10 +379,10 @@ def parse_Label(self, expr, label_term):
             for x in self._eval(expr):
                 if x is _feed_me:
                     yield x
-            print("^^ %s" % label)
+            print "^^", label
             self.currentError = x[1].withMessage([("Custom Exception:", label, None)])
             yield x[0], self.currentError
-        except ParseError as e:
+        except ParseError, e:
             raise self.err(e.withMessage([("Custom Exception:", label, None)]))
 
 
@@ -472,7 +467,7 @@ def rule_letter(self):
         except EOFError:
             yield _feed_me
             val, p = self.input.head()
-        if val in getattr(string, 'letters', string.ascii_letters):
+        if val in string.letters:
             self.input = self.input.tail()
             yield val, p
         else:
@@ -508,9 +503,6 @@ def __init__(self, grammar, base, globals=None):
         self.rules = {}
         self._localsStack = []
         self._globals = globals or {}
-        if basestring is str:
-            self._globals['basestring'] = str
-            self._globals['unichr'] = chr
         self.rules = decomposeGrammar(grammar)
         self.run = None
         self._spanStart = 0
@@ -562,7 +554,7 @@ def _eval(self, run, expr):
             try:
                 val, err = self._eval(run, args[0])
                 return val, err.withMessage([("Custom Exception:", label, None)])
-            except ParseError as e:
+            except ParseError, e:
                 raise e.withMessage([("Custom Exception:", label, None)])
 
         elif name == "Token":
@@ -578,8 +570,7 @@ def _eval(self, run, expr):
                     m = run.input
                     v, _ = self._eval(run, args[0])
                     ans.append(v)
-                except ParseError as e:
-                    err = e
+                except ParseError, err:
                     run.input = m
                     break
             return ans, err
@@ -607,8 +598,7 @@ def _eval(self, run, expr):
                     m = run.input
                     v, e = self._eval(run, args[2])
                     ans.append(v)
-                except ParseError as err:
-                    e = err
+                except ParseError, e:
                     run.input = m
                     break
             return ans, e
@@ -630,7 +620,7 @@ def _eval(self, run, expr):
                     ret, err = x
                     errors.append(err)
                     return ret, joinErrors(errors)
-                except ParseError as err:
+                except ParseError, err:
                     errors.append(err)
                     run.input = m
             raise joinErrors(errors)
@@ -640,7 +630,7 @@ def _eval(self, run, expr):
             m = run.input
             try:
                 self._eval(run, args[0])
-            except ParseError as err:
+            except ParseError, err:
                 run.input = m
                 return True, run.input.nullError()
             else:

ometa/test/test_tube.py

@@ -34,6 +34,7 @@ def setUp(self):
         _grammar =  r"""
             delimiter = '\r\n'
             initial = <(~delimiter anything)*>:val delimiter -> receiver.receive(val)
+            witharg :arg1 :arg2 = <(~delimiter anything)*>:a delimiter -> receiver.receive(arg1+arg2+a)
         """
         self.grammar = self._parseGrammar(_grammar)
 
@@ -75,3 +76,16 @@ def test_bindings(self):
         bindings = {'SMALL_INT': 3}
         TrampolinedParser(self._parseGrammar(grammar), receiver, bindings).receive('0')
         self.assertEqual(receiver.received, [3])
+
+
+    def test_currentRuleWithArgs(self):
+        """
+        TrampolinedParser should be able to invoke curruent rule with args.
+        """
+        receiver = TrampolinedReceiver()
+        receiver.currentRule = "witharg", "nice ", "day"
+        trampolinedParser = TrampolinedParser(self.grammar, receiver, {})
+        buf = b' oh yes\r\n'
+        for c in iterbytes(buf):
+            trampolinedParser.receive(c)
+        self.assertEqual(receiver.received, ["nice day oh yes"])

ometa/tube.py

@@ -26,7 +26,7 @@ def _setupInterp(self):
         'initial'.
         """
         self._interp = TrampolinedGrammarInterpreter(
-            grammar=self.grammar, ruleName=self.receiver.currentRule,
+            grammar=self.grammar, rule=self.receiver.currentRule,
             callback=None, globals=self.bindings)
 
 

terml/nodes.py

@@ -34,6 +34,10 @@ def __eq__(self, other):
             return False
 
 
+    def __hash__(self):
+        return hash((Term, self.tag, self.data, self.args))
+
+
     def __repr__(self):
         return "term('%s')" % (self._unparse(4).replace("'", "\\'"))
 
@@ -115,6 +119,9 @@ def __ne__(self, other):
     def __repr__(self):
         return "Tag(%r)" % (self.name,)
 
+    def __hash__(self):
+        return hash((Tag, self.name))
+
     def _unparse(self, indentLevel=0):
         return self.name
 

terml/test/test_terml.py

@@ -1,6 +1,6 @@
 from twisted.trial import unittest
 from ometa.runtime import ParseError
-from terml.nodes import Tag, Term, coerceToTerm, TermMaker
+from terml.nodes import Tag, Term, coerceToTerm, TermMaker, termMaker
 from terml.parser import TermLParser, character, parseTerm
 
 
@@ -156,3 +156,9 @@ def test_coerce(self):
             coerceToTerm({3: 4, "a": character('x'), (2, 3): [4, 5]}),
             parseTerm('{"a": \'x\', 3: 4, [2, 3]: [4, 5]}'))
 
+
+    def test_hash(self):
+        t = TermMaker()
+        a = t.Arbitrary('foo')
+        b = t.Arbitrary('foo')
+        self.assertEqual(hash(a), hash(b))