Merge pull request #8 from Vizonex/protocol-features

add integer parsing based off nodejs/llparse#32 And Fix Pausing

Author: Vizonex

llparse/compilator.py

@@ -337,7 +337,7 @@ def tailTo(
         out: list[str],
         node: IWrap[_frontend.node.Node],
         noAdvance: bool,
-        value: Optional[int],
+        value: Optional[int] = None,
     ):
         ctx = self.compilation
         target = ctx.unwrapNode(node).build(ctx)
@@ -501,7 +501,7 @@ def doBuild(self, out: list[str]):
 
         assert self.ref.otherwise
         otherwise = ctx.unwrapNode(self.ref.otherwise.node)
-        out.append(f"{ctx.currentField()} = (void*) (intptr_t) {otherwise};")
+        out.append(f"{ctx.currentField()} = (void*) (intptr_t) {otherwise.cachedDecel};")
         out.append(f"return {STATE_ERROR};")
 
 
@@ -512,7 +512,8 @@ def __init__(self, ref: _frontend.node.Sequence) -> None:
 
     def doBuild(self, out: list[str]):
         ctx = self.compilation
-
+        # TODO: llparse_match_t could be easily changed around to 
+        # Something that can't be overlapped with when compiled with other parsers...
         out.append("llparse_match_t match_seq;")
         out.append("")
 
@@ -639,7 +640,7 @@ def doBuild(self, out: list[str]):
         # Invoke callback
         callback = ctx.buildCode(ctx.unwrapCode(self.ref.callback, True))
 
-        out.append(f"err = {callback}({ctx.stateArg()}, start,{ctx.posArg()});")
+        out.append(f"err = {callback}({ctx.stateArg()}, start, {ctx.posArg()});")
 
         out.append("if (err != 0) {")
         tmp = []
@@ -676,6 +677,163 @@ def buildError(self, out: list[str], code: str):
         out.append(f"return {STATE_ERROR};")
 
 
+# Based off arthurschreiber's work with Indutny's Tips and requests added to the mix.
+
+# 0x80 I8
+# 0x8000 I16
+# 0x800000 I24
+# 0x1000000 U24
+
+class Int(Node):
+    def __init__(self, ref: _frontend.node.Int):
+        super().__init__(ref)
+        self.ref = ref
+        self.offset = ref.byteOffset
+    # I'm going to deviate from arthurschreiber's work a bit with indutny's suggestions.
+    # we should really be using bitwise operators like rshift and lshift
+    @property
+    def pair(self):
+        return self.compilation, self.compilation.stateField(self.ref.field)
+
+    def readInt8(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        out.append(f"{index} = ((*{ctx.posArg()}) & 0x80);")
+    
+    def readUInt8(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        out.append(f"{index} = (*{ctx.posArg()});")
+    
+    # LITTLE ENDIAN
+
+    def readInt16LE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            # Since BE Belongs to performing << aka left shifts we do >> right shifts 
+            out.append(f"{index} = ({index} >> 8) | ((*{ctx.posArg()}) & 0x80);")
+
+    def readUInt16LE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} >> 8) | (*{ctx.posArg()});")
+
+    def readInt24LE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        elif self.offset == 1:
+            out.append(f"{index} = ({index} >> 8) | (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} >> 8) | ((*{ctx.posArg()}) & 0x80);")
+
+    def readUInt24LE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} >> 8) | (*{ctx.posArg()});")
+ 
+    def readInt32LE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        elif self.offset in (1, 2):
+            out.append(f"{index} = ({index} >> 8) | (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} >> 8) | ((*{ctx.posArg()}) & 0x80);")
+
+    def readUInt32LE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} >> 8) | (*{ctx.posArg()});")
+    
+    # BIG ENDIAN 
+    
+    def readInt16BE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            # Since LE Belongs to >> we do "<<" instead 
+            out.append(f"{index} = ({index} << 8) | ((*{ctx.posArg()}) & 0x80);")
+    
+    def readUInt16BE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} << 8) | (*{ctx.posArg()});")
+
+    def readInt24BE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        elif self.offset == 1:
+            out.append(f"{index} = ({index} << 8) | (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} << 8) | ((*{ctx.posArg()}) & 0x80);")
+
+    def readUInt24BE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} << 8) | (*{ctx.posArg()});")
+ 
+    def readInt32BE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        elif self.offset in (1, 2):
+            out.append(f"{index} = ({index} << 8) | (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} << 8) | ((*{ctx.posArg()}) & 0x80);")
+
+    def readUInt32BE(self, out: list[str]) -> None:
+        ctx, index = self.pair
+        if self.offset == 0:
+            out.append(f"{index} = (*{ctx.posArg()});")
+        else:
+            out.append(f"{index} = ({index} << 8) | (*{ctx.posArg()});")
+ 
+        
+    def doBuild(self, out:list[str]):
+        self.prologue(out)
+        # I'm still supporting 3.9 but I plan to drop it's support in favor of match case soon...
+        bits = self.ref.bits
+        
+        if self.compilation.getFieldType(self.ref.field) == 'ptr':
+            raise ValueError(f'property {self.ref.field} should not use pointers but it was given \"ptr\"')
+
+        if bits == 1:
+            self.readInt8(out) if self.ref.signed else self.readUInt8(out)
+        elif bits == 2:
+            if self.ref.littleEndian:
+                self.readInt16LE(out) if self.ref.signed else self.readUInt16LE(out)
+            else:
+                self.readInt16BE(out) if self.ref.signed else self.readUInt16BE(out)
+        elif bits == 3:
+            if self.ref.littleEndian:
+                self.readInt24LE(out) if self.ref.signed else self.readUInt24LE(out)
+            else:
+                self.readInt24BE(out) if self.ref.signed else self.readUInt24BE(out)
+        else:
+            if self.ref.littleEndian:
+                self.readInt32LE(out) if self.ref.signed else self.readUInt32LE(out)
+            else:
+                self.readInt32BE(out) if self.ref.signed else self.readUInt32BE(out)
+        # TODO: uint64 & int64
+        
+        self.tailTo(out, self.ref.otherwise.node, self.ref.otherwise.noAdvance, None)
+
+
+
+
 MAX_CHAR = 0xFF
 TABLE_GROUP = 16
 
@@ -1077,13 +1235,14 @@ def unwrapNode(self, node: IWrap[_frontend.node.Node]):
             r = Consume(ref)
         elif isinstance(ref, _frontend.node.Empty):
             r = Empty(ref)
+        elif isinstance(ref, _frontend.node.Pause):
+            r = Pause(ref)
+
         elif isinstance(ref, _frontend.node.Error):
             r = Error(ref)
         elif isinstance(ref, _frontend.node.Invoke):
             r = Invoke(ref)
-        elif isinstance(ref, _frontend.node.Pause):
-            r = Pause(ref)
-
+        
         elif isinstance(ref, _frontend.node.SpanStart):
             r = SpanStart(ref)
 
@@ -1096,6 +1255,8 @@ def unwrapNode(self, node: IWrap[_frontend.node.Node]):
             r = Sequence(ref)
         elif isinstance(ref, _frontend.node.TableLookup):
             r = TableLookup(ref)
+        elif isinstance(ref, _frontend.node.Int):
+            r = Int(ref)
         else:
             raise TypeError(
                 f'refrence "{ref}" is an Invalid Code Type , TypeName:"{ref.__class__.__name__}"'

llparse/frontend.py

@@ -211,7 +211,7 @@ def ID():
             result = nodeImpl.Error(_frontend.node.Error(ID(), node.code, node.reason))
 
         elif isinstance(node, source.code.Pause):
-            result = nodeImpl.Pause(_frontend.node.Error(ID(), node.code, node.reason))
+            result = nodeImpl.Pause(_frontend.node.Pause(ID(), node.code, node.reason))
 
         elif isinstance(node, source.code.Comsume):
             result = nodeImpl.Consume(_frontend.node.Consume(ID(), node.field))
@@ -244,31 +244,37 @@ def ID():
 
         elif isinstance(node, source.code.Match):
             result = self.translateMatch(node)
+
+        elif isinstance(node, source.node.Int):
+            result = self.translateInt(node)
+
         else:
             raise Exception(f'Unknown Node Type for :"{node.name}" {type(node)}')
 
         otherwise = node.getOtherwiseEdge()
 
         if isinstance(result, list):
             # result:list[WrappedNode]
-            assert isinstance(node, source.code.Match)
-            _match = node
 
-            if not otherwise:
-                raise Exception(f'Node "{node.name}" has no ".otherwise()"')
+            assert isinstance(node, (source.code.Match, source.node.Int))
+            _match = node
+        
+            assert otherwise, (f'Node "{node.name}" has no ".otherwise()"')
 
-            else:
+            if isinstance(node, source.node.Match):
                 for child in result:
                     if not child.ref.otherwise:
                         child.ref.setOtherwise(
                             self.translate(otherwise.node), otherwise.noAdvance
                         )
+                transform = self.translateTransform(_match.getTransform())
+                for child in result:
+                    # TODO Vizonex : This might break , be sure to make a workaround function here...
+                    child.ref.setTransform(transform)
 
-            transform = self.translateTransform(_match.getTransform())
-            for child in result:
-                # TODO Vizonex : This might break , be sure to make a workaround function here...
-                child.ref.setTransform(transform)
-
+            
+            else:
+                result[-1].ref.setOtherwise(self.translate(otherwise.node), otherwise.noAdvance)
             assert len(result) >= 1
             return result[0]
 
@@ -299,6 +305,23 @@ def ID():
                 assert len(list(node)) == 0
 
             return single
+        
+    def translateInt(self, node: source.node.Int) -> list[IWrap[_frontend.node.Int]]:
+        inner = _frontend.node.Int(self.Id.id(node.name), node.field, node.bits, node.signed, node.little_endian, 0)
+        result = [self.implementation.node.Int(inner)]
+        # front is to avoid overlapping with python's functions (aka next)
+        front = self.Map[node] = result[0]
+        
+        for offset in range(1, node.bits):
+            unique_name = self.Id.id(f"{node.name}_byte{offset + 1}")
+            inner = _frontend.node.Int(unique_name, node.field, node.bits, node.signed, node.little_endian, offset)
+            outer = self.implementation.node.Int(inner)
+            result.append(outer)
+            # Integers will advance since they are unpacking values...
+            front.ref.setOtherwise(outer, False)
+            front = result[-1]
+        return result
+
 
     def maybeTableLookup(
         self, node: source.code.Match, trie: TrieSingle, children: MatchChildren

llparse/pybuilder/__init__.py

@@ -1,2 +1,8 @@
 from ..pybuilder.builder import *
 from ..pybuilder.loopchecker import *
+from ..pybuilder.main_code import (
+# I'll add more soon I feel a little lazy at the moment.
+    Node,
+    Match,
+    Int
+)

llparse/pybuilder/builder.py

@@ -1,7 +1,8 @@
 from typing import Literal, Optional, Union
-
 from ..pybuilder import main_code as code
 
+# typehinting node and code (TODO: Vizonex) Lets seperate the modules soon...
+node = code
 # from pydot import graph_from_dot_data
 
 
@@ -316,3 +317,38 @@ def property(self, ty: Literal["i8", "i16", "i32", "i64", "ptr"], name: str):
     def properties(self) -> list[Property]:
         """Return list of all allocated properties in parser's state."""
         return list(self.privProperties.values())
+
+    def intBE(self, field: str, bits:int):
+        """
+        :param field: State's property name
+        :param bits: Number of bits to use
+        """
+        return code.Int(field, bits, True, False)
+
+    def intLE(self, field: str, bits: int):
+        """
+        return a node for unpacking arrays to integers
+
+        :param field: State's property name
+        :param bits: Number of bits to use
+        """
+        return code.Int(field, bits, True, True)
+
+    def uintBE(self, field: str, bits: int):
+        """
+        return a node for unpacking arrays to integers
+        
+        :param field: State's property name
+        :param bits: Number of bits to use
+        """
+        return code.Int(field, bits, False, False)
+
+    def uintLE(self, field: str, bits: int):
+        """
+        return a node for unpacking arrays to integers
+        
+        :param field: State's property name
+        :param bits: Number of bits to use
+        """
+        return code.Int(field, bits, False, True)
+