Merge branch 'feature-fragment-size'; Version 4.4.1

Author: pjkundert

server/enip/device.py

@@ -201,18 +201,28 @@ def resolve( path, attribute=False ):
     Other valid paths segments (eg. {'port':...}, {'connection':...}) are not presently usable in
     our Controller communication simulation.
 
-    Call with attribute=True to force resolving to the Attribute level; otherwise, always returns
-    None for the attribute.
+    Call with attribute=<Truthy> to force resolving to the Attribute level; otherwise, always returns
+    None for the attribute.  If an attribute is required, but is not supplied in the path, then we
+    can default one.  This is an (unexpected) feature of the C*Logix PLCs; you can do a Read Tag
+    Fragmented asking for eg.  @0x6b/0x0008[0] (a Class, Instance and Element but no Attribute!).
+    Presumably there is an implied default Attribute number; we're guessing it's 1?  Undocumented.
 
     """
 
     result			= { 'class': None, 'instance': None, 'attribute': None }
     tag				= '' # developing symbolic tag "Symbol.Subsymbol"
 
     for term in path['segment']:
-        if ( result['class'] is not None and result['instance'] is not None
-             and ( not attribute or result['attribute'] is not None )):
-            break # All desired terms specified; done! (ie. ignore 'element')
+        if ( result['class'] is not None		# Got Class already
+             and result['instance'] is not None		# Got Instance already
+             and (
+                 result['attribute'] is not None	# Got Attribute already
+                 or not attribute			#   or no Attribute desired (must return None)
+                 or ( attribute is not True             #   or a default attribute is supplied
+                      and 'attribute' not in term )     #     and the term didn't contain a supplied one
+             )
+            ):
+            break # All desired terms specified; done! (ie. ignore subsequent 'element')
         working			= dict( term )
         while working:
             # Each term is something like {'class':5}, {'instance':1}, or (from symbol table):
@@ -240,6 +250,11 @@ def resolve( path, attribute=False ):
     assert not tag, \
         "Unrecognized symbolic name %r found in path %r" % ( tag, path['segment'] )
 
+    # Handle the case where a default Attribute value was provided, and none was supplied
+    if result['attribute'] is None and attribute and attribute is not True:
+        assert isinstance( attribute, int )
+        result['attribute']	= attribute
+    # Make sure we got everything we required
     assert ( result['class'] is not None and result['instance'] is not None
              and ( not attribute or result['attribute'] is not None )), \
         "Failed to resolve required Class (%r), Instance (%r) %s Attribute(%r) from path: %r" % (

server/enip/logix.py

@@ -136,10 +136,12 @@ class Logix( Message_Router ):
 
     """
 
-    # TODO: MAX_BYTES is arbitrary.  We're supposed to be able to return data sufficient to fill the
+    # MAX_BYTES is arbitrary.  We're supposed to be able to return data sufficient to fill the
     # remaining reply package size, but how can we do that?  We'd have to be informed of the
-    # remaining packet size available, as an argument to the produce method...
-    MAX_BYTES			= 500
+    # remaining packet size available, as an argument to the produce method...  The user may alter
+    # this limit (eg. according to the capacity of Forward Open channel).  However, it is still
+    # quite fragile, especially when producing Multiple Service Packet requests/responses.
+    MAX_BYTES			= 488 # Wild guess; leave room for ENIP headers, etc. in reply
 
     RD_TAG_NAM			= "Read Tag"
     RD_TAG_CTX			= "read_tag"
@@ -192,8 +194,7 @@ def reply_elements( self, attribute, data, context ):
         off			= 0
         if data.service in (self.RD_FRG_RPY, self.WR_FRG_RPY):
             off			= data[context].get( 'offset' ) or 0 # nonexistent/None/0 --> 0
-        assert siz and off % siz == 0, \
-            "Requested byte offset %d is not on a %d-byte data element boundary" % ( off, siz )
+        max_size		= data[context].get( 'max_size' ) or self.MAX_BYTES
 
         # Compute the extents of the full reply, given no byte offset, unlimited reply size and
         # complete data.  If no 'elements' has been provided (only possible when hand-forming a
@@ -203,8 +204,6 @@ def reply_elements( self, attribute, data, context ):
         cnt			= len( attribute )
         elm			= data[context].get( 'elements', cnt - beg ) # Read/Write Tag defaults to all
         endactual		= beg + elm
-        assert 0 < endactual <= cnt, \
-            "Attribute %s ending element invalid: %r" % ( attribute, (beg, endactual) )
 
         # Maximum elements for read is the capacity of the reply message, for write is the number
         # actually provided in request.  Compute this from the beginning element deduced from the
@@ -213,31 +212,42 @@ def reply_elements( self, attribute, data, context ):
         # incomplete number of data elements provided.  The 'end' can only get smaller than the
         # (known valid) 'endactual'.
         # 
-        # TODO: This is not strictly correct; we must be able to return a response w/ an offset that
-        # starts mid-element!  Also, for STRUCT types, the size of each UDT element (eg. 600 bytes)
-        # can be greater than the maximum connection size, preventing even one complete element from
-        # being returned.  For now, always round up to return a complete element; this may prevent
-        # some clients from accepting the data (eg. if the elements are too big, and a Small Forward
-        # Open is used.
-        begadvance		= off // siz
-        log.detail( "index: {index!r} beg: {beg}, cnt: {cnt}, elm: {elm}; endactual: {endactual}, begadvance: {begadvance}".format(
+        # This is not strictly correct; we must be able to return a response w/ an offset that
+        # starts/ends arbitrarily mid-element!  Also, for STRUCT types, the size of each UDT element
+        # (eg. 600 bytes) can be greater than the maximum connection size, preventing even one
+        # complete element from being returned.  For this function, always round outwards to return
+        # complete element(s); this will prevent some clients from accepting the data (eg. if the
+        # elements are too big, and a Small Forward Open is used; it is assumed that the response
+        # data will be trimmed elsewhere, according to the actual byte offset requested in the Read
+        # Tag Fragmented request.
+        begadvance		= off // siz # Rounds down to the start of the element at offset
+        offremains		= off - begadvance * siz # off is how many bytes into beg element?
+        log.info( "index: {index!r} beg: {beg}, cnt: {cnt}, elm: {elm}; endactual: {endactual}, begadvance: {begadvance}".format(
             index=index, beg=beg, cnt=cnt, elm=elm, endactual=endactual, begadvance=begadvance ))
-        assert begadvance * siz == off, \
-            "Fragment offset {off} is not on an element {siz}-byte boundary".format( off=off, siz=siz )
         beg		       += begadvance
         if data.service in (self.RD_TAG_RPY, self.RD_FRG_RPY):
-            endmax 		= beg + max( self.MAX_BYTES // siz, 1 )
+            # Return at least enough elements to satisfy max_size, beginning at offset 'off'.  We
+            # have a 'beg' Element that contains the first byte at offset 'off'; compute the endmax
+            # that contains the last byte at offset off+max_siz-1.  The data may specify the
+            # (remaining) .max_size payload available.
+            endadv		= max(( offremains + max_size + siz - 1 ) // siz, 1 ) # rounds up
+            endmax 		= beg + endadv
         else:
-            endmax		= beg + len( data[context].data )
+            endadv		= len( data[context].data )
+            endmax		= beg + endadv
             assert endmax <= endactual, \
                 "Attribute %s capacity exceeded; writing %d elements beginning at index %d" % (
                     attribute, len( data[context].data ), beg )
         end			= min( endactual, endmax )
+        log.info( "offset: {off:6d} siz: {siz:3d}, beg: {beg:3d}, endadv: {endadv:3d}, end: {end:3d}, endmax: {endmax:3d}, offremains: {offremains}".format(
+            off=off, siz=siz, beg=beg, end=end, endadv=endadv, endmax=endmax, offremains=offremains ))
         assert 0 <= beg < cnt, \
             "Attribute %r initial element invalid: %r" % ( attribute, (beg, end) )
+        assert elm <= cnt, \
+            "Attribute %r elements requested invalid: %r" % ( attribute, elm )
         assert beg < end, \
             "Attribute %r ending element before beginning: %r" % ( attribute, (beg, end) )
-        return (beg,end,endactual)
+        return (beg,end,endactual,offremains,max_size)
 
     def request( self, data, addr=None ):
         """Any exception should result in a reply being generated with a non-zero status."""
@@ -308,7 +318,7 @@ def request( self, data, addr=None ):
             # We need to find the attribute for all requests, and it better be ours!
             data.status		= 0x05 # On Failure: Request Path destination unknown
             data.status_ext	= {'size': 1, 'data':[0x0000]}
-            clid, inid, atid	= resolve( data.path, attribute=True )
+            clid, inid, atid	= resolve( data.path, attribute=1 ) # eg. @<cls>/<ins>[<elm>] defaults to Attribute 1!
             attribute		= lookup( clid, inid, atid )
             assert clid == self.class_id and inid == self.instance_id, \
                 "Path %r processed by wrong Object %r" % ( data.path['segment'], self )
@@ -384,24 +394,45 @@ def request( self, data, addr=None ):
             data.status_ext	= {'size': 1, 'data': [ 0x2105 ]} # Number of elements beyond end of tag
 
             # Compute (beg,end] for this reply, given data.path...element, data.elements/offset.
-            # The end element of the full request (not the size/data-limited end) is in endactual
-            beg,end,endactual	= self.reply_elements( attribute, data, context )
+            # The end element of the original request (not the size/data-limited end) is in
+            # endactual.  If a .offset (and optionally .max_size) is provided, these must be used to
+            # constrain the actual payload bytes returned; the [beg,end) should index elements
+            # containing the first byte to return (at .offset), up tothe last byte
+            # (.offset+.max_size-1).  Since we might have advances 'beg', we get back the adjusted
+            # 'offremains', as well the target 'max_size'.
+            beg,end,endactual,offremains,max_size \
+                                        = self.reply_elements( attribute, data, context )
             log.debug( "Replying w/ elements [%3d-%-3d/%3d] for %r", beg, end, endactual, data )
             if data.service in (self.RD_TAG_RPY, self.RD_FRG_RPY):
                 # Read Tag [Fragmented]
                 recs			= attribute[beg:end]
                 if attribute.parser.tag_type == STRUCT.tag_type:
                     # Render the STRUCT UDTs to binary.  Assume that each record has its data.input
-                    # representation
+                    # representation.
                     input		= b''
                     for r in recs:
                         input	       += octets_encode( r.data.input )
-                    recs		= dict( input=input )
+                    # For STRUCTs *only*, we support arbitrary .offset and max_size; trim it
+                    # here. (For other basic data types, we'll simply return the designated
+                    # elements, which may be less than, or slightly more than the .max_size /
+                    # self.MAX_BYTES by some portion of one element size) Trim it here.  If we've
+                    # returned the end element of the request, and all of its bytes, we're complete.
+                    trimmed		= input[offremains:offremains+max_size]
+                    recs		= dict( input=trimmed )
+                    completed		= end == endactual and offremains+max_size >= len( input )
+                else:
+                    # We don't presently support a non-zero .offset for indeterminately sized types
+                    # (eg. STRING/SSTRING, etc.), or a sub-element offset for basic data types.
+                    assert offremains == 0 or (
+                        attribute.parser.tag_type < STRING.tag_type
+                        and offremains % attribute.parser.struct_calcsize == 0 )
+                    completed		= end == endactual
                 data[context].data	= recs
-                log.detail( "%s Reading %3d elements %3d-%3d from %s: %r",
-                            self, end - beg, beg, end-1, attribute, data[context].data )
+                log.detail( "%s Reading %3d elements %3d-%3d %s from %s: %r",
+                            self, end - beg, beg, end-1, "(done)" if completed else "(more)",
+                            attribute, data[context].data )
                 # Final .status is 0x00 if all requested elements were shipped; 0x06 if not
-                data.status		= 0x00 if end == endactual else 0x06
+                data.status		= 0x00 if completed else 0x06
                 data.pop( 'status_ext' ) # non-empty dotdict level; use pop instead of del
             else:
                 # Write Tag [Fragmented].  We know the type is right.
@@ -668,7 +699,7 @@ def setup_tag( key, val ):
         # doesn't exist.  Then, find the Attribute, ensuring it is consistent if it exists.
         cls,ins,att		= 0x02,1,None # The (Logix?) Message Router, by default
         if 'path' in val and val['path']:
-            cls,ins,att	= resolve( val['path'], attribute=True )
+            cls,ins,att	= resolve( val['path'], attribute=True ) # No default Attribute for new Tags
         # See if the tag's Instance exists.  If not, we'll need to create it.  If the Class'
         # "meta" Instance exists, we'll use it to create the Instance (its always at
         # Instance 0).  Otherwise, we'll create an Object class with the appropriate

server/enip/parser.py

@@ -684,7 +684,7 @@ def enip_format( data, sort_keys=False, indent=4 ):
                 beg,end		= 'bytearray(',')'
             else:
                 beg,end		= 'bytes(',')'
-            result	       += "{beg}hexload('''".format( beg=beg )
+            result	       += "{beg}hexload(r'''".format( beg=beg )
             result	       += ''.join( newline + prefix + row for row in misc.hexdumper( val ))
             result	       += newline + "'''){end},".format( end=end )
             continue

server/enip/parser_test.py

@@ -82,7 +82,7 @@ def test_enip_format():
     'empty.sub':                    [],
     'empty.totally_null':           None,
     'empty.true_false':             (True, False),
-    'long_bytes':                   bytes(hexload('''
+    'long_bytes':                   bytes(hexload(r'''
         00000000:  6f 00 00 00 06 00 00 00  52 54 31 2d 31 37 00 00   |o.......RT1-17..|
         00000010:  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00   |................|
         00000020:  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00   |................|
@@ -122,7 +122,7 @@ def test_enip_format():
         00000240:  0c 00 00 00 07 00 00 00  0a 00 00 00 28 00 00 00   |............(...|
         00000250:  34 00 00 00 fe 58 0a 00                            |4....X..|
     ''')),
-    'short_bytes':                  bytes(hexload('''
+    'short_bytes':                  bytes(hexload(r'''
         00000000:  6f 00 00 00                                        |o...|
     ''')),
     'some_ascii':                   'The quick brown fox \\\\ jumped over the "lazy" dog',

server/enip/udt_test.py

@@ -334,7 +334,7 @@ def logix_remote_udt_pylogix( count, svraddr, kwargs ):
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.elements": 1,
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.type": 672,
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.structure_tag": 36345,
-                "enip.CIP.send_data.CPF.item[1].connection_data.request.status": 0,
+                "enip.CIP.send_data.CPF.item[1].connection_data.request.status": 6, 
             }, {
                 # Then, starts with a Read Tag
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.path.segment[1].element": 0,
@@ -347,7 +347,7 @@ def logix_remote_udt_pylogix( count, svraddr, kwargs ):
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.path.segment[0].symbolic": "ExampleSensor",
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.path.segment[1].element": 0,
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.elements": 360,
-                "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.offset": 215400,
+                "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.offset": 215696, # w/ 488 byte MAX_BYTES
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.type": 672,
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.read_frag.structure_tag": 36345,
                 "enip.CIP.send_data.CPF.item[1].connection_data.request.status": 0,