Add unset field semantics for structs ("strict structs") (#607)

* Add unset field semantics (disc #536)

Signed-off-by: Simon Beyzerov <[email protected]>

* Fix up PR

Signed-off-by: Adam Glustein <[email protected]>

* Further clean up of PR; add docs, improve error messages, fix inheritance bug

Signed-off-by: Adam Glustein <[email protected]>

* Change lint to use 3.11

Signed-off-by: Adam Glustein <[email protected]>

* Fix 3.10+ Union annotations

Signed-off-by: Adam Glustein <[email protected]>

* Fix typo in docs

Signed-off-by: Adam Glustein <[email protected]>

* Store optionality on the field object, and embed whether or not the field is set to None in the struct's bitmask

Signed-off-by: Adam Glustein <[email protected]>

* More PR comments

Signed-off-by: Adam Glustein <[email protected]>

* Fix str/repr for strict structs

Signed-off-by: Adam Glustein <[email protected]>

* WIP PR comments

Signed-off-by: Adam Glustein <[email protected]>

* PR comments

Signed-off-by: Adam Glustein <[email protected]>

* Store set and none masks in the same vector, provide accessors

Signed-off-by: Adam Glustein <[email protected]>

* Tests: add collectts test case, merge wiring_access tests, add fromts test

Signed-off-by: Adam Glustein <[email protected]>

* Update pydantic schema

Signed-off-by: Adam Glustein <[email protected]>

* Add tests for pydantic validation

Signed-off-by: Adam Glustein <[email protected]>

---------

Signed-off-by: Simon Beyzerov <[email protected]>
Signed-off-by: Adam Glustein <[email protected]>
Co-authored-by: Simon Beyzerov <[email protected]>

Author: AdamGlustein

cpp/csp/cppnodes/baselibimpl.cpp

@@ -705,6 +705,9 @@ DECLARE_CPPNODE( struct_fromts )
                 );
         }
 
+        if( unlikely( !out.get() -> validate() ) )
+            CSP_THROW( ValueError, "Struct " << cls.value() -> name() << " is not valid; required fields " << out -> formatAllUnsetStrictFields() << " did not tick" );
+
         CSP_OUTPUT( std::move( out ) );
     }
 
@@ -758,6 +761,9 @@ DECLARE_CPPNODE( struct_collectts )
                            }
                 );
         }
+        
+        if( unlikely( !out.get() -> validate() ) )
+            CSP_THROW( ValueError, "Struct " << cls.value() -> name() << " is not valid; required fields " << out -> formatAllUnsetStrictFields() << " did not tick" );
 
         CSP_OUTPUT( std::move( out ) );
     }

cpp/csp/engine/InputAdapter.h

@@ -4,6 +4,7 @@
 #include <csp/core/Time.h>
 #include <csp/engine/Enums.h>
 #include <csp/engine/RootEngine.h>
+#include <csp/engine/Struct.h>
 #include <csp/engine/TimeSeriesProvider.h>
 
 namespace csp
@@ -22,6 +23,8 @@ class InputAdapter : public TimeSeriesProvider, public EngineOwned
     virtual void start( DateTime start, DateTime end ) {}
     virtual void stop() {}
 
+    virtual const char * name() const { return "InputAdapter"; }
+
     template< typename T > void outputTickTyped( DateTime timestamp, const T & value )
     {
         outputTickTyped( rootEngine() -> cycleCount(), timestamp, value );
@@ -55,6 +58,13 @@ class InputAdapter : public TimeSeriesProvider, public EngineOwned
 template<typename T>
 bool InputAdapter::consumeTick( const T & value )
 {
+    if constexpr( CspType::Type::fromCType<T>::type == CspType::TypeTraits::STRUCT )
+    {
+        if( unlikely( !( value -> validate() ) ) )
+            CSP_THROW( ValueError, "Struct " << value -> meta() -> name() << " from adapter type " << name() 
+                << " is not valid; required fields " << value -> formatAllUnsetStrictFields() << " were not set on init" );
+    }
+    
     switch( pushMode() )
     {
         case PushMode::LAST_VALUE:

cpp/csp/engine/Struct.cpp

@@ -1,20 +1,26 @@
 #include <csp/core/System.h>
 #include <csp/engine/Struct.h>
 #include <algorithm>
+#include <ranges>
+#include <string>
+#include <sstream>
 
 namespace csp
 {
 
 StructField::StructField( CspTypePtr type, const std::string & fieldname, 
-                          size_t size, size_t alignment ) :
+                          size_t size, size_t alignment, bool isOptional ) :
     m_fieldname( fieldname ),
     m_offset( 0 ),
     m_size( size ),
     m_alignment( alignment ),
     m_maskOffset( 0 ),
     m_maskBit( 0 ),
+    m_noneMaskBit( 0 ),
     m_maskBitMask( 0 ),
-    m_type( type )
+    m_noneMaskBitMask( 0 ),
+    m_type( type ),
+    m_isOptional( isOptional )
 {
 }
 
@@ -33,13 +39,13 @@ and adjustments required for the hidden fields
 
 */
 
-StructMeta::StructMeta( const std::string & name, const Fields & fields,
-                        std::shared_ptr<StructMeta> base ) : m_name( name ), m_base( base ), m_fields( fields ),
+StructMeta::StructMeta( const std::string & name, const Fields & fields, bool isStrict,
+                        std::shared_ptr<StructMeta> base ) : m_name( name ), m_base( base ), m_fields( fields ), m_optionalFieldsBitMasks(),
                                                              m_size( 0 ), m_partialSize( 0 ), m_partialStart( 0 ), m_nativeStart( 0 ), m_basePadding( 0 ),
                                                              m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), m_firstNativePartialField( 0 ),
-                                                             m_isPartialNative( true ), m_isFullyNative( true )
+                                                             m_isPartialNative( true ), m_isFullyNative( true ), m_isStrict( isStrict )
 {
-    if( m_fields.empty() && !m_base)
+    if( m_fields.empty() && !m_base )
         CSP_THROW( TypeError, "Struct types must define at least 1 field" );
 
     //sort by sizes, biggest first, to get proper alignment
@@ -95,25 +101,64 @@ StructMeta::StructMeta( const std::string & name, const Fields & fields,
     //Setup masking bits for our fields
     //NOTE we can be more efficient by sticking masks into any potential alignment gaps, dont want to spend time on it 
     //at this point
-    m_maskSize     = !m_fields.empty() ? 1 + ( ( m_fields.size() - 1 ) / 8 ) : 0;
+
+    size_t optionalFieldCount = std::count_if( m_fields.begin(), m_fields.end(), []( const auto & f ) { return f -> isOptional(); } );
+    
+    m_maskSize     = !m_fields.empty() ? 1 + ( ( m_fields.size() + optionalFieldCount - 1 ) / 8 ) : 0;
     m_size         = offset + m_maskSize;
     m_partialSize  = m_size - baseSize;
     m_maskLoc      = m_size - m_maskSize;
+    
+    uint8_t numRemainingBits = ( m_fields.size() + optionalFieldCount ) % 8;
+    m_lastByteMask = ( 1u << numRemainingBits ) - 1;
 
     size_t  maskLoc = m_maskLoc;
     uint8_t maskBit = 0;
-    for( auto & f : m_fields )
+
+    // Set optional fields first so that their 2-bits never cross a byte boundary
+    // Put both the set bits and none bits in the same vector to avoid fragmentation
+    m_optionalFieldsBitMasks.resize( 2 * m_maskSize );
+    for( size_t i = 0; i < m_fields.size(); ++i )
+    {
+        auto & f = m_fields[ i ];
+        if( f -> isOptional() )
+        {
+            f -> setMaskOffset( maskLoc, maskBit );
+            optionalSetBitMask( maskLoc - m_maskLoc ) |= ( 1 << maskBit ); // set bits for this byte
+            optionalNoneBitMask( maskLoc - m_maskLoc ) |= ( 1 << ++maskBit ); // none bits for this byte
+            if( ++maskBit == 8 )
+            {
+                maskBit = 0;
+                ++maskLoc;
+            }
+        }
+    }
+
+    for( size_t i = 0; i < m_fields.size(); ++i )
     {
-        f -> setMaskOffset( maskLoc, maskBit );
-        if( ++maskBit == 8 )
+        auto & f = m_fields[ i ];
+        if( !f -> isOptional() )
         {
-            maskBit = 0;
-            ++maskLoc;
+            f -> setMaskOffset( maskLoc, maskBit );
+            if( ++maskBit == 8 )
+            {
+                maskBit = 0;
+                ++maskLoc;
+            }
         }
     }
 
     if( m_base )
     {
+        // The complete inheritance hierarchy must agree on strict/non-strict
+        if( m_isStrict != m_base -> isStrict() )
+        {
+            CSP_THROW( ValueError, 
+                    "Struct " << m_name << " was declared " << ( m_isStrict ? "strict" : "non-strict" ) << " but derives from "
+                    << m_base -> name() << " which is " << ( m_base -> isStrict() ? "strict" : "non-strict" )
+            );
+        }
+
         m_fields.insert( m_fields.begin(), m_base -> m_fields.begin(), m_base -> m_fields.end() );
         m_fieldnames.insert( m_fieldnames.begin(), m_base -> m_fieldnames.begin(), m_base -> m_fieldnames.end() );
 
@@ -453,23 +498,60 @@ void StructMeta::clear( Struct * s ) const
         m_base -> clear( s );
 }
 
+std::string StructMeta::formatAllUnsetStrictFields( const Struct * s ) const
+{
+    bool first = true;
+    std::stringstream ss;
+    ss << "["; 
+    
+    for( size_t i = 0; i < m_fields.size(); ++i )
+    {
+        const auto & f = m_fields[ i ];
+        if( !f -> isSet( s ) && !f -> isNone( s ) )
+        {
+            if( !first )
+                ss << ", ";
+            else
+                first = false;
+            ss << f -> fieldname();
+        }
+    }
+    ss << "]";
+    return ss.str();
+}
+
 bool StructMeta::allFieldsSet( const Struct * s ) const
 {
-    size_t  numLocalFields = m_fields.size() - m_firstPartialField;
-    uint8_t numRemainingBits = numLocalFields % 8;
+    /*
+    We can use bit operations to validate the struct. 
+        1. Let M1 be the bitmask with 1s that align with the set bit of optional fields and 
+        2. Let M2 be the bitmask with 1s that align with the none bit of optional fields. 
+            -- Both M1 and M2 are computed trivially when we create the meta.
+        3. Let M1* = M1 & mask. M1* now is the bitmask of all set optional fields.
+        4. Similarly, let M2* = M2 & mask, such that M2* is the bitmask of all none optional fields.
+        5. Let M3 = mask | (M1* << 1) | (M2* >> 1). Since the shifted set/none bitsets will fill in that optional fields other bit,
+            the struct can validate iff M3 is all 1s.
+
+    Notes:
+        1) We do this on a byte by byte basis currently. If we add 32/64 bit padding to the struct mask, we could do this as one single step for most structs.
+        2) There is an edge condition if a) the set bit of an optional field is the last bit of a byte or b) the none bit of an optional field is the first bit of a byte.
+            So, when we create the meta, we ensure this never happens by being smart about the ordering of fields in the mask.
+    */
 
     const uint8_t * m = reinterpret_cast<const uint8_t *>( s ) + m_maskLoc;
-    const uint8_t * e = m + m_maskSize - bool( numRemainingBits );
-    for( ; m < e; ++m )
+    const uint8_t * e = m + m_maskSize - bool( m_lastByteMask );
+
+    size_t i = 0;
+    for( ; m < e; ++m, ++i )
     {
-        if( *m != 0xFF )
+        if( ( *m | ( ( *m & optionalSetBitMask( i ) ) << 1 ) | ( ( *m & optionalNoneBitMask( i ) ) >> 1 ) ) != 0xFF )
             return false;
     }
 
-    if( numRemainingBits > 0 )
+    if( m_lastByteMask )
     {
-        uint8_t bitmask = ( 1u << numRemainingBits ) - 1;
-        if( ( *m & bitmask ) != bitmask )
+        uint8_t masked = *m & m_lastByteMask;
+        if( ( masked | ( ( ( masked & optionalSetBitMask( i ) ) << 1 ) & m_lastByteMask ) | ( ( masked & optionalNoneBitMask( i ) ) >> 1 ) ) != m_lastByteMask )
             return false;
     }
 
@@ -494,6 +576,11 @@ void StructMeta::destroy( Struct * s ) const
         m_base -> destroy( s );
 }
 
+[[nodiscard]] bool StructMeta::validate( const Struct * s ) const
+{   
+    return !isStrict() || allFieldsSet( s );
+}
+
 Struct::Struct( const std::shared_ptr<const StructMeta> & meta )
 {
     //Initialize meta shared_ptr