Flex: for lists/maps with non-null elements/values skip null values

Also explicitly test detection of Map with non-null values.

Note about Map: while a map with null values can be cast to one with
non-null values, it will error at runtime when iterating over entries
as it fails to cast a null value.

Author: greenrobot-team

generator/lib/src/code_chunks.dart

@@ -449,6 +449,26 @@ class CodeChunks {
     );
   }
 
+  /// Extracts the key and value types by removing "Map<" prefix and ">" suffix.
+  static String _getMapKeyValueTypes(String typeString) {
+    // This String operation is very fragile and should be replaced with info
+    // from ModelProperty created by EntityResolver.
+    return typeString.substring(
+      4, // "Map<"
+      typeString.length - 1, // ">"
+    );
+  }
+
+  /// Extracts the element type by removing "List<" prefix and ">" suffix.
+  static String _getListElementType(String typeString) {
+    // This String operation is very fragile and should be replaced with info
+    // from ModelProperty created by EntityResolver.
+    return typeString.substring(
+      5, // "List<"
+      typeString.length - 1, // ">"
+    );
+  }
+
   static String objectFromFB(ModelEntity entity) {
     // collect code for the template at the end of this function
     final constructorLines = <String>[]; // used as constructor arguments
@@ -579,50 +599,58 @@ class CodeChunks {
               // Check if list needs casting (List<Object> or List<Object?>)
               final isListOfObject = p.fieldType.startsWith('List<Object');
               final String flexDeserializer;
+              final bool skipNull;
               final String? defaultValue;
               final String castSuffix;
               if (isValue) {
                 // dynamic or Object? - can be any FlexBuffer value
                 flexDeserializer = 'fromFlexBuffer';
+                skipNull = false;
                 defaultValue = null; // nullable only, no default needed
                 castSuffix = '';
               } else if (isMap) {
                 flexDeserializer = 'flexBufferToMap';
-                defaultValue = '<String, dynamic>{}';
-                castSuffix = '';
+                final keyValueTypes = _getMapKeyValueTypes(p.fieldType);
+                final hasDynamicValueType = keyValueTypes.endsWith('dynamic');
+                // For maps with non-null values skip nulls (so not
+                // Map<String, dynamic> or Map<String, Object/List/Map?>).
+                skipNull = !hasDynamicValueType && !keyValueTypes.endsWith('?');
+                defaultValue = '<$keyValueTypes>{}';
+                // Cast to the correct Map type if not Map<String, dynamic>
+                if (hasDynamicValueType) {
+                  castSuffix = '';
+                } else {
+                  castSuffix = '?.cast<$keyValueTypes>()';
+                }
               } else if (isListOfMaps) {
                 flexDeserializer = 'flexBufferToListOfMaps';
-                // Extract the element type, e.g. "Map<String, dynamic>"
-                // by removing "List<" prefix and ">" suffix.
-                final elementType = p.fieldType.substring(
-                  5,
-                  p.fieldType.length - 1,
-                );
+                final elementType = _getListElementType(p.fieldType);
+                // The deserializer already skips nulls
+                skipNull = false;
                 defaultValue = '<$elementType>[]';
-                // Cast needed for Map<String, Object?> (Object not supported)
+                // Cast needed for Map<String, Object?> and Map<String, Object>
                 if (elementType == 'Map<String, dynamic>') {
                   castSuffix = '';
                 } else {
                   castSuffix = '?.cast<$elementType>()';
                 }
               } else if (isListOfObject) {
                 flexDeserializer = 'flexBufferToList';
-                // Extract the element type, e.g. "Object" or "Object?"
-                // by removing "List<" prefix and ">" suffix.
-                final elementType = p.fieldType.substring(
-                  5,
-                  p.fieldType.length - 1,
-                );
+                final elementType = _getListElementType(p.fieldType);
+                // For lists with non-null elements skip nulls
+                skipNull = !elementType.endsWith('?');
                 defaultValue = '<$elementType>[]';
                 castSuffix = '?.cast<$elementType>()';
               } else {
                 // List<dynamic>
                 flexDeserializer = 'flexBufferToList';
+                skipNull = false;
                 defaultValue = '<dynamic>[]';
                 castSuffix = '';
               }
+              final skipNullArg = skipNull ? ', skipNull: true' : '';
               final deserializeExpr =
-                  '$obxInt.$flexDeserializer(buffer, rootOffset, $offset)$castSuffix';
+                  '$obxInt.$flexDeserializer(buffer, rootOffset, $offset$skipNullArg)$castSuffix';
               if (p.fieldIsNullable || defaultValue == null) {
                 // Nullable field or value type (dynamic/Object?) - no default
                 return deserializeExpr;

generator/test/code_builder_test.dart

@@ -825,6 +825,9 @@ void main() {
       
         // Auto-detected Map<String, Object?> - nullable
         Map<String, Object?>? flexObject;
+        
+        // Auto-detected Map<String, Object> (non-nullable values) - nullable
+        Map<String, Object>? flexObjectNonNull;
       
         // Non-nullable with default empty map
         Map<String, dynamic> flexNonNull = {};
@@ -839,7 +842,7 @@ void main() {
       await testEnv.run(source);
 
       var properties = testEnv.model.entities[0].properties;
-      expectFlexProperties(properties, 5);
+      expectFlexProperties(properties, 6);
     });
 
     test('Flex List type detection', () async {

objectbox/lib/src/native/bindings/flexbuffers.dart

@@ -19,36 +19,49 @@ Uint8List toFlexBuffer(Object value) {
 /// Returns null if no data is stored for this field. The returned value can be
 /// null, bool, int, double, String, `List<dynamic>`, `Map<String, dynamic>`,
 /// or nested combinations of these.
+///
+/// If [skipNullCollectionValues], if a vector element is null or a map value is
+/// null the list element or map entry is not returned. This can be used to
+/// allow casting the return value of this method to [List] with non-null
+/// elements (like `List<Object>`) and [Map] with non-null values (like
+/// `Map<String, Object>`).
 @pragma('vm:prefer-inline')
-dynamic fromFlexBuffer(BufferContext buffer, int offset, int field) {
+dynamic fromFlexBuffer(BufferContext buffer, int offset, int field,
+    {bool skipNullCollectionValues = false}) {
   // Note: Uint8ListReader returns a Uint8List? cast to List<int>, so just cast
   // it back (if that ever changes, add a custom reader)
   final bytes = const Uint8ListReader(lazy: false)
       .vTableGetNullable(buffer, offset, field) as Uint8List?;
   if (bytes == null) return null;
   final ref = flex.Reference.fromBuffer(bytes.buffer);
-  return _convertReference(ref);
+  return _convertReference(ref, skipNullCollectionValues);
 }
 
 /// Deserializes FlexBuffer bytes to a `Map<String, dynamic>`.
 @pragma('vm:prefer-inline')
 Map<String, dynamic>? flexBufferToMap(
-        BufferContext buffer, int offset, int field) =>
-    fromFlexBuffer(buffer, offset, field) as Map<String, dynamic>?;
+        BufferContext buffer, int offset, int field, {bool skipNull = false}) =>
+    fromFlexBuffer(buffer, offset, field, skipNullCollectionValues: skipNull)
+        as Map<String, dynamic>?;
 
 /// Deserializes FlexBuffer bytes to a `List<dynamic>`.
 @pragma('vm:prefer-inline')
-List<dynamic>? flexBufferToList(BufferContext buffer, int offset, int field) =>
-    fromFlexBuffer(buffer, offset, field) as List<dynamic>?;
+List<dynamic>? flexBufferToList(BufferContext buffer, int offset, int field,
+        {bool skipNull = false}) =>
+    fromFlexBuffer(buffer, offset, field, skipNullCollectionValues: skipNull)
+        as List<dynamic>?;
 
 /// Deserializes FlexBuffer bytes to a `List<Map<String, dynamic>>`.
 @pragma('vm:prefer-inline')
 List<Map<String, dynamic>>? flexBufferToListOfMaps(
         BufferContext buffer, int offset, int field) =>
-    flexBufferToList(buffer, offset, field)?.cast<Map<String, dynamic>>();
+    flexBufferToList(buffer, offset, field, skipNull: true)
+        ?.cast<Map<String, dynamic>>();
 
 /// Recursively converts a FlexBuffer Reference to a Dart object.
-dynamic _convertReference(flex.Reference ref) {
+///
+/// For [skipNullCollectionValues] see [fromFlexBuffer].
+dynamic _convertReference(flex.Reference ref, bool skipNullCollectionValues) {
   if (ref.isNull) {
     return null;
   } else if (ref.isBool) {
@@ -68,11 +81,22 @@ dynamic _convertReference(flex.Reference ref) {
     final keyIterator = keys.iterator;
     final valueIterator = values.iterator;
     while (keyIterator.moveNext() && valueIterator.moveNext()) {
-      map[keyIterator.current] = _convertReference(valueIterator.current);
+      final value =
+          _convertReference(valueIterator.current, skipNullCollectionValues);
+      if (value != null || !skipNullCollectionValues) {
+        map[keyIterator.current] = value;
+      }
     }
     return map;
   } else if (ref.isVector) {
-    return ref.vectorIterable.map(_convertReference).toList();
+    final list = <dynamic>[];
+    for (final element in ref.vectorIterable) {
+      final value = _convertReference(element, skipNullCollectionValues);
+      if (value != null || !skipNullCollectionValues) {
+        list.add(value);
+      }
+    }
+    return list;
   }
 
   throw UnsupportedError('Unsupported FlexBuffer value type');

objectbox_test/test/entity_flex.dart

@@ -12,6 +12,9 @@ class FlexMapEntity {
   // Auto-detected Map<String, Object?> - nullable
   Map<String, Object?>? flexObject;
 
+  // Auto-detected Map<String, Object> (non-nullable values) - nullable
+  Map<String, Object>? flexObjectNonNull;
+
   // Non-nullable with default empty map
   Map<String, dynamic> flexNonNull = {};
 
@@ -22,6 +25,7 @@ class FlexMapEntity {
   FlexMapEntity({
     this.flexDynamic,
     this.flexObject,
+    this.flexObjectNonNull,
     this.flexNonNull = const {},
     this.flexExplicit,
   });

objectbox_test/test/flex_test.dart

@@ -43,31 +43,39 @@ void main() {
         'int': 42,
         'double': 3.14,
         'bool': false,
-        'null': null,
       };
+      final testMapWithNull = <String, Object?>{
+        'null': null,
+      }..addAll(testMap);
       final entity = FlexMapEntity(
-          flexDynamic: testMap,
-          flexObject: testMap,
-          flexNonNull: testMap,
-          flexExplicit: testMap);
+          flexDynamic: testMapWithNull,
+          flexObject: testMapWithNull,
+          flexObjectNonNull: testMap,
+          flexNonNull: testMapWithNull,
+          flexExplicit: testMapWithNull);
       final id = box.put(entity);
 
       assertTestMap(Map<String, dynamic> map) {
         expect(map['string'], 'hello');
         expect(map['int'], 42);
         expect(map['double'], 3.14);
         expect(map['bool'], false);
+      }
+
+      assertTestMapWithNull(Map<String, dynamic> map) {
+        assertTestMap(map);
         expect(map['null'], isNull);
         // The map also returns null if it doesn't contain the key,
         // so explicitly check it does.
         expect(map.containsKey('null'), true);
       }
 
       final read = box.get(id)!;
-      assertTestMap(read.flexDynamic!);
-      assertTestMap(read.flexObject!);
-      assertTestMap(read.flexNonNull);
-      assertTestMap(read.flexExplicit!);
+      assertTestMapWithNull(read.flexDynamic!);
+      assertTestMapWithNull(read.flexObject!);
+      assertTestMap(read.flexObjectNonNull!);
+      assertTestMapWithNull(read.flexNonNull);
+      assertTestMapWithNull(read.flexExplicit!);
     });
 
     test('put and get nested map', () {
@@ -317,7 +325,10 @@ void main() {
         expect(list.length, 2);
         final first = list[0];
         expect(first['key1'], 'value1');
-        expect(first['nullable'], isNull);
+        // Because the List element type of the properties is defined as
+        // non-null, null values are skipped, which also affects nested maps
+        // (and lists).
+        expect(first.containsKey('nullable'), false);
         final second = list[1];
         expect(second['key2'], 'value2');
         expect((second['nested'] as Map)['a'], 1);

objectbox_test/test/objectbox-model.json

@@ -877,7 +877,7 @@
     },
     {
       "id": "21:7625885984770420113",
-      "lastPropertyId": "5:4318338462051520175",
+      "lastPropertyId": "6:1851577594233191679",
       "name": "FlexMapEntity",
       "properties": [
         {
@@ -905,6 +905,11 @@
           "id": "5:4318338462051520175",
           "name": "flexExplicit",
           "type": 13
+        },
+        {
+          "id": "6:1851577594233191679",
+          "name": "flexObjectNonNull",
+          "type": 13
         }
       ],
       "relations": []