Make list handling more straightforward

- do not store parsed list lengths
- use list lengths calculated via len() for serialisation
- assert equality of list lengths that are parsed with equal length

Author: HSZemi

README.md

@@ -44,7 +44,6 @@ from genieutils.datfile import DatFile
 
 data = DatFile.parse('path/to/empires2_x2_p1.dat')
 for civ in data.civs:
-    civ.units[434].bird.task_size -= 1
     civ.units[434].bird.tasks.pop()
 data.save('path/to/modded/empires2_x2_p1.dat')
 ```

src/genieutils/civ.py

@@ -8,13 +8,10 @@
 class Civ(GenieClass):
     player_type: int
     name: str
-    resources_size: int
     tech_tree_id: int
     team_bonus_id: int
     resources: list[float]
     icon_set: int
-    units_size: int
-    unit_pointers: list[int]
     units: list[Unit | None]
 
     @classmethod
@@ -32,26 +29,27 @@ def from_bytes(cls, content: ByteHandler) -> 'Civ':
         return cls(
             player_type=player_type,
             name=name,
-            resources_size=resources_size,
             tech_tree_id=tech_tree_id,
             team_bonus_id=team_bonus_id,
             resources=resources,
             icon_set=icon_set,
-            units_size=units_size,
-            unit_pointers=unit_pointers,
             units=units,
         )
 
+    @property
+    def unit_pointers(self) -> list[int]:
+        return [(0 if u is None else 1) for u in self.units]
+
     def to_bytes(self) -> bytes:
         return b''.join([
             self.write_int_8(self.player_type),
             self.write_debug_string(self.name),
-            self.write_int_16(self.resources_size),
+            self.write_int_16(len(self.resources)),
             self.write_int_16(self.tech_tree_id),
             self.write_int_16(self.team_bonus_id),
             self.write_float_array(self.resources),
             self.write_int_8(self.icon_set),
-            self.write_int_16(self.units_size),
+            self.write_int_16(len(self.units)),
             self.write_int_32_array(self.unit_pointers),
-            self.write_class_array_with_pointers(self.unit_pointers, self.units),
+            self.write_class_array(self.units),
         ])

src/genieutils/common.py

@@ -72,14 +72,8 @@ def write_class(self, value: 'GenieClass') -> bytes:
             return retval
         return b''
 
-    def write_class_array(self, value: Sequence['GenieClass']) -> bytes:
-        retval = b''.join(self.write_class(v) for v in value)
-        if retval:
-            return retval
-        return b''
-
-    def write_class_array_with_pointers(self, pointers: list[int], value: list['GenieClass']) -> bytes:
-        retval = b''.join(self.write_class(v) for i, v in enumerate(value) if pointers[i])
+    def write_class_array(self, value: Sequence['GenieClass | None']) -> bytes:
+        retval = b''.join(self.write_class(v) for v in value if v is not None)
         if retval:
             return retval
         return b''

src/genieutils/datfile.py

@@ -20,27 +20,17 @@
 @dataclass
 class DatFile(GenieClass):
     version: str
-    terrain_restrictions_size: int
-    terrains_used_1: int
     float_ptr_terrain_tables: list[int]
     terrain_pass_graphic_pointers: list[int]
     terrain_restrictions: list[TerrainRestriction]
-    player_colours_size: int
     player_colours: list[PlayerColour]
-    sounds_size: int
     sounds: list[Sound]
-    graphics_size: int
-    graphic_pointers: list[int]
     graphics: list[Graphic | None]
     terrain_block: TerrainBlock
     random_maps: RandomMaps
-    effects_size: int
     effects: list[Effect]
-    unit_headers_size: int
     unit_headers: list[UnitHeaders]
-    civs_size: int
     civs: list[Civ]
-    techs_size: int
     techs: list[Tech]
     time_slice: int
     unit_kill_rate: int
@@ -98,63 +88,63 @@ def from_bytes(cls, content: ByteHandler) -> 'DatFile':
         razing_kill_total = content.read_int_32()
         tech_tree = content.read_class(TechTree)
         return cls(
-            version,
-            terrain_restrictions_size,
-            terrains_used_1,
-            float_ptr_terrain_tables,
-            terrain_pass_graphic_pointers,
-            terrain_restrictions,
-            player_colours_size,
-            player_colours,
-            sounds_size,
-            sounds,
-            graphics_size,
-            graphic_pointers,
-            graphics,
-            terrain_block,
-            random_maps,
-            effects_size,
-            effects,
-            unit_headers_size,
-            unit_headers,
-            civs_size,
-            civs,
-            techs_size,
-            techs,
-            time_slice,
-            unit_kill_rate,
-            unit_kill_total,
-            unit_hit_point_rate,
-            unit_hit_point_total,
-            razing_kill_rate,
-            razing_kill_total,
-            tech_tree,
+            version=version,
+            float_ptr_terrain_tables=float_ptr_terrain_tables,
+            terrain_pass_graphic_pointers=terrain_pass_graphic_pointers,
+            terrain_restrictions=terrain_restrictions,
+            player_colours=player_colours,
+            sounds=sounds,
+            graphics=graphics,
+            terrain_block=terrain_block,
+            random_maps=random_maps,
+            effects=effects,
+            unit_headers=unit_headers,
+            civs=civs,
+            techs=techs,
+            time_slice=time_slice,
+            unit_kill_rate=unit_kill_rate,
+            unit_kill_total=unit_kill_total,
+            unit_hit_point_rate=unit_hit_point_rate,
+            unit_hit_point_total=unit_hit_point_total,
+            razing_kill_rate=razing_kill_rate,
+            razing_kill_total=razing_kill_total,
+            tech_tree=tech_tree,
         )
 
+    @property
+    def graphic_pointers(self) -> list[int]:
+        return [(0 if g is None else 1) for g in self.graphics]
+
     def to_bytes(self) -> bytes:
+        terrain_restrictions_size = len(self.terrain_restrictions)
+        assert len(self.float_ptr_terrain_tables) == len(self.terrain_pass_graphic_pointers) == terrain_restrictions_size
+        terrains_used = 0
+        if self.terrain_restrictions:
+            terrains_used = len(self.terrain_restrictions[0].passable_buildable_dmg_multiplier)
+
         return b''.join([
             self.write_string(8, self.version),
-            self.write_int_16(self.terrain_restrictions_size),
-            self.write_int_16(self.terrains_used_1),
+            self.write_int_16(terrain_restrictions_size),
+            self.write_int_16(terrains_used),
             self.write_int_32_array(self.float_ptr_terrain_tables),
             self.write_int_32_array(self.terrain_pass_graphic_pointers),
             self.write_class_array(self.terrain_restrictions),
-            self.write_int_16(self.player_colours_size),
+            self.write_int_16(len(self.player_colours)),
             self.write_class_array(self.player_colours),
-            self.write_int_16(self.sounds_size),
+            self.write_int_16(len(self.sounds)),
             self.write_class_array(self.sounds),
-            self.write_int_16(self.graphics_size),
+            self.write_int_16(len(self.graphics)),
             self.write_int_32_array(self.graphic_pointers),
-            self.write_class_array_with_pointers(self.graphic_pointers, self.graphics),
+            self.write_class_array(self.graphics),
             self.write_class(self.terrain_block),
             self.write_class(self.random_maps),
-            self.write_int_32(self.effects_size),
+            self.write_int_32(len(self.effects)),
             self.write_class_array(self.effects),
-            self.write_int_32(self.unit_headers_size),
+            self.write_int_32(len(self.unit_headers)),
             self.write_class_array(self.unit_headers),
-            self.write_int_16(self.civs_size),
+            self.write_int_16(len(self.civs)),
             self.write_class_array(self.civs),
-            self.write_int_16(self.techs_size),
+            self.write_int_16(len(self.techs)),
             self.write_class_array(self.techs),
             self.write_int_32(self.time_slice),
             self.write_int_32(self.unit_kill_rate),

src/genieutils/effect.py

@@ -34,7 +34,6 @@ def to_bytes(self) -> bytes:
 @dataclass
 class Effect(GenieClass):
     name: str
-    effect_command_count: int
     effect_commands: list[EffectCommand]
 
     @classmethod
@@ -44,13 +43,12 @@ def from_bytes(cls, content: ByteHandler) -> 'Effect':
         effect_commands = content.read_class_array(EffectCommand, effect_command_count)
         return cls(
             name=name,
-            effect_command_count=effect_command_count,
             effect_commands=effect_commands,
         )
 
     def to_bytes(self) -> bytes:
         return b''.join([
             self.write_debug_string(self.name),
-            self.write_int_16(self.effect_command_count),
+            self.write_int_16(len(self.effect_commands)),
             self.write_class_array(self.effect_commands),
         ])

src/genieutils/graphic.py

@@ -89,7 +89,6 @@ class Graphic(GenieClass):
     player_color: int
     transparent_selection: int
     coordinates: tuple[int, int, int, int]
-    delta_count: int
     sound_id: int
     wwise_sound_id: int
     angle_sounds_used: int
@@ -143,7 +142,6 @@ def from_bytes(cls, content: ByteHandler) -> 'Graphic':
             player_color=player_color,
             transparent_selection=transparent_selection,
             coordinates=coordinates,
-            delta_count=delta_count,
             sound_id=sound_id,
             wwise_sound_id=wwise_sound_id,
             angle_sounds_used=angle_sounds_used,
@@ -172,7 +170,7 @@ def to_bytes(self) -> bytes:
             self.write_int_16(self.player_color),
             self.write_int_8(self.transparent_selection),
             self.write_int_16_array(self.coordinates),
-            self.write_int_16(self.delta_count),
+            self.write_int_16(len(self.deltas)),
             self.write_int_16(self.sound_id),
             self.write_int_32(self.wwise_sound_id),
             self.write_int_8(self.angle_sounds_used),

src/genieutils/randommaps.py

@@ -278,7 +278,6 @@ def to_bytes(self) -> bytes:
 
 @dataclass
 class RandomMaps(GenieClass):
-    random_map_count: int
     random_maps_ptr: int
     map_info_1: list[MapInfo]
     map_info_2: list[MapInfo]
@@ -290,15 +289,15 @@ def from_bytes(cls, content: ByteHandler) -> 'RandomMaps':
         map_info_1 = content.read_class_array(MapInfo, random_map_count)
         map_info_2 = content.read_class_array(MapInfo, random_map_count)
         return cls(
-            random_map_count=random_map_count,
             random_maps_ptr=random_maps_ptr,
             map_info_1=map_info_1,
             map_info_2=map_info_2,
         )
 
     def to_bytes(self) -> bytes:
+        assert len(self.map_info_1) == len(self.map_info_2)
         return b''.join([
-            self.write_int_32(self.random_map_count, signed=False),
+            self.write_int_32(len(self.map_info_1), signed=False),
             self.write_int_32(self.random_maps_ptr),
             self.write_class_array(self.map_info_1),
             self.write_class_array(self.map_info_2),

src/genieutils/sound.py

@@ -35,7 +35,6 @@ def to_bytes(self) -> bytes:
 class Sound(GenieClass):
     id: int
     play_delay: int
-    items_size: int
     cache_time: int
     total_probability: int
     items: list[SoundItem]
@@ -51,7 +50,6 @@ def from_bytes(cls, content: ByteHandler) -> 'Sound':
         return cls(
             id=id_,
             play_delay=play_delay,
-            items_size=items_size,
             cache_time=cache_time,
             total_probability=total_probability,
             items=items,
@@ -61,7 +59,7 @@ def to_bytes(self) -> bytes:
         return b''.join([
             self.write_int_16(self.id),
             self.write_int_16(self.play_delay),
-            self.write_int_16(self.items_size),
+            self.write_int_16(len(self.items)),
             self.write_int_32(self.cache_time),
             self.write_int_16(self.total_probability),
             self.write_class_array(self.items),