Skip to content

Commit 62f1d4e

Browse files
heinezenheinezen
committed
convert: Avoid multiple reads on enum lookup members.
1 parent fafb7e6 commit 62f1d4e

File tree

1 file changed

+159
-137
lines changed

1 file changed

+159
-137
lines changed

‎openage/convert/value_object/read/genie_structure.py‎

Lines changed: 159 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2014-2024 the openage authors. See copying.md for legal info.
1+
# Copyright 2014-2025 the openage authors. See copying.md for legal info.
22

33
# TODO pylint: disable=C,R
44

@@ -405,7 +405,7 @@ def _read_primitive(
405405
f"unknown data member definition {var_type} for member '{var_name}'")
406406

407407
if data_count < 0:
408-
raise SyntaxError("invalid length %d < 0 in %s for member '%s'" % (
408+
raise ValueError("invalid length %d < 0 in %s for member '%s'" % (
409409
data_count, var_type, var_name))
410410

411411
if struct_type not in STRUCT_TYPE_LOOKUP:
@@ -420,182 +420,204 @@ def _read_primitive(
420420
# lookup c type to python struct scan type
421421
symbol = STRUCT_TYPE_LOOKUP[struct_type]
422422

423-
# read that stuff!!11
423+
# figure out the data format
424424
struct_format = "< %d%s" % (data_count, symbol)
425425

426-
if export != SKIP:
427-
result = struct.unpack_from(struct_format, raw, offset)
426+
if export == SKIP:
427+
# just calculate the offset and skip the reading process
428+
offset += struct.calcsize(struct_format)
428429

429-
if is_custom_member:
430-
if not var_type.verify_read_data(self, result):
431-
raise SyntaxError("invalid data when reading %s "
432-
"at offset %# 08x" % (var_name, offset))
430+
return offset, [], False
433431

434-
# TODO: move these into a read entry hook/verification method
435-
if symbol == "s":
436-
# stringify char array
437-
result = decode_until_null(result[0])
432+
# else, read that stuff!!11
433+
unpacked = struct.unpack_from(struct_format, raw, offset)
438434

439-
if export == READ_GEN:
440-
if storage_type is StorageType.STRING_MEMBER:
441-
gen_member = StringMember(var_name, result)
435+
if is_custom_member:
436+
if not var_type.verify_read_data(self, unpacked):
437+
raise ValueError("invalid data when reading %s "
438+
"at offset %# 08x" % (var_name, offset))
442439

443-
else:
444-
raise SyntaxError("%s at offset %# 08x: Data read via %s "
445-
"cannot be stored as %s;"
446-
" expected %s"
447-
% (var_name, offset, var_type, storage_type,
448-
StorageType.STRING_MEMBER))
440+
# TODO: move these into a read entry hook/verification method
441+
if symbol == "s":
442+
# stringify char array
443+
result = decode_until_null(unpacked[0])
449444

450-
generated_value_members.append(gen_member)
445+
if export == READ_GEN:
446+
if storage_type is StorageType.STRING_MEMBER:
447+
gen_member = StringMember(var_name, result)
451448

452-
elif is_array:
453-
if export == READ_GEN:
454-
# Turn every element of result into a member
455-
# and put them into an array
456-
array_members = []
457-
allowed_member_type = None
449+
else:
450+
raise SyntaxError("%s at offset %# 08x: Data read via %s "
451+
"cannot be stored as %s;"
452+
" expected %s"
453+
% (var_name, offset, var_type, storage_type,
454+
StorageType.STRING_MEMBER))
455+
456+
generated_value_members.append(gen_member)
458457

458+
elif is_array:
459+
result = unpacked
460+
461+
if export == READ_GEN:
462+
# Turn every element of result into a member
463+
# and put them into an array
464+
array_members = []
465+
allowed_member_type = None
466+
467+
if storage_type is StorageType.ARRAY_INT:
468+
allowed_member_type = StorageType.INT_MEMBER
469+
470+
elif storage_type is StorageType.ARRAY_FLOAT:
471+
allowed_member_type = StorageType.FLOAT_MEMBER
472+
473+
elif storage_type is StorageType.ARRAY_BOOL:
474+
allowed_member_type = StorageType.BOOLEAN_MEMBER
475+
476+
elif storage_type is StorageType.ARRAY_ID:
477+
allowed_member_type = StorageType.ID_MEMBER
478+
479+
elif storage_type is StorageType.ARRAY_STRING:
480+
allowed_member_type = StorageType.STRING_MEMBER
481+
482+
for elem in unpacked:
459483
if storage_type is StorageType.ARRAY_INT:
460-
allowed_member_type = StorageType.INT_MEMBER
484+
gen_member = IntMember(var_name, elem)
485+
array_members.append(gen_member)
461486

462487
elif storage_type is StorageType.ARRAY_FLOAT:
463-
allowed_member_type = StorageType.FLOAT_MEMBER
488+
gen_member = FloatMember(var_name, elem)
489+
array_members.append(gen_member)
464490

465491
elif storage_type is StorageType.ARRAY_BOOL:
466-
allowed_member_type = StorageType.BOOLEAN_MEMBER
492+
gen_member = BooleanMember(var_name, elem)
493+
array_members.append(gen_member)
467494

468495
elif storage_type is StorageType.ARRAY_ID:
469-
allowed_member_type = StorageType.ID_MEMBER
496+
gen_member = IDMember(var_name, elem)
497+
array_members.append(gen_member)
470498

471499
elif storage_type is StorageType.ARRAY_STRING:
472-
allowed_member_type = StorageType.STRING_MEMBER
500+
gen_member = StringMember(var_name, elem)
501+
array_members.append(gen_member)
502+
503+
else:
504+
raise SyntaxError("%s at offset %# 08x: Data read via %s "
505+
"cannot be stored as %s;"
506+
" expected %s, %s, %s, %s or %s"
507+
% (var_name, offset, var_type, storage_type,
508+
StorageType.ARRAY_INT,
509+
StorageType.ARRAY_FLOAT,
510+
StorageType.ARRAY_BOOL,
511+
StorageType.ARRAY_ID,
512+
StorageType.ARRAY_STRING))
513+
514+
# Create the array
515+
array = ArrayMember(var_name, allowed_member_type, array_members)
516+
generated_value_members.append(array)
517+
518+
elif data_count == 1:
519+
# store first tuple element
520+
result = unpacked[0]
521+
522+
if symbol == "f":
523+
if not math.isfinite(result):
524+
raise SyntaxError("invalid float when "
525+
"reading %s at offset %# 08x" % (
526+
var_name, offset))
527+
528+
if is_custom_member:
529+
# save the lookup key / plain value (used for some storage types)
530+
lookup_key = result
473531

474-
for elem in result:
475-
if storage_type is StorageType.ARRAY_INT:
476-
gen_member = IntMember(var_name, elem)
477-
array_members.append(gen_member)
532+
# do an additional lookup to get the stored enum value
533+
result = var_type.entry_hook(result)
478534

479-
elif storage_type is StorageType.ARRAY_FLOAT:
480-
gen_member = FloatMember(var_name, elem)
481-
array_members.append(gen_member)
535+
if isinstance(var_type, EnumLookupMember):
536+
if export == READ_GEN:
537+
# store differently depending on storage type
538+
if storage_type is StorageType.INT_MEMBER:
539+
# store as plain integer value
540+
gen_member = IntMember(var_name, lookup_key)
482541

483-
elif storage_type is StorageType.ARRAY_BOOL:
484-
gen_member = BooleanMember(var_name, elem)
485-
array_members.append(gen_member)
542+
elif storage_type is StorageType.ID_MEMBER:
543+
# store as plain integer value
544+
gen_member = IDMember(var_name, lookup_key)
486545

487-
elif storage_type is StorageType.ARRAY_ID:
488-
gen_member = IDMember(var_name, elem)
489-
array_members.append(gen_member)
546+
elif storage_type is StorageType.BITFIELD_MEMBER:
547+
# store as plain integer value
548+
gen_member = BitfieldMember(var_name, lookup_key)
490549

491-
elif storage_type is StorageType.ARRAY_STRING:
492-
gen_member = StringMember(var_name, elem)
493-
array_members.append(gen_member)
550+
elif storage_type is StorageType.STRING_MEMBER:
551+
# store by looking up value from dict
552+
gen_member = StringMember(var_name, result)
494553

495554
else:
496555
raise SyntaxError("%s at offset %# 08x: Data read via %s "
497556
"cannot be stored as %s;"
498-
" expected %s, %s, %s, %s or %s"
557+
" expected %s, %s, %s or %s"
499558
% (var_name, offset, var_type, storage_type,
500-
StorageType.ARRAY_INT,
501-
StorageType.ARRAY_FLOAT,
502-
StorageType.ARRAY_BOOL,
503-
StorageType.ARRAY_ID,
504-
StorageType.ARRAY_STRING))
505-
506-
# Create the array
507-
array = ArrayMember(var_name, allowed_member_type, array_members)
508-
generated_value_members.append(array)
559+
StorageType.INT_MEMBER,
560+
StorageType.ID_MEMBER,
561+
StorageType.BITFIELD_MEMBER,
562+
StorageType.STRING_MEMBER))
509563

510-
elif data_count == 1:
511-
# store first tuple element
512-
result = result[0]
564+
generated_value_members.append(gen_member)
513565

514-
if symbol == "f":
515-
if not math.isfinite(result):
516-
raise SyntaxError("invalid float when "
517-
"reading %s at offset %# 08x" % (
518-
var_name, offset))
566+
elif isinstance(var_type, ContinueReadMember):
567+
if result == ContinueReadMember.result.ABORT:
568+
# don't go through all other members of this class!
569+
stop_reading_members = True
519570

520-
if export == READ_GEN:
521-
# Store the member as ValueMember
522-
if is_custom_member:
523-
lookup_result = var_type.entry_hook(result)
524-
525-
if isinstance(var_type, EnumLookupMember):
526-
# store differently depending on storage type
527-
if storage_type is StorageType.INT_MEMBER:
528-
# store as plain integer value
529-
gen_member = IntMember(var_name, result)
530-
531-
elif storage_type is StorageType.ID_MEMBER:
532-
# store as plain integer value
533-
gen_member = IDMember(var_name, result)
534-
535-
elif storage_type is StorageType.BITFIELD_MEMBER:
536-
# store as plain integer value
537-
gen_member = BitfieldMember(var_name, result)
538-
539-
elif storage_type is StorageType.STRING_MEMBER:
540-
# store by looking up value from dict
541-
gen_member = StringMember(var_name, lookup_result)
542-
543-
else:
544-
raise SyntaxError("%s at offset %# 08x: Data read via %s "
545-
"cannot be stored as %s;"
546-
" expected %s, %s, %s or %s"
547-
% (var_name, offset, var_type, storage_type,
548-
StorageType.INT_MEMBER,
549-
StorageType.ID_MEMBER,
550-
StorageType.BITFIELD_MEMBER,
551-
StorageType.STRING_MEMBER))
552-
553-
elif isinstance(var_type, ContinueReadMember):
554-
if storage_type is StorageType.BOOLEAN_MEMBER:
555-
gen_member = StringMember(var_name, lookup_result)
556-
557-
else:
558-
raise SyntaxError("%s at offset %# 08x: Data read via %s "
559-
"cannot be stored as %s;"
560-
" expected %s"
561-
% (var_name, offset, var_type, storage_type,
562-
StorageType.BOOLEAN_MEMBER))
563-
564-
else:
565-
if storage_type is StorageType.INT_MEMBER:
566-
gen_member = IntMember(var_name, result)
567-
568-
elif storage_type is StorageType.FLOAT_MEMBER:
569-
gen_member = FloatMember(var_name, result)
570-
571-
elif storage_type is StorageType.BOOLEAN_MEMBER:
572-
gen_member = BooleanMember(var_name, result)
573-
574-
elif storage_type is StorageType.ID_MEMBER:
575-
gen_member = IDMember(var_name, result)
571+
if export == READ_GEN:
572+
if storage_type is StorageType.BOOLEAN_MEMBER:
573+
gen_member = StringMember(var_name, result)
576574

577575
else:
578576
raise SyntaxError("%s at offset %# 08x: Data read via %s "
579577
"cannot be stored as %s;"
580-
" expected %s, %s, %s or %s"
578+
" expected %s"
581579
% (var_name, offset, var_type, storage_type,
582-
StorageType.INT_MEMBER,
583-
StorageType.FLOAT_MEMBER,
584-
StorageType.BOOLEAN_MEMBER,
585-
StorageType.ID_MEMBER))
580+
StorageType.BOOLEAN_MEMBER))
581+
582+
generated_value_members.append(gen_member)
583+
584+
else:
585+
if export == READ_GEN:
586+
if storage_type is StorageType.INT_MEMBER:
587+
gen_member = IntMember(var_name, result)
588+
589+
elif storage_type is StorageType.FLOAT_MEMBER:
590+
gen_member = FloatMember(var_name, result)
591+
592+
elif storage_type is StorageType.BOOLEAN_MEMBER:
593+
gen_member = BooleanMember(var_name, result)
594+
595+
elif storage_type is StorageType.ID_MEMBER:
596+
gen_member = IDMember(var_name, result)
597+
598+
else:
599+
raise SyntaxError("%s at offset %# 08x: Data read via %s "
600+
"cannot be stored as %s;"
601+
" expected %s, %s, %s or %s"
602+
% (var_name, offset, var_type, storage_type,
603+
StorageType.INT_MEMBER,
604+
StorageType.FLOAT_MEMBER,
605+
StorageType.BOOLEAN_MEMBER,
606+
StorageType.ID_MEMBER))
586607

587608
generated_value_members.append(gen_member)
588609

589-
# run entry hook for non-primitive members
590-
if is_custom_member:
591-
result = var_type.entry_hook(result)
610+
# run entry hook for non-primitive members
611+
# if isinstance(var_type, ContinueReadMember) and is_custom_member:
612+
# if is_custom_member:
613+
# result = var_type.entry_hook(unpacked)
592614

593-
if result == ContinueReadMember.result.ABORT:
594-
# don't go through all other members of this class!
595-
stop_reading_members = True
615+
# if result == ContinueReadMember.result.ABORT:
616+
# # don't go through all other members of this class!
617+
# stop_reading_members = True
596618

597-
# store member's data value
598-
setattr(self, var_name, result)
619+
# store member's data value
620+
setattr(self, var_name, result)
599621

600622
# increase the current file position by the size we just read
601623
offset += struct.calcsize(struct_format)

0 commit comments

Comments
 (0)