Merge 'Extend space available for message data' (#1837)

Author: cjohnson57

ASM/build/asm_symbols.txt

@@ -111,6 +111,7 @@ RANDO_CONTEXT:
 .include "drop_overrides/obj_comb.asm"
 .include "drop_overrides/actor.asm"
 .include "rand_seed.asm"
+.include "messages.asm"
 
 .align 0x10
 .importobj "../build/bundle.o"

ASM/src/build.asm

@@ -2999,6 +2999,32 @@ skip_GS_BGS_text:
 .orga 0xB575C8
     sw      t6, 0x00(a1)
 
+; Dynamically load the en/jp message files for text lookup. Both files are utilized to make room
+; for additional text. The jp file is filled first. The segment value for the requested text ID
+; is used to manipulate the language bit to tell Message_OpenText (func_800DC838) which file
+; to load and search. Hook at VRAM 0x800DCB60 in message.s
+.orga 0xB52AC0
+    jal     set_message_file_to_search
+    nop
+
+; The message lookup function uses a fixed reference to the first entry's segment to strip it from
+; the combined segment/offset word. Since we have mixed segments in the table now, this is no
+; longer safe. Load the correct segment into register a2 from the current message.
+.orga 0xB4C980
+    j       load_correct_message_segment
+    nop
+
+; Since message lookup already occurs in the above hook, remove the lookup from both the JP and EN
+; branches.
+.orga 0xB52AD0 ; JP branch
+    nop
+    nop
+    nop
+    nop
+.orga 0xB52B64 ; EN branch
+    nop
+    nop
+
 ;==================================================================================================
 ; Null Boomerang Pointer in Links Instance
 ;==================================================================================================

ASM/src/hacks.asm

@@ -0,0 +1,62 @@
+; Manipulates the save context language bit based
+; on the segment value for the requested text ID.
+; Message lookup is moved here from the vanilla
+; branches.
+; t6 used for branching to JP/EN file loading
+; in parent function func_800DC838 at VRAM 0x800DCB68
+; 0 == JP
+; 1 == EN
+set_message_file_to_search:
+    ; displaced code
+    lhu     a1, 0x0046($sp)
+    lw      a0, 0x0040($sp)
+
+    ; Saved by vanilla code prior to jump to
+    ; message lookup function. Called here so
+    ; we can safely store $ra.
+    sw      t0, 0x002C($sp)
+
+    ; Message lookup saves to the stack without
+    ; changing the stack pointer. Save to an
+    ; unused variable to avoid changing the stack.
+    or      t4, $zero, $ra
+
+    ; call JP message lookup function exclusively
+    ; since the JP/EN tables are merged.
+    jal     0x800D69EC
+    nop
+
+    ; a1 contains the segment/offset word, formatted as
+    ; ssoooooo
+    ; where "s" is the segment and "o" is the offset.
+    ; Vanilla crashes on a failed text lookup already,
+    ; so we don't need to worry about bad values.
+    srl     t0, a1, 0x18
+    andi    t0, t0, 0x08
+    sltiu   t6, t0, 0x0001
+
+    or      $ra, $zero, t4
+    jr      $ra
+    nop
+
+load_correct_message_segment:
+    ; displaced code
+    lbu     t6, 0x0002(v1)
+    lw      a1, 0x0004(v1)
+
+    ; shift out the offset bits
+    srl     a2, a1, 0x18
+    sll     a2, a2, 0x18
+
+    ; Remaining code from func_800D69EC since
+    ; it doesn't save $ra
+    lw      a1, 0x0004(v1)
+    addiu   v0, a0, 0x2200
+    sb      t6, 0x0008(v0)
+    lw      a3, 0x000C(v1)
+    subu    t7, a1, a2
+    addiu   v1, v1, 0x0008
+    subu    t8, a3, a1
+    sw      t7, 0x0000(v0)
+    jr      $ra
+    sw      t8, 0x0004(v0)

ASM/src/messages.asm

@@ -1037,7 +1037,7 @@ def get_junk_hint(spoiler, world, checked):
 def get_important_check_hint(spoiler, world, checked):
     top_level_locations = []
     for location in world.get_filled_locations():
-        if (HintArea.at(location).text(world.settings.clearer_hints) not in top_level_locations 
+        if (HintArea.at(location).text(world.settings.clearer_hints) not in top_level_locations
             and (HintArea.at(location).text(world.settings.clearer_hints) + ' Important Check') not in checked
             and "pocket" not in HintArea.at(location).text(world.settings.clearer_hints)):
             top_level_locations.append(HintArea.at(location).text(world.settings.clearer_hints))
@@ -1059,7 +1059,7 @@ def get_important_check_hint(spoiler, world, checked):
                 or (location.item.type == 'SmallKey' and not (world.settings.shuffle_smallkeys == 'dungeon' or world.settings.shuffle_smallkeys == 'vanilla'))
                 or (location.item.type == 'HideoutSmallKey' and not world.settings.shuffle_hideoutkeys == 'vanilla')
                 or (location.item.type == 'BossKey' and not (world.settings.shuffle_bosskeys == 'dungeon' or world.settings.shuffle_bosskeys == 'vanilla'))
-                or (location.item.type == 'GanonBossKey' and not (world.settings.shuffle_ganon_bosskey == 'vanilla' 
+                or (location.item.type == 'GanonBossKey' and not (world.settings.shuffle_ganon_bosskey == 'vanilla'
                 or world.settings.shuffle_ganon_bosskey == 'dungeon' or world.settings.shuffle_ganon_bosskey == 'on_lacs'
                 or world.settings.shuffle_ganon_bosskey == 'stones' or world.settings.shuffle_ganon_bosskey == 'medallions'
                 or world.settings.shuffle_ganon_bosskey == 'dungeons' or world.settings.shuffle_ganon_bosskey == 'tokens'))):

Hints.py

@@ -5,9 +5,10 @@
 from TextBox import line_wrap
 from Utils import find_last
 
-TEXT_START = 0x92D000
+ENG_TEXT_START = 0x92D000
+JPN_TEXT_START = 0x8EB000
 ENG_TEXT_SIZE_LIMIT = 0x39000
-JPN_TEXT_SIZE_LIMIT = 0x3A150
+JPN_TEXT_SIZE_LIMIT = 0x3B000
 
 JPN_TABLE_START = 0xB808AC
 ENG_TABLE_START = 0xB849EC
@@ -19,6 +20,9 @@
 EXTENDED_TABLE_START = JPN_TABLE_START # start writing entries to the jp table instead of english for more space
 EXTENDED_TABLE_SIZE = JPN_TABLE_SIZE + ENG_TABLE_SIZE # 0x8360 bytes, 4204 entries
 
+EXTENDED_TEXT_START = JPN_TABLE_START # start writing text to the jp table instead of english for more space
+EXTENDED_TEXT_SIZE_LIMIT = JPN_TEXT_SIZE_LIMIT + ENG_TEXT_SIZE_LIMIT # 0x74000 bytes
+
 # name of type, followed by number of additional bytes to read, follwed by a function that prints the code
 CONTROL_CODES = {
     0x00: ('pad', 0, lambda _: '<pad>' ),
@@ -455,14 +459,14 @@ def size(self):
         return size
 
     # writes the code to the given offset, and returns the offset of the next byte
-    def write(self, rom, offset):
-        rom.write_byte(TEXT_START + offset, self.code)
+    def write(self, rom, text_start, offset):
+        rom.write_byte(text_start + offset, self.code)
 
         extra_bytes = 0
         if self.code in CONTROL_CODES:
             extra_bytes = CONTROL_CODES[self.code][1]
             bytes_to_write = int_to_bytes(self.data, extra_bytes)
-            rom.write_bytes(TEXT_START + offset + 1, bytes_to_write)
+            rom.write_bytes(text_start + offset + 1, bytes_to_write)
 
         return offset + 1 + extra_bytes
 
@@ -498,7 +502,7 @@ def get_python_string(self):
 
     # check if this is an unused message that just contains it's own id as text
     def is_id_message(self):
-        if self.unpadded_length != 5:
+        if self.unpadded_length != 5 or self.id == 0xFFFC:
             return False
         for i in range(4):
             code = self.text_codes[i].code
@@ -609,20 +613,20 @@ def transform(self, replace_ending=False, ending=None, always_allow_skip=True, s
 
     # writes a Message back into the rom, using the given index and offset to update the table
     # returns the offset of the next message
-    def write(self, rom, index, offset):
+    def write(self, rom, index, text_start, offset, bank):
         # construct the table entry
         id_bytes = int_to_bytes(self.id, 2)
         offset_bytes = int_to_bytes(offset, 3)
-        entry = id_bytes + bytes([self.opts, 0x00, 0x07]) + offset_bytes
+        entry = id_bytes + bytes([self.opts, 0x00, bank]) + offset_bytes
         # write it back
         entry_offset = EXTENDED_TABLE_START + 8 * index
         rom.write_bytes(entry_offset, entry)
 
         for code in self.text_codes:
-            offset = code.write(rom, offset)
+            offset = code.write(rom, text_start, offset)
 
         while offset % 4 > 0:
-            offset = Text_Code(0x00, 0).write(rom, offset) # pad to 4 byte align
+            offset = Text_Code(0x00, 0).write(rom, text_start, offset) # pad to 4 byte align
 
         return offset
 
@@ -651,8 +655,14 @@ def __init__(self, raw_text, index, id, opts, offset, length):
 
     # read a single message from rom
     @classmethod
-    def from_rom(cls, rom, index):
-        entry_offset = ENG_TABLE_START + 8 * index
+    def from_rom(cls, rom, index, eng=True):
+        if eng:
+            table_start = ENG_TABLE_START
+            text_start = ENG_TEXT_START
+        else:
+            table_start = JPN_TABLE_START
+            text_start = JPN_TEXT_START
+        entry_offset = table_start + 8 * index
         entry = rom.read_bytes(entry_offset, 8)
         next = rom.read_bytes(entry_offset + 8, 8)
 
@@ -661,7 +671,7 @@ def from_rom(cls, rom, index):
         offset = bytes_to_int(entry[5:8])
         length = bytes_to_int(next[5:8]) - offset
 
-        raw_text = rom.read_bytes(TEXT_START + offset, length)
+        raw_text = rom.read_bytes(text_start + offset, length)
 
         return cls(raw_text, index, id, opts, offset, length)
 
@@ -922,19 +932,50 @@ def read_messages(rom):
         index += 1
         table_offset += 8
 
+    # Also grab 0xFFFC entry from JP table.
+    messages.append(read_fffc_message(rom))
     return messages
 
+# The JP text table is the only source for ID 0xFFFC, which is used by the
+# title and file select screens. Preserve this table entry and text data when
+# overwriting the JP data. The regular read_messages function only reads English
+# data.
+def read_fffc_message(rom):
+    table_offset = JPN_TABLE_START
+    index = 0
+    while True:
+        entry = rom.read_bytes(table_offset, 8)
+        id = bytes_to_int(entry[0:2])
+
+        if id == 0xFFFC:
+            message = Message.from_rom(rom, index, eng=False)
+            break
+
+        index += 1
+        table_offset += 8
+
+    return message
+
 # write the messages back
 def repack_messages(rom, messages, permutation=None, always_allow_skip=True, speed_up_text=True):
 
-    rom.update_dmadata_record(TEXT_START, TEXT_START, TEXT_START + ENG_TEXT_SIZE_LIMIT)
+    rom.update_dmadata_record(ENG_TEXT_START, ENG_TEXT_START, ENG_TEXT_START + ENG_TEXT_SIZE_LIMIT)
+    rom.update_dmadata_record(JPN_TEXT_START, JPN_TEXT_START, JPN_TEXT_START + JPN_TEXT_SIZE_LIMIT)
 
     if permutation is None:
         permutation = range(len(messages))
 
     # repack messages
     offset = 0
-    text_size_limit = ENG_TEXT_SIZE_LIMIT
+    text_start = JPN_TEXT_START
+    text_size_limit = EXTENDED_TEXT_SIZE_LIMIT
+    text_bank = 0x08 # start with the Japanese text bank
+    jp_bytes = 0
+    # An extra dummy message is inserted after exhausting the JP text file.
+    # Written message IDs are independent of the python list index, but the
+    # index has to be maintained for old/new lookups. This wouldn't be an
+    # issue if text shuffle didn't exist.
+    jp_index_offset = 0
 
     for old_index, new_index in enumerate(permutation):
         old_message = messages[old_index]
@@ -943,21 +984,57 @@ def repack_messages(rom, messages, permutation=None, always_allow_skip=True, spe
         new_message.id = old_message.id
 
         # modify message, making it represent how we want it to be written
-        new_message.transform(True, old_message.ending, always_allow_skip, speed_up_text)
+        if new_message.id != 0xFFFC:
+            new_message.transform(True, old_message.ending, always_allow_skip, speed_up_text)
+
+        # check if there is space to write the message
+        message_size = new_message.size()
+        if message_size + offset > JPN_TEXT_SIZE_LIMIT and text_start == JPN_TEXT_START:
+            # Add a dummy entry to the table for the last entry in the
+            # JP file. This is used by the game to calculate message
+            # length. Since the next entry in the English table has an
+            # offset of zero, which would lead to a negative length.
+            # 0xFFFD is used as the text ID for this in vanilla.
+            # Text IDs need to be in order across the table for the
+            # split to work.
+            entry = bytes([0xFF, 0xFD, 0x00, 0x00, text_bank]) + int_to_bytes(offset, 3)
+            entry_offset = EXTENDED_TABLE_START + 8 * old_index
+            rom.write_bytes(entry_offset, entry)
+            # if there is no room then switch to the English text bank
+            text_bank = 0x07
+            text_start = ENG_TEXT_START
+            jp_bytes = offset
+            jp_index_offset = 1
+            offset = 0
+
+        # Special handling for text ID 0xFFFC, which has hard-coded offsets to
+        # the JP file in function Font_LoadOrderedFont in z_kanfont.c
+        if new_message.id == 0xFFFC:
+            # hard-coded offset including segment
+            rom.write_int16(0xAD1CE2, (text_bank << 8) + ((offset & 0xFFFF0000) >> 16) + (1 if offset & 0xFFFF > 0x8000 else 0))
+            rom.write_int16(0xAD1CE6, offset & 0XFFFF)
+            # hard-coded message length, represented by offset of end of message
+            rom.write_int16(0xAD1D16, (text_bank << 8) + (((offset + new_message.size()) & 0xFFFF0000) >> 16) + (1 if (offset + new_message.size()) & 0xFFFF > 0x8000 else 0))
+            rom.write_int16(0xAD1D1E, (offset + new_message.size()) & 0XFFFF)
+            # hard-coded segment, default JP file (0x08)
+            rom.write_int16(0xAD1D12, (text_bank << 8))
+            # hard-coded text file start address in rom, default JP
+            rom.write_int16(0xAD1D22, ((text_start & 0xFFFF0000) >> 16) + (1 if text_start & 0xFFFF > 0x8000 else 0))
+            rom.write_int16(0xAD1D2E, text_start & 0XFFFF)
 
         # actually write the message
-        offset = new_message.write(rom, old_index, offset)
+        offset = new_message.write(rom, old_index + jp_index_offset, text_start, offset, text_bank)
 
         new_message.id = remember_id
 
     # raise an exception if too much is written
     # we raise it at the end so that we know how much overflow there is
-    if offset > text_size_limit:
-        raise(TypeError("Message Text table is too large: 0x" + "{:x}".format(offset) + " written / 0x" + "{:x}".format(ENG_TEXT_SIZE_LIMIT) + " allowed."))
+    if jp_bytes + offset > text_size_limit:
+        raise(TypeError("Message Text table is too large: 0x" + "{:x}".format(jp_bytes + offset) + " written / 0x" + "{:x}".format(EXTENDED_TEXT_SIZE_LIMIT) + " allowed."))
 
-    # end the table
-    table_index = len(messages)
-    entry = bytes([0xFF, 0xFD, 0x00, 0x00, 0x07]) + int_to_bytes(offset, 3)
+    # end the table, accounting for additional entry for file split
+    table_index = len(messages) + (1 if text_bank == 0x07 else 0)
+    entry = bytes([0xFF, 0xFD, 0x00, 0x00, text_bank]) + int_to_bytes(offset, 3)
     entry_offset = EXTENDED_TABLE_START + 8 * table_index
     rom.write_bytes(entry_offset, entry)
     table_index += 1
@@ -982,6 +1059,7 @@ def is_exempt(m):
         )
         shuffle_exempt = [
             0x208D,         # "One more lap!" for Cow in House race.
+            0xFFFC,         # Character data from JP table used on title and file select screens
         ]
         is_hint = (except_hints and m.id in hint_ids)
         is_error_message = (m.id == ERROR_MESSAGE)

Messages.py

@@ -0,0 +1,16 @@
+Message_FindMessage (debug) -> func_800D6A90 (1.0 EN) / func_800D69EC (1.0 JP)
+save context + 0x2200 = play->msgCtx.font
+line 12279 in data.s in disassembly
+80112E34 = pointer to JP table (8010A94C) line 10152
+80112E38 = pointer to EN table (8010EA8C) line 11196
+credit table? 80112CAC line 12254
+findmessage called near lbl_800DCB60 (JP) / lbl_800DCC04 (EN)
+
+text ID FFFC hard-coded offsets for JP file in func_8005BD78 / Font_LoadOrderedFont in kanfont.s / z_kanfont.c
+DMA request needs to be modified to point to EN file if FFFC relocated there
+a3 = 0803A150 change to actual FFFC offset, modify segment to 07 if in EN file
+t6 = 08000000 change to 07000000 if FFFC in EN file
+t7 = 0803A340 change to actual FFFD offset, modify segment to 07 if in EN file (used for FFFC length)
+t8 = 008EB000 change to 0092D000 for start of file pointer
+
+

Notes/message_table_notes.txt

@@ -128,6 +128,7 @@ issue. You should always Hard Reset to avoid this issue entirely.
 * The model used for Ruto's Letter is now rotated onto its side to better differentiate from other bottles.
 * The dummy boss key chest on the wall of Forest Temple's twisted hallway now matches the real version of the chest in the straightened version of the room.
 * The Kakariko Well water will no longer be up as adult to facilitate a glitch strategy for entering the well.
+* The message table has been extended to allow further developments which require more added textboxes.
 
 #### Plandomizer
 * Plandomizer now allows you to specify locations that are valid but do not exist in your current seed, for example, an MQ-specific location when that dungeon is Vanilla.