[jak3] Set up ckernel (#3308)

This sets up the C Kernel for Jak 3, and makes it possible to build and
load code built with `goalc --jak3`.

There's not too much interesting here, other than they switched to a
system where symbol IDs (unique numbers less than 2^14) are generated at
compile time, and those get included in the object file itself.

This is kind of annoying, since it means all tools that produce a GOAL
object file need to work together to assign unique symbol IDs. And since
the symbol IDs can't conflict, and are only a number between 0 and 2^14,
you can't just hash and hope for no collisions.

We work around this by ignoring the IDs and re-assigning our own. I
think this is very similar to what the C Kernel did on early builds of
Jak 3 which supported loading old format level files, which didn't have
the IDs included.

As far as I can tell, this shouldn't cause any problems. It defeats all
of their fancy tricks to save memory by not storing the symbol string,
but we don't care.

Author: water111

common/formatter/formatter.cpp

@@ -191,7 +191,7 @@ bool can_node_be_inlined(const FormatterTreeNode& curr_node, int cursor_pos) {
 }
 
 std::vector<std::string> apply_formatting(const FormatterTreeNode& curr_node,
-                                          std::vector<std::string> output = {},
+                                          std::vector<std::string> /*output*/ = {},
                                           int cursor_pos = 0) {
   using namespace formatter_rules;
   if (!curr_node.token && curr_node.refs.empty()) {

common/formatter/rules/rule_config.cpp

@@ -9,7 +9,7 @@ namespace config {
 FormFormattingConfig new_flow_rule(int start_index) {
   FormFormattingConfig cfg;
   cfg.hang_forms = false;
-  cfg.inline_until_index = [start_index](const std::vector<std::string>& curr_lines) {
+  cfg.inline_until_index = [start_index](const std::vector<std::string>& /*curr_lines*/) {
     return start_index;
   };
   return cfg;

common/goal_constants.h

@@ -62,15 +62,9 @@ constexpr int LEVEL_MAX = 6;
 constexpr int LEVEL_TOTAL = LEVEL_MAX + 1;
 }  // namespace jak2
 
-// TODO copypaste from jak 2 for now
 namespace jak3 {
 // for now, we don't have the ability to extend the size of the symbol table
 constexpr s32 GOAL_MAX_SYMBOLS = 0x4000;
-constexpr s32 SYM_TABLE_MEM_SIZE = 0x30000;
-// from the "off-by-one" symbol pointer
-constexpr int SYM_TO_STRING_OFFSET = 0xff37;
-constexpr int SYM_TO_HASH_OFFSET = 0x1fe6f;
-
 // amount of levels in level heap
 constexpr int LEVEL_MAX = 10;
 // total amount of levels, including ones outside level heap (default-level)
@@ -84,7 +78,7 @@ constexpr s32 max_symbols(GameVersion version) {
     case GameVersion::Jak2:
       return jak2::GOAL_MAX_SYMBOLS;
     case GameVersion::Jak3:
-      return jak2::GOAL_MAX_SYMBOLS;
+      return jak3::GOAL_MAX_SYMBOLS;
   }
 }
 

common/link_types.h

@@ -54,6 +54,21 @@ struct LinkHeaderV4 {
   uint32_t code_size;  // length of object data before link data starts
 };
 
+struct LinkHeaderV5Core {
+  uint32_t length_to_get_to_code;  // 4 length.. of link data?
+  uint16_t version;                // 8
+  uint16_t unknown;                // 10
+  uint32_t length_to_get_to_link;  // 12
+  uint32_t link_length;            // 16
+  uint8_t n_segments;              // 20
+  char name[59];                   // 21 (really??)
+};
+
+struct LinkHeaderV5 {
+  uint32_t type_tag;  // 0 always 0?
+  LinkHeaderV5Core core;
+};
+
 // when a u32/s32 symbol link contains this value, (s7 + <val>) should be a 4-byte aligned address,
 // not including the 1 byte symbol offset. (no effect in jak 1).
 constexpr u32 LINK_SYM_NO_OFFSET_FLAG = 0xbadbeef;

common/symbols.h

@@ -150,6 +150,180 @@ constexpr int FIX_FIXED_SYM_END_OFFSET = 0xec;
 
 }  // namespace jak2_symbols
 
+namespace jak3_symbols {
+
+constexpr int FIX_SYM_EMPTY_CAR = -0x8;
+constexpr int S7_OFF_FIX_SYM_EMPTY_PAIR = -0x6 - 1;
+constexpr int FIX_SYM_EMPTY_CDR = -0x4;
+constexpr int FIX_SYM_FALSE = 0x0;  // GOAL boolean #f (note that this is equal to the $s7 register)
+constexpr int FIX_SYM_TRUE = 0x4;   // GOAL boolean #t
+constexpr int FIX_SYM_FUNCTION_TYPE = 0x8;
+constexpr int FIX_SYM_BASIC = 0xc;
+constexpr int FIX_SYM_STRING_TYPE = 0x10;
+constexpr int FIX_SYM_SYMBOL_TYPE = 0x14;
+constexpr int FIX_SYM_TYPE_TYPE = 0x18;
+constexpr int FIX_SYM_OBJECT_TYPE = 0x1c;
+constexpr int FIX_SYM_LINK_BLOCK = 0x20;
+constexpr int FIX_SYM_INTEGER = 0x24;
+constexpr int FIX_SYM_SINTEGER = 0x28;
+constexpr int FIX_SYM_UINTEGER = 0x2c;
+constexpr int FIX_SYM_BINTEGER = 0x30;
+constexpr int FIX_SYM_INT8 = 0x34;
+constexpr int FIX_SYM_INT16 = 0x38;
+constexpr int FIX_SYM_INT32 = 0x3c;
+constexpr int FIX_SYM_INT64 = 0x40;
+constexpr int FIX_SYM_INT128 = 0x44;
+constexpr int FIX_SYM_UINT8 = 0x48;
+constexpr int FIX_SYM_UINT16 = 0x4c;
+constexpr int FIX_SYM_UINT32 = 0x50;
+constexpr int FIX_SYM_UINT64 = 0x54;
+constexpr int FIX_SYM_UINT128 = 0x58;
+constexpr int FIX_SYM_FLOAT = 0x5c;
+constexpr int FIX_SYM_PROCESS_TREE = 0x60;
+constexpr int FIX_SYM_PROCESS_TYPE = 0x64;
+constexpr int FIX_SYM_THREAD = 0x68;
+constexpr int FIX_SYM_STRUCTURE = 0x6c;
+constexpr int FIX_SYM_PAIR_TYPE = 0x70;
+constexpr int FIX_SYM_POINTER = 0x74;
+constexpr int FIX_SYM_NUMBER = 0x78;
+constexpr int FIX_SYM_ARRAY = 0x7c;
+constexpr int FIX_SYM_VU_FUNCTION = 0x80;
+constexpr int FIX_SYM_CONNECTABLE = 0x84;
+constexpr int FIX_SYM_STACK_FRAME = 0x88;
+constexpr int FIX_SYM_FILE_STREAM = 0x8c;
+constexpr int FIX_SYM_HEAP = 0x90;
+constexpr int FIX_SYM_NOTHING_FUNC = 0x94;
+constexpr int FIX_SYM_DELETE_BASIC = 0x98;
+constexpr int FIX_SYM_STATIC = 0x9c;
+constexpr int FIX_SYM_GLOBAL_HEAP = 0xa0;
+constexpr int FIX_SYM_DEBUG = 0xa4;
+constexpr int FIX_SYM_LOADING_LEVEL = 0xa8;
+constexpr int FIX_SYM_LOADING_PACKAGE = 0xac;
+constexpr int FIX_SYM_PROCESS_LEVEL_HEAP = 0xb0;
+constexpr int FIX_SYM_STACK = 0xb4;
+constexpr int FIX_SYM_SCRATCH = 0xb8;
+constexpr int FIX_SYM_SCRATCH_TOP = 0xbc;
+constexpr int FIX_SYM_ZERO_FUNC = 0xc0;
+constexpr int FIX_SYM_ASIZE_OF_BASIC_FUNC = 0xc4;
+constexpr int FIX_SYM_COPY_BASIC_FUNC = 0xc8;  // bugged name
+constexpr int FIX_SYM_LEVEL = 0xcc;
+constexpr int FIX_SYM_ART_GROUP = 0xd0;
+constexpr int FIX_SYM_TEXTURE_PAGE_DIR = 0xd4;
+constexpr int FIX_SYM_TEXTURE_PAGE = 0xd8;
+constexpr int FIX_SYM_SOUND = 0xdc;
+constexpr int FIX_SYM_DGO = 0xe0;
+constexpr int FIX_SYM_TOP_LEVEL = 0xe4;
+constexpr int FIX_SYM_QUOTE = 0xe8;
+constexpr int FIX_SYM_LISTENER_LINK_BLOCK = 0xec;
+constexpr int FIX_SYM_LISTENER_FUNCTION = 0xf0;
+constexpr int FIX_SYM_STACK_TOP = 0xf4;
+constexpr int FIX_SYM_STACK_BASE = 0xf8;
+constexpr int FIX_SYM_STACK_SIZE = 0xfc;
+constexpr int FIX_SYM_KERNEL_FUNCTION = 0x100;
+constexpr int FIX_SYM_KERNEL_PACKAGES = 0x104;
+constexpr int FIX_SYM_KERNEL_BOOT_MESSAGE = 0x108;
+constexpr int FIX_SYM_KERNEL_BOOT_MODE = 0x10c;
+constexpr int FIX_SYM_KERNEL_BOOT_LEVEL = 0x110;
+constexpr int FIX_SYM_KERNEL_BOOT_ART_GROUP = 0x114;
+constexpr int FIX_SYM_KERNEL_DEBUG = 0x118;
+constexpr int FIX_SYM_KERNEL_VERSION = 0x11c;
+constexpr int FIX_SYM_KERNEL_DISPATCHER = 0x120;
+constexpr int FIX_SYM_SYNC_DISPATCHER = 0x124;
+constexpr int FIX_SYM_PRINT_COLLUMN = 0x128;
+constexpr int FIX_SYM_DEBUG_SEGMENT = 300;
+constexpr int FIX_SYM_ENABLE_METHOD_SET = 0x130;
+constexpr int FIX_SYM_SQL_RESULT = 0x134;
+constexpr int FIX_SYM_COLLAPSE_QUOTE = 0x138;
+constexpr int FIX_SYM_LEVEL_TYPE_LIST = 0x13C;
+constexpr int FIX_SYM_DECI_COUNT = 0x140;
+constexpr int FIX_SYM_USER = 0x144;
+constexpr int FIX_SYM_VIDEO_MODE = 0x148;
+constexpr int FIX_SYM_BOOT_VIDEO_MODE = 0x14C;
+constexpr int FIX_SYM_BOOT = 0x150;
+constexpr int FIX_SYM_DEMO = 0x154;
+constexpr int FIX_SYM_DEMO_SHARED = 0x158;
+constexpr int FIX_SYM_PREVIEW = 0x15C;
+constexpr int FIX_SYM_KIOSK = 0x160;
+constexpr int FIX_SYM_PLAY_BOOT = 0x164;
+constexpr int FIX_SYM_SIN = 0x168;
+constexpr int FIX_SYM_COS = 0x16C;
+constexpr int FIX_SYM_PUT_DISPLAY_ENV = 0x170;
+constexpr int FIX_SYM_SYNCV = 0x174;
+constexpr int FIX_SYM_SYNC_PATH = 0x178;
+constexpr int FIX_SYM_RESET_PATH = 0x17C;
+constexpr int FIX_SYM_RESET_GRAPH = 0x180;
+constexpr int FIX_SYM_DMA_SYNC = 0x184;
+constexpr int FIX_SYM_GS_PUT_IMR = 0x188;
+constexpr int FIX_SYM_GS_GET_IMR = 0x18C;
+constexpr int FIX_SYM_GS_STORE_IMAGE = 400;
+constexpr int FIX_SYM_FLUSH_CACHE = 0x194;
+constexpr int FIX_SYM_CPAD_OPEN = 0x198;
+constexpr int FIX_SYM_CPAD_GET_DATA = 0x19C;
+constexpr int FIX_SYM_MOUSE_GET_DATA = 0x1A0;
+constexpr int FIX_SYM_KEYBD_GET_DATA = 0x1A4;
+constexpr int FIX_SYM_INSTALL_HANDLER = 0x1A8;
+constexpr int FIX_SYM_INSTALL_DEBUG_HANDLER = 0x1AC;
+constexpr int FIX_SYM_FILE_STREAM_OPEN = 0x1B0;
+constexpr int FIX_SYM_FILE_STREAM_CLOSE = 0x1B4;
+constexpr int FIX_SYM_FILE_STREAM_LENGTH = 0x1B8;
+constexpr int FIX_SYM_FILE_STREAM_SEEK = 0x1BC;
+constexpr int FIX_SYM_FILE_STREAM_READ = 0x1C0;
+constexpr int FIX_SYM_FILE_STREAM_WRITE = 0x1C4;
+constexpr int FIX_SYM_SCF_GET_LANGUAGE = 0x1C8;
+constexpr int FIX_SYM_SCF_GET_TIME = 0x1CC;
+constexpr int FIX_SYM_SCF_GET_ASPECT = 0x1D0;
+constexpr int FIX_SYM_SCF_GET_VOLUME = 0x1D4;
+constexpr int FIX_SYM_SCF_GET_TERRITORY = 0x1D8;
+constexpr int FIX_SYM_SCF_GET_TIMEOUT = 0x1DC;
+constexpr int FIX_SYM_SCF_GET_INACTIVE_TIMEOUT = 0x1E0;
+constexpr int FIX_SYM_DMA_TO_IOP = 0x1E4;
+constexpr int FIX_SYM_KERNEL_SHUTDOWN = 0x1E8;
+constexpr int FIX_SYM_AYBABTU = 0x1EC;
+constexpr int FIX_SYM_STRING_TO_SYMBOL = 0x1F0;
+constexpr int FIX_SYM_SYMBOL_TO_STRING = 500;
+constexpr int FIX_SYM_PRINT = 0x1F8;
+constexpr int FIX_SYM_INSPECT = 0x1FC;
+constexpr int FIX_SYM_LOAD = 0x200;
+constexpr int FIX_SYM_LOADB = 0x204;
+constexpr int FIX_SYM_LOADO = 0x208;
+constexpr int FIX_SYM_UNLOAD = 0x20C;
+constexpr int FIX_SYM_FORMAT = 0x210;
+constexpr int FIX_SYM_MALLOC = 0x214;
+constexpr int FIX_SYM_KMALLOC = 0x218;
+constexpr int FIX_SYM_KMEMOPEN = 0x21C;
+constexpr int FIX_SYM_KMEMCLOSE = 0x220;
+constexpr int FIX_SYM_NEW_DYNAMIC_STRUCTURE = 0x224;
+constexpr int FIX_SYM_METHOD_SET = 0x228;
+constexpr int FIX_SYM_LINK = 0x22C;
+constexpr int FIX_SYM_LINK_BUSY = 0x230;
+constexpr int FIX_SYM_LINK_RESET = 0x234;
+constexpr int FIX_SYM_LINK_BEGIN = 0x238;
+constexpr int FIX_SYM_LINK_RESUME = 0x23C;
+constexpr int FIX_SYM_DGO_LOAD = 0x240;
+constexpr int FIX_SYM_SQL_QUERY = 0x244;
+constexpr int FIX_SYM_MC_RUN = 0x248;
+constexpr int FIX_SYM_MC_FORMAT = 0x24C;
+constexpr int FIX_SYM_MC_UNFORMAT = 0x250;
+constexpr int FIX_SYM_MC_CREATE_FILE = 0x254;
+constexpr int FIX_SYM_MC_SAVE = 600;
+constexpr int FIX_SYM_MC_LOAD = 0x25C;
+constexpr int FIX_SYM_MC_CHECK_RESULT = 0x260;
+constexpr int FIX_SYM_MC_GET_SLOT_INFO = 0x264;
+constexpr int FIX_SYM_MC_MAKEFILE = 0x268;
+constexpr int FIX_SYM_KSET_LANGUAGE = 0x26C;
+constexpr int FIX_SYM_RPC_CALL = 0x270;
+constexpr int FIX_SYM_RPC_BUSY = 0x274;
+constexpr int FIX_SYM_TEST_LOAD_DGO_C = 0x278;
+constexpr int FIX_SYM_SYMLINK2 = 0x27c;
+constexpr int FIX_SYM_SYMLINK3 = 0x280;
+constexpr int FIX_SYM_ULTIMATE_MEMCPY = 0x284;
+constexpr int FIX_SYM_PLAY = 0x288;
+constexpr int FIX_SYM_SYMBOL_STRING = 0x28c;
+constexpr int FIX_SYM_KERNEL_SYMBOL_WARNINGS = 0x290;
+constexpr int FIX_SYM_NETWORK_BOOTSTRAP = 0x294;
+
+}  // namespace jak3_symbols
+
 constexpr int false_symbol_offset() {
   return jak1_symbols::FIX_SYM_FALSE;
 }

common/versions/versions.h

@@ -34,6 +34,7 @@ constexpr u32 TX_PAGE_VERSION = 8;
 namespace jak3 {
 constexpr u32 ART_FILE_VERSION = 8;
 constexpr u32 LEVEL_FILE_VERSION = 36;
+constexpr u32 DGO_FILE_VERSION = 1;
 constexpr u32 TX_PAGE_VERSION = 8;
 }  // namespace jak3
 

decompiler/ObjectFile/LinkedObjectFileCreation.cpp

@@ -472,7 +472,7 @@ static void link_v5(LinkedObjectFile& f,
   for (int i = 0; i < n_segs; i++) {
     segment_data_offsets[i] = header->length_to_get_to_code + seg_info_array[i].data;
     segment_link_offsets[i] = header->length_to_get_to_link + seg_info_array[i].relocs;
-    ASSERT(seg_info_array[i].magic == 1);
+    ASSERT(seg_info_array[i].magic == 1);  // if set, always use symlink2.
   }
 
   // check that the data region is filled

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -367,7 +367,7 @@ void ObjectFileDB::ir2_analyze_all_types(const fs::path& output_file,
         for (const auto& [obj_name, type_names] : all_type_names) {
           for (const auto& type_name : type_names) {
             if (str_util::starts_with(sym_name, type_name) &&
-                type_name.length() > longest_match_length) {
+                (int)type_name.length() > longest_match_length) {
               longest_match_length = type_name.length();
               longest_match = type_name;
               longest_match_object_name = obj_name;