tests: add jak3 typeconsistency test and ensure offline tests are working (#3310)

Author: xTVaser

.vs/launch.vs.json

@@ -68,6 +68,18 @@
         "jak2"
       ]
     },
+    {
+      "type": "default",
+      "project": "CMakeLists.txt",
+      "projectTarget": "offline-test.exe (bin\\offline-test.exe)",
+      "name": "Tests - Offline Tests - Jak 3",
+      "args": [
+        "--iso_data_path",
+        "${workspaceRoot}/iso_data/jak3",
+        "--game",
+        "jak3"
+      ]
+    },
     {
       "type": "default",
       "project": "CMakeLists.txt",

Taskfile.yml

@@ -90,7 +90,7 @@ tasks:
       - cmd: python ./scripts/cpp/format-includes.py
       - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp -i
       - task: format-json
-  
+
   # DECOMPILING
   decomp:
     cmds:
@@ -165,9 +165,6 @@ tasks:
         ignore_error: true
       - python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ --game {{.GAME}}
       - task: offline-test-file
-  type-test-jak1:
-    cmds:
-      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak1TypeConsistency*" --gtest_break_on_failure'
   type-test:
     cmds:
-      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak2TypeConsistency*" --gtest_break_on_failure'
+      - cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*{{.TYPE_CONSISTENCY_TEST_FILTER}}*" --gtest_break_on_failure'

common/goos/Reader.cpp

@@ -238,9 +238,14 @@ Object Reader::read_from_string(const std::string& str,
  * Read a file
  */
 Object Reader::read_from_file(const std::vector<std::string>& file_path, bool check_encoding) {
-  std::string joined_path = fmt::format("{}", fmt::join(file_path, "/"));
+  std::string file_descriptor = fmt::format("{}", fmt::join(file_path, "/"));
+  const auto joined_file_path = file_util::get_file_path(file_path);
 
-  auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_path);
+  if (!fs::exists(joined_file_path)) {
+    throw std::runtime_error(fmt::format("Cannot read {}, file doesn't exist", joined_file_path));
+  }
+
+  auto textFrag = std::make_shared<FileText>(joined_file_path, file_descriptor);
   db.insert(textFrag);
 
   auto result = internal_read(textFrag, check_encoding);

common/goos/TextDB.cpp

@@ -85,9 +85,9 @@ std::pair<int, int> SourceText::get_containing_line(int offset) {
 /*!
  * Read text from a file.
  */
-FileText::FileText(const std::string& filename, const std::string& description_name)
-    : m_filename(filename), m_desc_name(description_name) {
-  m_text = file_util::read_text_file(m_filename);
+FileText::FileText(const std::string& file_path, const std::string& description_name)
+    : m_filepath(file_path), m_desc_name(description_name) {
+  m_text = file_util::read_text_file(m_filepath);
   build_offsets();
 }
 

common/goos/TextDB.h

@@ -80,13 +80,13 @@ class ProgramString : public SourceText {
  */
 class FileText : public SourceText {
  public:
-  FileText(const std::string& filename, const std::string& description_name);
+  FileText(const std::string& file_path, const std::string& description_name);
 
   std::string get_description() { return m_desc_name; }
   ~FileText() = default;
 
  private:
-  std::string m_filename;
+  std::string m_filepath;
   std::string m_desc_name;
 };
 

common/util/FileUtil.cpp

@@ -636,6 +636,9 @@ std::vector<fs::path> find_files_in_dir(const fs::path& dir, const std::regex& p
 
 std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std::regex& pattern) {
   std::vector<fs::path> files = {};
+  if (!fs::exists(base_dir)) {
+    return files;
+  }
   for (auto& p : fs::recursive_directory_iterator(base_dir)) {
     if (p.is_regular_file()) {
       if (std::regex_match(p.path().filename().string(), pattern)) {

decompiler/config/jak1/all-types.gc

@@ -13,7 +13,7 @@
    (z float :offset 64)
    (w float :offset 96))
   :method-count-assert 9
-  :size-assert         #x10
+  :size-assert          #x10
   :flag-assert         #x900000010
   )
 

decompiler/config/jak3/all-types.gc

@@ -252,7 +252,7 @@
 ;; (define-extern lognot function) ;; (function int int)
 ;; (define-extern false-func function) ;; (function symbol)
 ;; (define-extern true-func function) ;; (function symbol)
-;; (define-extern format object) ;; (function _varargs_ object)
+(define-extern format (function _varargs_ object))
 ;; (define-extern basic-type? function) ;; (function basic type symbol)
 ;; (define-extern type-type? function) ;; (function type type symbol)
 ;; (define-extern type? function) ;; (function object type symbol)
@@ -2826,12 +2826,11 @@
     (basic-reserved-method-28 () none) ;; 28
     )
   )
-|#
 
-;; (deftype part-id (uint32)
-;;   ()
-;;   :flag-assert #x900000004
-;;   )
+(deftype part-id (uint32)
+  ()
+  :flag-assert #x900000004
+  )
 
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

scripts/tasks/.env.default

@@ -1,3 +1,4 @@
 GAME=jak1
 DECOMP_CONFIG=jak1/jak1_config.jsonc
 DECOMP_CONFIG_VERSION=ntsc_v1
+TYPE_CONSISTENCY_TEST_FILTER=Jak1TypeConsistency

scripts/tasks/update-env.py

@@ -69,6 +69,12 @@
   "jak3": "ntsc_v1"
 }
 
+type_consistency_filter_map = {
+  "jak1": "Jak1TypeConsistency",
+  "jak2": "Jak2TypeConsistency",
+  "jak3": "Jak3TypeConsistency"
+}
+
 if args.game:
   if args.game not in valid_games:
     print("Unsupported game '{}'".format(args.game))
@@ -78,12 +84,14 @@
   if (curr != file["GAME"]) or file["DECOMP_CONFIG_VERSION"] not in decomp_config_version_map[file["GAME"]]:
     file["DECOMP_CONFIG"] = decomp_config_map[file["GAME"]]
     file["DECOMP_CONFIG_VERSION"] = default_config_version_map[file["GAME"]]
+    file["TYPE_CONSISTENCY_TEST_FILTER"] = type_consistency_filter_map[file["GAME"]]
 if args.decomp_config:
   if args.decomp_config not in decomp_config_version_map[file["GAME"]]:
     print("Unsupported decomp config '{}' for game '{}'".format(args.decomp_config, file["GAME"]))
     sys.exit(1)
   file["DECOMP_CONFIG"] = decomp_config_map[file["GAME"]]
   file["DECOMP_CONFIG_VERSION"] = decomp_config_version_map[file["GAME"]][args.decomp_config]
+  file["TYPE_CONSISTENCY_TEST_FILTER"] = type_consistency_filter_map[file["GAME"]]
 
 with open(env_path, 'w') as env_file:
   for item in file.items():