rewriter: emit ia2_type_registry_* calls and turn on type_confusion tests

`WITHOUT_SANDBOX` is added in `CMakeLists.txt`,
as the tracer bug that causes tests to hang indefinitely is fixed in #488,
but #488 is now moved out of this PR stack, as it needs more work.

Author: kkysen

tests/type_confusion/CMakeLists.txt

@@ -10,6 +10,7 @@ define_test(
     PKEY 1
     NEEDS_LD_WRAP
     CRITERION_TEST
+    WITHOUT_SANDBOX # TODO remove once tracer bug (#488) is fixed
 )
 
 # Build the wrapper lib

tests/type_confusion/main.c

@@ -25,36 +25,28 @@ Test(type_confusion, normal) {
   dav1d_close(&c);
 }
 
-Test(type_confusion, uninitialized,
-     //  .signal = SIGABRT, // TODO turn on once type confusion checking is working
-) {
+Test(type_confusion, uninitialized, .signal = SIGABRT) {
   dav1d_open(&c, &settings);
   // Try to use another `Dav1dContext` that hasn't been constructed/opened yet.
   dav1d_get_picture(&c2, &pic);
   dav1d_close(&c);
 }
 
-Test(type_confusion, wrong_type,
-     //  .signal = SIGABRT, // TODO turn on once type confusion checking is working
-) {
+Test(type_confusion, wrong_type, .signal = SIGABRT) {
   dav1d_open(&c, &settings);
   // Try to use another type (`Dav1dPicture`) instead.
   dav1d_get_picture((Dav1dContext *)&pic, &pic);
   dav1d_close(&c);
 }
 
-Test(type_confusion, null,
-     //  .signal = SIGABRT, // TODO turn on once type confusion checking is working
-) {
+Test(type_confusion, null, .signal = SIGABRT) {
   dav1d_open(&c, &settings);
   // Try to `NULL`.
   dav1d_get_picture(NULL, &pic);
   dav1d_close(&c);
 }
 
-Test(type_confusion, use_after_free,
-     //  .signal = SIGABRT, // TODO turn on once type confusion checking is working
-) {
+Test(type_confusion, use_after_free, .signal = SIGABRT) {
   dav1d_open(&c, &settings);
   dav1d_close(&c);
   // Try to use an already destructed `Dav1dContext`.

tools/rewriter/GenCallAsm.cpp

@@ -1147,6 +1147,60 @@ std::string emit_asm_wrapper(
 
   emit_prologue(aw, caller_pkey, target_pkey, arch);
 
+  // Check types that have (both) {con,de}structors.
+  // Check types in the type registry.
+  // If the function is itself a constructor or destructor,
+  // call `ia2_type_registry_{con,de}struct`.
+  // Otherwise, go through all of the args,
+  // and if they have (both) {con,de}structors
+  // and are one of the 6 x86 register args,
+  // call `ia2_type_registry_check`.
+  // TODO make x86 only
+  std::string_view ia2_type_registry_fn_kind;
+  std::string ia2_type_registry_fn_name;
+  std::vector<size_t> arg_indices_to_check;
+  if (target_name) {
+    if (ctx.constructors.find(*target_name) != ctx.constructors.end()) {
+      ia2_type_registry_fn_kind = "construct";
+      arg_indices_to_check.emplace_back(0);
+    } else if (ctx.destructors.find(*target_name) != ctx.destructors.end()) {
+      ia2_type_registry_fn_kind = "destruct";
+      arg_indices_to_check.emplace_back(0);
+    } else {
+      ia2_type_registry_fn_kind = "check";
+      for (auto i = 0; i < sig.api.args.size(); i++) {
+        const auto &arg = sig.api.args[i];
+        const auto &type = ctx.types.get(arg.type);
+        if (!type.has_structors()) {
+          continue;
+        }
+        if (i >= x86_int_param_reg_order.size()) {
+          llvm::errs() << "skipping checking type " << type.canonical_name << " type ID " << type.id << " for arg #" << i << " of function " << *target_name << "\n";
+          continue;
+        }
+        arg_indices_to_check.emplace_back(i);
+      }
+    }
+    ia2_type_registry_fn_name = llvm::formatv("ia2_type_registry_{0}", ia2_type_registry_fn_kind);
+  }
+
+  if (!arg_indices_to_check.empty()) {
+    emit_save_args(aw, arch);
+    emit_align_stack(aw, arch);
+    for (const auto i : arg_indices_to_check) {
+      const auto &arg = sig.api.args[i];
+      const auto &type = ctx.types.get(arg.type);
+      add_comment_line(aw, llvm::formatv("{0}ing type {1}, type ID {2}, arg #{3}", ia2_type_registry_fn_kind, type.canonical_name, type.id, i));
+      // `ia2_type_registry_*` arg 0 is a ptr of type `T*` from arg i.
+      add_asm_line(aw, llvm::formatv("movq {0}(%rsp), %{1}", 8 + 8 * (x86_int_param_reg_order.size() - 1 - i), x86_int_param_reg_order[0]));
+      // `ia2_type_registry_*` arg 1 is the `TypeId`.
+      add_asm_line(aw, llvm::formatv("movq ${0}, %{1}", type.id, x86_int_param_reg_order[1]));
+      emit_direct_call(aw, arch, ia2_type_registry_fn_name);
+    }
+    emit_revert_align_stack(aw, arch);
+    emit_restore_args(aw, arch, /* pop */ true);
+  }
+
   // Call the pre-condition functions for this target function.
   // The calls happens in the caller's compartment.
   // If there are any post-condition functions, save the args for that, too.