Fix button prompts not appearing which fixes shrine exits, dialogues etc

This also fixes a bunch of other visibility checks that the game does! Like enemies etc.

Author: Crementif

resources/BreathOfTheWild_BetterVR/patch_FixVisibilityChecks.asm

@@ -3,15 +3,61 @@ moduleMatches = 0x6267BFD0
 
 .origin = codecave
 
+0x03191BA8 = Camera__getAltPerspectiveProjection:
+;0x03191BA0 = Camera__getPerspectiveProjection:
+
 custom_checkIfCameraCanSeePos:
-mflr r0
-stw r0, 0x04(r1)
+mflr r11
+stwu r1, -0x20(r1)
+stw r11, 0x24(r1)
+stw r3, 0x1C(r1)
+stw r5, 0x18(r1)
+stw r6, 0x14(r1)
+stw r7, 0x10(r1)
+stw r8, 0x0C(r1)
+
+mr r8, r0
+
+lis r3, Camera__getPerspectiveProjection@ha
+addi r3, r3, Camera__getPerspectiveProjection@l
+mtctr r3
+lis r3, Camera__sInstance@ha
+lwz r3, Camera__sInstance@l(r3)
+bctrl
+mr r5, r3
+
+lis r3, Camera__getAltPerspectiveProjection@ha
+addi r3, r3, Camera__getAltPerspectiveProjection@l
+mtctr r3
+lis r3, Camera__sInstance@ha
+lwz r3, Camera__sInstance@l(r3)
+bctrl
+mr r6, r3
+
+lwz r3, 0x1C(r1)
+; r5 = perspective projection
+; r6 = alt perspective projection
+lwz r7, 0x24(r1) ; load LR
+mr r0, r8 ; restore original r0
 bla import.coreinit.hook_CheckIfCameraCanSeePos
-lwz r0, 0x04(r1)
+
+lwz r8, 0x0C(r1)
+lwz r7, 0x10(r1)
+lwz r6, 0x14(r1)
+lwz r5, 0x18(r1)
+;lwz r3, 0x1C(r1) ; r3 should be the return value of CheckIfCameraCanSeePos
+lwz r0, 0x24(r1)
+addi r1, r1, 0x20
 mtlr r0
 blr
 
-;0x0318FFA8 = ba custom_checkIfCameraCanSeePos
+; fix all visibility checks to use our custom function that will do a query twice for each eye
+0x0318FFA8 = li r0, 0
+0x0318FFAC = ba custom_checkIfCameraCanSeePos
+
+0x03190118 = li r0, 1
+0x0319011C = ba custom_checkIfCameraCanSeePos
+
 
 ; always make AttCheck::ScreenRelated check return true
 ;0x035D67BC = li r3, 1

src/hooking/camera.cpp

@@ -592,11 +592,15 @@ std::pair<glm::vec3, glm::fquat> CemuHooks::CalculateVRWorldPose(const BESeadLoo
     // in-game camera
     glm::mat4x3 viewMatrix = camera.mtx.getLEMatrix();
     glm::mat4 worldGame = glm::inverse(glm::mat4(viewMatrix));
-    glm::vec3 basePos = s_wsCameraPosition;
-    glm::quat baseRot = s_wsCameraRotation;
-    auto [swing, baseYaw] = swingTwistY(baseRot);
+    glm::vec3 basePos = glm::vec3(worldGame[3]);
+    glm::quat baseRot = glm::quat_cast(worldGame);
+
+    // overwrite with our stored camera pos/rot
+    basePos = s_wsCameraPosition;
+    baseRot = s_wsCameraRotation;
 
-    if (CemuHooks::IsFirstPerson()) {
+    auto [swing, baseYaw] = swingTwistY(baseRot);
+    if (IsFirstPerson()) {
         // take link's direction, then rotate the headset position
         BEMatrix34 playerMtx = {};
         readMemory(s_playerMtxAddress, &playerMtx);
@@ -613,7 +617,6 @@ std::pair<glm::vec3, glm::fquat> CemuHooks::CalculateVRWorldPose(const BESeadLoo
         }
 
         basePos = playerPos;
-
         if (auto eventSettings = GetFirstPersonSettingsForActiveEvent()) {
             if (eventSettings->ignoreCameraRotation) {
                 glm::fquat playerRot = playerMtx.getRotLE();
@@ -638,12 +641,11 @@ std::pair<glm::vec3, glm::fquat> CemuHooks::CalculateVRWorldPose(const BESeadLoo
     return { newPos, newRot };
 }
 
-constexpr float VISIBILITY_CHECK_BUFFER = 1.5f;
 void CemuHooks::hook_CheckIfCameraCanSeePos(PPCInterpreter_t* hCPU) {
     hCPU->instructionPointer = hCPU->sprNew.LR;
 
     if (VRManager::instance().XR->GetRenderer() == nullptr) {
-        hCPU->gpr[3] = 1;
+        hCPU->gpr[3] = 0;
         return;
     }
 
@@ -656,19 +658,18 @@ void CemuHooks::hook_CheckIfCameraCanSeePos(PPCInterpreter_t* hCPU) {
     BESeadLookAtCamera camera = {};
     readMemory(camPtr, &camera);
 
-    struct {
-        uint32_t x, y, z;
-    } posRaw;
-    readMemory(posPtr, &posRaw);
+    //uint32_t mainProjectionPtr = hCPU->gpr[5];
+    //BESeadPerspectiveProjection mainProjection = {};
+    //readMemory(mainProjectionPtr, &mainProjection);
+    //uint32_t altProjectionPtr = hCPU->gpr[6];
+    //BESeadPerspectiveProjection altProjection = {};
+    //readMemory(altProjectionPtr, &altProjection);
 
-    auto swapFloat = [](uint32_t val) -> float {
-        uint32_t swapped = ((val & 0xFF) << 24) | ((val & 0xFF00) << 8) | ((val & 0xFF0000) >> 8) | ((val & 0xFF000000) >> 24);
-        float f;
-        memcpy(&f, &swapped, 4);
-        return f;
-    };
+    //Log::print<INFO>("Main Projection{}: {}", hCPU->gpr[0] == 0 ? " yes " : " no ", mainProjection);
+    //Log::print<INFO>("Alt Projection{}: {}", hCPU->gpr[0] == 1 ? " yes " : " no ", altProjection);
 
-    glm::vec3 center(swapFloat(posRaw.x), swapFloat(posRaw.y), swapFloat(posRaw.z));
+    BEVec3 center;
+    readMemory(posPtr, &center);
 
     Frustum frustum;
     bool visible = false;
@@ -677,19 +678,23 @@ void CemuHooks::hook_CheckIfCameraCanSeePos(PPCInterpreter_t* hCPU) {
         OpenXR::EyeSide side = (i == 0) ? OpenXR::EyeSide::LEFT : OpenXR::EyeSide::RIGHT;
         if (auto fovOpt = VRManager::instance().XR->GetRenderer()->GetFOV(side)) {
             auto [pos, rot] = CalculateVRWorldPose(camera, side);
+
+            // pull the camera backwards a bit to account for it being a third-person game that encompassed a bigger area
+            pos += rot * glm::vec3(0.0f, 0.0f, 1.0f);
+
             glm::mat4 view = glm::inverse(glm::translate(glm::mat4(1.0f), pos) * glm::mat4_cast(rot));
-            glm::mat4 proj = calculateProjectionMatrix(nearClip, farClip, fovOpt.value());
+            glm::mat4 proj = glm::transpose(calculateProjectionMatrix(nearClip, farClip, fovOpt.value()));
             glm::mat4 vp = proj * view;
 
             frustum.update(vp);
-            if (frustum.checkSphere(center, radius+VISIBILITY_CHECK_BUFFER)) {
+            if (frustum.checkSphere(center.getLE(), radius)) {
                 visible = true;
                 break;
             }
         }
     }
 
-    Log::print<INFO>("Checking visibility of {} (rad = {}, near = {}, far = {}): {}", center, radius, nearClip, farClip, visible ? "visible" : "invisible");
+    Log::print<PPC>("Checking visibility of {} (rad = {}, near = {}, far = {}): {}", center, radius, nearClip, farClip, visible ? "visible" : "invisible");
 
     hCPU->gpr[3] = visible ? 1 : 0;
 }