Fix button prompts not appearing for shrines, terminals etc.

So, the dungeon exits (but also many other button prompts, like the dungeon entrances, and interactables) use three AttClient checks to see whether the player should be shown the option to interact with it:
- Screen check (uses the camera to check whether the interactor (or AttClient) is visible)
- AreaCylinderFan check (boundary box)
- Line (of sight) check (using a raycast)

However, the Screen check was basically broken due to the VR rendering using a different aspect ratio but also due to having two frustums instead of one. I added a custom visibility query check (which the screen check uses underneath), but it broke enemies (presumably), so for now I just hardcoded a workaround that just patches the Screen checks to always return visible.

Author: Crementif

include/game_structs.h

@@ -524,6 +524,57 @@ struct ActCamera : ActorWiiU {
 static_assert(offsetof(ActCamera, origCamMtx) == 0x550, "ActCamera.origCamMtx offset mismatch");
 static_assert(offsetof(ActCamera, finalCamMtx) == 0x5C0, "ActCamera.finalCamMtx offset mismatch");
 
+struct BESeadCamera {
+    BEMatrix34 mtx;
+    BEType<uint32_t> __vftable;
+};
+struct BESeadLookAtCamera : BESeadCamera {
+    BEVec3 pos;
+    BEVec3 at;
+    BEVec3 up;
+
+    bool operator==(const BESeadLookAtCamera& other) const {
+        return pos == other.pos && at == other.at && up == other.up;
+    }
+};
+static_assert(sizeof(BESeadCamera) == 0x34, "BESeadCamera size mismatch");
+static_assert(sizeof(BESeadLookAtCamera) == 0x58, "BESeadLookAtCamera size mismatch");
+
+// not identical memory layout wise
+struct Frustum {
+    glm::vec4 planes[6];
+
+    void update(const glm::mat4& vp) {
+        // Left
+        planes[0] = glm::vec4(vp[0][3] + vp[0][0], vp[1][3] + vp[1][0], vp[2][3] + vp[2][0], vp[3][3] + vp[3][0]);
+        // Right
+        planes[1] = glm::vec4(vp[0][3] - vp[0][0], vp[1][3] - vp[1][0], vp[2][3] - vp[2][0], vp[3][3] - vp[3][0]);
+        // Bottom
+        planes[2] = glm::vec4(vp[0][3] + vp[0][1], vp[1][3] + vp[1][1], vp[2][3] + vp[2][1], vp[3][3] + vp[3][1]);
+        // Top
+        planes[3] = glm::vec4(vp[0][3] - vp[0][1], vp[1][3] - vp[1][1], vp[2][3] - vp[2][1], vp[3][3] - vp[3][1]);
+        // Near
+        planes[4] = glm::vec4(vp[0][3] + vp[0][2], vp[1][3] + vp[1][2], vp[2][3] + vp[2][2], vp[3][3] + vp[3][2]);
+        // Far
+        planes[5] = glm::vec4(vp[0][3] - vp[0][2], vp[1][3] - vp[1][2], vp[2][3] - vp[2][2], vp[3][3] - vp[3][2]);
+
+        for (int i = 0; i < 6; ++i) {
+            float len = glm::length(glm::vec3(planes[i]));
+            planes[i] /= len;
+        }
+    }
+
+    bool checkSphere(const glm::vec3& center, float radius) const {
+        for (int i = 0; i < 6; ++i) {
+            float dist = glm::dot(glm::vec3(planes[i]), center) + planes[i].w;
+            if (dist < -radius) {
+                return false;
+            }
+        }
+        return true;
+    }
+};
+
 #pragma pack(pop)
 
 inline std::string contactLayerNames[] = {

include/pch.h

@@ -550,23 +550,6 @@ struct BESeadPerspectiveProjection : BESeadProjection {
 static_assert(sizeof(BESeadProjection) == 0x94, "BESeadProjection size mismatch");
 static_assert(sizeof(BESeadPerspectiveProjection) == 0xB8, "BESeadPerspectiveProjection size mismatch");
 
-#pragma pack(push, 1)
-struct BESeadCamera {
-    BEMatrix34 mtx;
-    BEType<uint32_t> __vftable;
-};
-struct BESeadLookAtCamera : BESeadCamera {
-    BEVec3 pos;
-    BEVec3 at;
-    BEVec3 up;
-
-    bool operator==(const BESeadLookAtCamera& other) const {
-        return pos == other.pos && at == other.at && up == other.up;
-    }
-};
-#pragma pack(pop)
-static_assert(sizeof(BESeadCamera) == 0x34, "BESeadCamera size mismatch");
-static_assert(sizeof(BESeadLookAtCamera) == 0x58, "BESeadLookAtCamera size mismatch");
 struct data_VRProjectionMatrixOut {
     BEType<float> aspectRatio;
     BEType<float> fovY;

resources/BreathOfTheWild_BetterVR/patch_FixVisibilityChecks.asm

@@ -0,0 +1,17 @@
+[BetterVR_FirstPersonMode_V208]
+moduleMatches = 0x6267BFD0
+
+.origin = codecave
+
+custom_checkIfCameraCanSeePos:
+mflr r0
+stw r0, 4(r1)
+bla import.coreinit.hook_CheckIfCameraCanSeePos
+lwz r0, 4(r1)
+mtlr r0
+blr
+
+;0x0318FFA8 = ba custom_checkIfCameraCanSeePos
+
+; always make AttCheck::ScreenRelated check return true
+0x035D67BC = li r3, 1

src/hooking/camera.cpp

@@ -538,6 +538,112 @@ void CemuHooks::hook_ModifyProjectionUsingCamera(PPCInterpreter_t* hCPU) {
     writeMemory(projectionPtr, &perspectiveProjection);
 }
 
+std::pair<glm::vec3, glm::fquat> CemuHooks::CalculateVRWorldPose(const BESeadLookAtCamera& camera, uint8_t side) {
+    // in-game camera
+    glm::mat4x3 viewMatrix = camera.mtx.getLEMatrix();
+    glm::mat4 worldGame = glm::inverse(glm::mat4(viewMatrix));
+    glm::vec3 basePos = s_wsCameraPosition;
+    glm::quat baseRot = s_wsCameraRotation;
+    auto [swing, baseYaw] = swingTwistY(baseRot);
+
+    if (CemuHooks::IsFirstPerson()) {
+        // take link's direction, then rotate the headset position
+        BEMatrix34 playerMtx = {};
+        readMemory(s_playerMtxAddress, &playerMtx);
+        glm::fvec3 playerPos = playerMtx.getPos().getLE();
+
+        if (s_isRiding) {
+            playerPos.y -= hardcodedRidingOffset;
+        }
+        else if (s_isSwimming) {
+            playerPos.y += hardcodedSwimOffset;
+        }
+        else {
+            playerPos.y += GetSettings().playerHeightSetting.getLE();
+        }
+
+        basePos = playerPos;
+
+        if (auto settings = GetFirstPersonSettingsForActiveEvent()) {
+            if (settings->ignoreCameraRotation) {
+                glm::fquat playerRot = playerMtx.getRotLE();
+                auto [swing, yaw] = swingTwistY(playerRot);
+                baseYaw = yaw * glm::angleAxis(glm::radians(180.0f), glm::fvec3(0.0f, 1.0f, 0.0f));
+            }
+        }
+    }
+
+    // vr camera
+    std::optional<XrPosef> currPoseOpt = VRManager::instance().XR->GetRenderer()->GetPose((OpenXR::EyeSide)side);
+    if (!currPoseOpt.has_value()) {
+        return { basePos, baseRot };
+    }
+
+    glm::fvec3 eyePos = ToGLM(currPoseOpt.value().position);
+    glm::fquat eyeRot = ToGLM(currPoseOpt.value().orientation);
+
+    glm::vec3 newPos = basePos + (baseYaw * eyePos);
+    glm::fquat newRot = baseYaw * eyeRot;
+
+    return { newPos, newRot };
+}
+
+constexpr float VISIBILITY_CHECK_BUFFER = 1.5f;
+void CemuHooks::hook_CheckIfCameraCanSeePos(PPCInterpreter_t* hCPU) {
+    hCPU->instructionPointer = hCPU->sprNew.LR;
+
+    if (VRManager::instance().XR->GetRenderer() == nullptr) {
+        hCPU->gpr[3] = 1;
+        return;
+    }
+
+    uint32_t camPtr = hCPU->gpr[3];
+    uint32_t posPtr = hCPU->gpr[4];
+    float radius = hCPU->fpr[1].fp0;
+    float nearClip = hCPU->fpr[2].fp0;
+    float farClip = hCPU->fpr[3].fp0;
+
+    BESeadLookAtCamera camera = {};
+    readMemory(camPtr, &camera);
+
+    struct {
+        uint32_t x, y, z;
+    } posRaw;
+    readMemory(posPtr, &posRaw);
+
+    auto swapFloat = [](uint32_t val) -> float {
+        uint32_t swapped = ((val & 0xFF) << 24) | ((val & 0xFF00) << 8) | ((val & 0xFF0000) >> 8) | ((val & 0xFF000000) >> 24);
+        float f;
+        memcpy(&f, &swapped, 4);
+        return f;
+    };
+
+    glm::vec3 center(swapFloat(posRaw.x), swapFloat(posRaw.y), swapFloat(posRaw.z));
+
+    Frustum frustum;
+    bool visible = false;
+
+    for (int i = 0; i < 2; ++i) {
+        OpenXR::EyeSide side = (i == 0) ? OpenXR::EyeSide::LEFT : OpenXR::EyeSide::RIGHT;
+        if (auto fovOpt = VRManager::instance().XR->GetRenderer()->GetFOV(side)) {
+            auto [pos, rot] = CalculateVRWorldPose(camera, side);
+            glm::mat4 view = glm::inverse(glm::translate(glm::mat4(1.0f), pos) * glm::mat4_cast(rot));
+            glm::mat4 proj = calculateProjectionMatrix(nearClip, farClip, fovOpt.value());
+            glm::mat4 vp = proj * view;
+
+            frustum.update(vp);
+            if (frustum.checkSphere(center, radius+VISIBILITY_CHECK_BUFFER)) {
+                visible = true;
+                break;
+            }
+        }
+    }
+
+    Log::print<INFO>("Checking visibility of {} (rad = {}, near = {}, far = {}): {}", center, radius, nearClip, farClip, visible ? "visible" : "invisible");
+
+    hCPU->gpr[3] = visible ? 1 : 0;
+}
+
 void CemuHooks::hook_EndCameraSide(PPCInterpreter_t* hCPU) {
     hCPU->instructionPointer = hCPU->sprNew.LR;
 

src/hooking/cemu_hooks.h

@@ -32,6 +32,7 @@ class CemuHooks {
         osLib_registerHLEFunction("coreinit", "hook_ModifyLightPrePassProjectionMatrix", &hook_ModifyLightPrePassProjectionMatrix);
         osLib_registerHLEFunction("coreinit", "hook_OverwriteSeadPerspectiveProjectionSet", &hook_OverwriteSeadPerspectiveProjectionSet);
         osLib_registerHLEFunction("coreinit", "hook_ModifyProjectionUsingCamera", &hook_ModifyProjectionUsingCamera);
+        osLib_registerHLEFunction("coreinit", "hook_CheckIfCameraCanSeePos", &hook_CheckIfCameraCanSeePos);
         osLib_registerHLEFunction("coreinit", "hook_UpdateCameraForGameplay", &hook_UpdateCameraForGameplay);
         osLib_registerHLEFunction("coreinit", "hook_GetRenderCamera", &hook_GetRenderCamera);
         osLib_registerHLEFunction("coreinit", "hook_GetRenderProjection", &hook_GetRenderProjection);
@@ -197,6 +198,8 @@ class CemuHooks {
 
     static bool IsScreenOpen(ScreenId screen);
     static void InitWindowHandles();
+    static std::pair<glm::vec3, glm::fquat> CalculateVRWorldPose(const BESeadLookAtCamera& camera, uint8_t side);
+
     static void hook_UpdateSettings(PPCInterpreter_t* hCPU);
 
     // Actor Hooks
@@ -207,6 +210,7 @@ class CemuHooks {
     static void hook_BeginCameraSide(PPCInterpreter_t* hCPU);
     static void hook_ModifyLightPrePassProjectionMatrix(PPCInterpreter_t* hCPU);
     static void hook_ModifyProjectionUsingCamera(PPCInterpreter_t* hCPU);
+    static void hook_CheckIfCameraCanSeePos(PPCInterpreter_t* hCPU);
     static void hook_OverwriteSeadPerspectiveProjectionSet(PPCInterpreter_t* hCPU);
     static void hook_UpdateCameraForGameplay(PPCInterpreter_t* hCPU);
     static void hook_GetRenderCamera(PPCInterpreter_t* hCPU);