Add support for unwinding during prologue/epilogue

(cherry picked from commit 5393efb)
(cherry picked from commit befad70)
(cherry picked from commit 463ddee)

Author: Keno

libunwind/src/CompactUnwinder.hpp

@@ -311,6 +311,50 @@ int CompactUnwinder_x86_64<A>::stepWithCompactEncodingRBPFrame(
   uint32_t savedRegistersLocations =
       EXTRACT_BITS(compactEncoding, UNWIND_X86_64_RBP_FRAME_REGISTERS);
 
+  // If we have not stored EBP yet
+  if (functionStart == registers.getIP()) {
+    uint64_t rsp = registers.getSP();
+    // old esp is ebp less return address
+    registers.setSP(rsp+8);
+    // pop return address into eip
+    registers.setIP(addressSpace.get64(rsp));
+
+    return UNW_STEP_SUCCESS;
+  } else if (functionStart + 1 == registers.getIP()) {
+    uint64_t rsp = registers.getSP();
+    // old esp is ebp less return address
+    registers.setSP(rsp + 16);
+    // pop return address into eip
+    registers.setIP(addressSpace.get64(rsp + 8));
+
+    return UNW_STEP_SUCCESS;
+  }
+
+  // If we're about to return, we've already popped the base pointer
+  uint8_t b = addressSpace.get8(registers.getIP());
+
+  // This is a hack to detect VZEROUPPER but in between popq rbp and ret
+  // It's not pretty but it works
+  if (b == 0xC5) {
+    if ((b = addressSpace.get8(registers.getIP() + 1)) == 0xF8 &&
+        (b = addressSpace.get8(registers.getIP() + 2)) == 0x77)
+      b = addressSpace.get8(registers.getIP() + 3);
+    else
+      goto skip_ret;
+  }
+
+  if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) {
+    uint64_t rbp = registers.getSP();
+    // old esp is ebp less return address
+    registers.setSP(rbp + 16);
+    // pop return address into eip
+    registers.setIP(addressSpace.get64(rbp + 8));
+
+    return UNW_STEP_SUCCESS;
+  }
+
+  skip_ret:
+
   uint64_t savedRegisters = registers.getRBP() - 8 * savedRegistersOffset;
   for (int i = 0; i < 5; ++i) {
     switch (savedRegistersLocations & 0x7) {
@@ -431,6 +475,118 @@ int CompactUnwinder_x86_64<A>::stepWithCompactEncodingFrameless(
       }
     }
   }
+
+  // Note that the order of these registers is so that
+  // registersSaved[0] is the one that will be pushed onto the stack last.
+  // Thus, if we want to walk this from the top, we need to go in reverse.
+  assert(regCount <= 6);
+
+  // check whether we are still in the prologue
+  uint64_t curAddr = functionStart;
+  if (regCount > 0) {
+    for (int8_t i = (int8_t)(regCount) - 1; i >= 0; --i) {
+      if (registers.getIP() == curAddr) {
+        // None of the registers have been modified yet, so we don't need to reload them
+        framelessUnwind(addressSpace, registers.getSP() + 8 * (regCount - (uint64_t)(i + 1)), registers);
+        return UNW_STEP_SUCCESS;
+      } else {
+        assert(curAddr < registers.getIP());
+      }
+
+
+      // pushq %rbp and pushq %rbx is 1 byte. Everything else 2
+      if ((UNWIND_X86_64_REG_RBP == registersSaved[i]) ||
+          (UNWIND_X86_64_REG_RBX == registersSaved[i]))
+        curAddr += 1;
+      else
+        curAddr += 2;
+    }
+  }
+  if (registers.getIP() == curAddr) {
+    // None of the registers have been modified yet, so we don't need to reload them
+    framelessUnwind(addressSpace, registers.getSP() + 8*regCount, registers);
+    return UNW_STEP_SUCCESS;
+  } else {
+    assert(curAddr < registers.getIP());
+  }
+
+
+  // And now for the epilogue
+  {
+    uint8_t  i  = 0;
+    uint64_t p  = registers.getIP();
+    uint8_t  b  = 0;
+
+    while (true) {
+      b = addressSpace.get8(p++);
+      // This is a hack to detect VZEROUPPER but in between the popq's and ret
+      // It's not pretty but it works
+      if (b == 0xC5) {
+        if ((b = addressSpace.get8(p++)) == 0xF8 && (b = addressSpace.get8(p++)) == 0x77)
+          b = addressSpace.get8(p++);
+        else
+          break;
+      }
+      //  popq %rbx    popq %rbp
+      if (b == 0x5B || b == 0x5D) {
+        i++;
+      } else if (b == 0x41) {
+        b = addressSpace.get8(p++);
+        if (b == 0x5C || b == 0x5D || b == 0x5E || b == 0x5F)
+          i++;
+        else
+          break;
+      } else if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) {
+        // i pop's haven't happened yet
+        uint64_t savedRegisters = registers.getSP() + 8 * i;
+        if (regCount > 0) {
+          for (int8_t j = (int8_t)(regCount) - 1; j >= (int8_t)(regCount) - i; --j) {
+            uint64_t addr = savedRegisters - 8 * (regCount - (uint64_t)(j));
+            switch (registersSaved[j]) {
+              case UNWIND_X86_64_REG_RBX:
+                registers.setRBX(addressSpace.get64(addr));
+                break;
+              case UNWIND_X86_64_REG_R12:
+                registers.setR12(addressSpace.get64(addr));
+                break;
+              case UNWIND_X86_64_REG_R13:
+                registers.setR13(addressSpace.get64(addr));
+                break;
+              case UNWIND_X86_64_REG_R14:
+                registers.setR14(addressSpace.get64(addr));
+                break;
+              case UNWIND_X86_64_REG_R15:
+                registers.setR15(addressSpace.get64(addr));
+                break;
+              case UNWIND_X86_64_REG_RBP:
+                registers.setRBP(addressSpace.get64(addr));
+                break;
+              default:
+                _LIBUNWIND_DEBUG_LOG("bad register for frameless, encoding=%08X for "
+                             "function starting at 0x%llX",
+                              encoding, functionStart);
+                _LIBUNWIND_ABORT("invalid compact unwind encoding");
+            }
+          }
+        }
+        framelessUnwind(addressSpace, savedRegisters, registers);
+        return UNW_STEP_SUCCESS;
+      } else {
+        break;
+      }
+    }
+  }
+
+  /*
+   0x10fe2733a:  5b                             popq   %rbx
+   0x10fe2733b:  41 5c                          popq   %r12
+   0x10fe2733d:  41 5d                          popq   %r13
+   0x10fe2733f:  41 5e                          popq   %r14
+   0x10fe27341:  41 5f                          popq   %r15
+   0x10fe27343:  5d                             popq   %rbp
+   */
+
+
   uint64_t savedRegisters = registers.getSP() + stackSize - 8 - 8 * regCount;
   for (uint32_t i = 0; i < regCount; ++i) {
     switch (registersSaved[i]) {