gdb: refactor amd64_analyze_prologue

Refactor amd64_analyze_prologue so it clearly reflects what is the order
of operations in the prologue that we expect to encounter, as is the
case for i386's implementation.

Approved-By: Andrew Burgess <[email protected]>

Author: pawel-kupczak

gdb/amd64-tdep.c

@@ -2129,6 +2129,30 @@ amd64_alloc_frame_cache (void)
   return cache;
 }
 
+/* Check whether PC is at "endbr64" instruction.  If so, return PC past it.
+   Otherwise, return PC passed to this function.  */
+
+static CORE_ADDR
+amd64_skip_endbr (gdbarch *gdbarch, CORE_ADDR pc)
+{
+  static const gdb_byte endbr64[4] = { 0xf3, 0x0f, 0x1e, 0xfa };
+
+  bfd_endian byte_order = gdbarch_byte_order (gdbarch);
+  gdb_byte buf[3];
+  gdb_byte op = read_code_unsigned_integer (pc, 1, byte_order);
+
+  /* Check for the `endbr64` instruction, skip it if found.  */
+  if (op == endbr64[0])
+    {
+      read_code (pc + 1, buf, 3);
+
+      if (memcmp (buf, &endbr64[1], 3) == 0)
+	return pc + 4;
+    }
+
+  return pc;
+}
+
 /* GCC 4.4 and later, can put code in the prologue to realign the
    stack pointer.  Check whether PC points to such code, and update
    CACHE accordingly.  Return the first instruction after the code
@@ -2466,70 +2490,30 @@ amd64_x32_analyze_stack_align (CORE_ADDR pc, CORE_ADDR current_pc,
   return std::min (pc + offset + 2, current_pc);
 }
 
-/* Do a limited analysis of the prologue at PC and update CACHE
-   accordingly.  Bail out early if CURRENT_PC is reached.  Return the
-   address where the analysis stopped.
-
-   We will handle only functions beginning with:
-
-      pushq %rbp        0x55
-      movq %rsp, %rbp   0x48 0x89 0xe5 (or 0x48 0x8b 0xec)
-
-   or (for the X32 ABI):
-
-      pushq %rbp        0x55
-      movl %esp, %ebp   0x89 0xe5 (or 0x8b 0xec)
-
-   The `endbr64` instruction can be found before these sequences, and will be
-   skipped if found.
+/* Analyze frame setup instructions at PC on behalf of amd64_analyze_prologue
+   and update CACHE accordingly.  Bail out early if CURRENT_PC is reached.
+   Return the address where the analysis stopped.
 
-   Any function that doesn't start with one of these sequences will be
-   assumed to have no prologue and thus no valid frame pointer in
-   %rbp.  */
+   See comment on amd64_analyze_prologue for the sequences handled.  The
+   movq/movl after the push of %rbp is considered optional.  'endbr64' is
+   handled before this function.  */
 
 static CORE_ADDR
-amd64_analyze_prologue (struct gdbarch *gdbarch,
-			CORE_ADDR pc, CORE_ADDR current_pc,
-			struct amd64_frame_cache *cache)
+amd64_analyze_frame_setup (gdbarch *gdbarch, CORE_ADDR pc,
+			   CORE_ADDR current_pc, amd64_frame_cache *cache)
 {
-  enum bfd_endian byte_order = gdbarch_byte_order (gdbarch);
-  /* The `endbr64` instruction.  */
-  static const gdb_byte endbr64[4] = { 0xf3, 0x0f, 0x1e, 0xfa };
   /* There are two variations of movq %rsp, %rbp.  */
   static const gdb_byte mov_rsp_rbp_1[3] = { 0x48, 0x89, 0xe5 };
   static const gdb_byte mov_rsp_rbp_2[3] = { 0x48, 0x8b, 0xec };
   /* Ditto for movl %esp, %ebp.  */
   static const gdb_byte mov_esp_ebp_1[2] = { 0x89, 0xe5 };
   static const gdb_byte mov_esp_ebp_2[2] = { 0x8b, 0xec };
 
+  bfd_endian byte_order = gdbarch_byte_order (gdbarch);
   gdb_byte buf[3];
-  gdb_byte op;
-
-  if (current_pc <= pc)
-    return current_pc;
-
-  if (gdbarch_ptr_bit (gdbarch) == 32)
-    pc = amd64_x32_analyze_stack_align (pc, current_pc, cache);
-  else
-    pc = amd64_analyze_stack_align (pc, current_pc, cache);
-
-  op = read_code_unsigned_integer (pc, 1, byte_order);
+  gdb_byte op = read_code_unsigned_integer (pc, 1, byte_order);
 
-  /* Check for the `endbr64` instruction, skip it if found.  */
-  if (op == endbr64[0])
-    {
-      read_code (pc + 1, buf, 3);
-
-      if (memcmp (buf, &endbr64[1], 3) == 0)
-	pc += 4;
-
-      op = read_code_unsigned_integer (pc, 1, byte_order);
-    }
-
-  if (current_pc <= pc)
-    return current_pc;
-
-  if (op == 0x55)		/* pushq %rbp */
+  if (op == 0x55) /* pushq %rbp.  */
     {
       /* Take into account that we've executed the `pushq %rbp' that
 	 starts this instruction sequence.  */
@@ -2569,6 +2553,46 @@ amd64_analyze_prologue (struct gdbarch *gdbarch,
   return pc;
 }
 
+/* Do a limited analysis of the prologue at PC and update CACHE
+   accordingly.  Bail out early if CURRENT_PC is reached.  Return the
+   address where the analysis stopped.
+
+   We will handle only functions beginning with:
+
+      pushq %rbp	0x55
+      movq %rsp, %rbp   0x48 0x89 0xe5 (or 0x48 0x8b 0xec)
+
+   or (for the X32 ABI):
+
+      pushq %rbp	0x55
+      movl %esp, %ebp   0x89 0xe5 (or 0x8b 0xec)
+
+   The `endbr64` instruction can be found before these sequences, and will be
+   skipped if found.
+
+   Any function that doesn't start with one of these sequences will be
+   assumed to have no prologue and thus no valid frame pointer in
+   %rbp.  */
+
+static CORE_ADDR
+amd64_analyze_prologue (gdbarch *gdbarch, CORE_ADDR pc, CORE_ADDR current_pc,
+			amd64_frame_cache *cache)
+{
+  if (current_pc <= pc)
+    return current_pc;
+
+  if (gdbarch_ptr_bit (gdbarch) == 32)
+    pc = amd64_x32_analyze_stack_align (pc, current_pc, cache);
+  else
+    pc = amd64_analyze_stack_align (pc, current_pc, cache);
+
+  pc = amd64_skip_endbr (gdbarch, pc);
+  if (current_pc <= pc)
+    return current_pc;
+
+  return amd64_analyze_frame_setup (gdbarch, pc, current_pc, cache);
+}
+
 /* Work around false termination of prologue - GCC PR debug/48827.
 
    START_PC is the first instruction of a function, PC is its minimal already