create machine frame in VM-exit handler to reconstruct full callstack…

… in WinDbg

src/hvpp/hvpp/vcpu.asm

@@ -94,9 +94,23 @@ INCLUDE ia32/common.inc
 ;   This method captures current CPU context and calls vcpu_t::entry_host()
 ;   method.
 ;
+;   Note that this procedure is declared with FRAME attribute.
+;
+;   This attribute causes MASM to generate a function table entry in
+;   .pdata and unwind information in .xdata for a function's structured
+;   exception handling unwind behavior.  If ehandler is present, this
+;   proc is entered in the .xdata as the language specific handler.
+;
+;   When the FRAME attribute is used, it must be followed by an .ENDPROLOG
+;   directive.
+;   (ref: https://docs.microsoft.com/cs-cz/cpp/build/raw-pseudo-operations)
+;
+;   Among exception handling, which isn't very important for us, this
+;   is especially useful for recreating callstack in WinDbg.
+;
 ;--
 
-    ?entry_host_@vcpu_t@hvpp@@CAXXZ PROC
+    ?entry_host_@vcpu_t@hvpp@@CAXXZ PROC FRAME
         push    rcx
 
 ;
@@ -122,9 +136,81 @@ INCLUDE ia32/common.inc
         lea     rcx, qword ptr [rsp + VCPU_OFFSET]
 
 ;
-; Create shadow space
+; Create dummy machine frame.
+;
+; This code is not critical for any hypervisor functionality, but
+; it'll help WinDbg to "append" callstack of the application which
+; initiated VM-exit to the callstack of the hypervisor.  In another
+; words - the callstack of the application which initiated VM-exit
+; will be shown "below" the vcpu_t::entry_host_() method.
+;
+; This is achieved by forcing WinDbg to think this is in fact
+; interrupt handler.  When interrupt occurs in 64-bit mode, the
+; CPU pushes [ SS, RSP, EFLAGS, CS, RIP ] on the stack before
+; execution of the interrupt handler (we call it "machine frame").
+; At the end of the interrupt handler, these values are restored
+; using the IRETQ instruction.
+; See Vol3A[6.14.2(64-Bit Mode Stack Frame)] for more details.
+;
+; We will manually emulate this behavior by pushing 5 values
+; (representing the registers mentioned above) on the stack
+; and then issuing directive .pushframe.  The .pushframe directive
+; doesn't emit any assembly instruction - instead, an opcode
+; UWOP_PUSH_MACHFRAME is emitted into the unwind information of
+; the executable file.  WinDbg recognizes this opcode and will
+; look at the RIP and RSP values of the machine frame to recreate
+; callstack.  Other fields than RIP and RSP appear to be ignored
+; by WinDbg (and we don't use them either).
+;
+; TL;DR:
+;   WinDbg recreates callstacks of interrupt handlers by looking
+;   at RIP and RSP fields of the machine frame.  Therefore, make
+;   WinDbg think this function is an interrupt handler by pushing
+;   fake machine frame + issuing .pushframe.
+;
+; Note1:
+;   nt!KiSystemCall64(Shadow) work in similar way.
+;
+; Note2:
+;   Correct values of RIP and RSP fields are set in the
+;   vcpu_t::entry_host() method (vcpu.cpp).
+;
+; See: https://docs.microsoft.com/en-us/cpp/build/struct-unwind-code
 ;
+        push    KGDT64_R3_DATA or RPL_MASK  ; push dummy SS selector
+        push    2                           ; push dummy RSP
+        push    3                           ; push dummy EFLAGS
+        push    KGDT64_R3_CODE or RPL_MASK  ; push dummy CS selector
+        push    5                           ; push dummy RIP
+
+        .pushframe
+
+;
+; Create shadow space.
+;
+; The .allocstack directive will tell the compiler to emit
+; UWOP_ALLOC_SMALL opcode into the unwind information of the
+; executable file.  This will help WinDbg to recognize that
+; in this stack-space there are local variables (and reserved
+; shadow space for callees, respectivelly) and that this space
+; belongs to this function.
+;
+; This is also needed for the callstack reconstruction to work
+; correctly - otherwise WinDbg might wrongly look in this space
+; to look for return addresses or stack pointers (RIP/RSP).
+;
+
         sub     rsp, SHADOW_SPACE
+        .allocstack  SHADOW_SPACE
+
+;
+; Finally, issue the .endprolog directive.
+;
+; This will signal end of prologue declarations (such as allocation
+; of the shadow space).
+;
+        .endprolog
+
         call    ?entry_host@vcpu_t@hvpp@@AEAAXXZ
 
 ;

src/hvpp/hvpp/vcpu.cpp

@@ -19,7 +19,7 @@ auto vcpu_t::initialize(vmexit_handler* handler) noexcept -> error_code_t
   //
   // Fill out initial stack with garbage.
   //
-  memset(stack_, 0xcc, sizeof(stack_));
+  memset(stack_.data, 0xcc, sizeof(stack_));
 
   //
   // Reset guest and exit context.
@@ -403,7 +403,7 @@ void vcpu_t::setup_host() noexcept
   // RIP - aka instruction pointer - points to function which will be called
   // on every VM-exit.
   //
-  host_rsp(reinterpret_cast<uint64_t>(std::end(stack_)));
+  host_rsp(reinterpret_cast<uint64_t>(std::end(stack_.data)));
   host_rip(reinterpret_cast<uint64_t>(&vcpu_t::entry_host_));
 }
 
@@ -515,7 +515,7 @@ void vcpu_t::setup_guest() noexcept
   // This isn't a problem, because both guest and host will NOT be running at
   // the same time on the same VCPU.
   //
-  guest_rsp(reinterpret_cast<uint64_t>(std::end(stack_)));
+  guest_rsp(reinterpret_cast<uint64_t>(std::end(stack_.data)));
   guest_rip(reinterpret_cast<uint64_t>(&vcpu_t::entry_guest_));
 }
 
@@ -557,6 +557,18 @@ void vcpu_t::entry_host() noexcept
     exit_context_.rip    = guest_rip();
     exit_context_.rflags = guest_rflags();
 
+    //
+    // WinDbg will show full callstack (hypervisor + interrupted application)
+    // after these two lines are executed.
+    // See vcpu.asm for more details.
+    //
+    // Note that machine_frame.rip is supposed to hold return address.
+    // exit_instruction_length() is added to the guest_rip() to create
+    // this value.
+    //
+    stack_.machine_frame.rip = exit_context_.rip + exit_instruction_length();
+    stack_.machine_frame.rsp = exit_context_.rsp;
+
     {
       handler_->handle(*this);
 

src/hvpp/hvpp/vcpu.h

@@ -15,8 +15,6 @@ using namespace ia32;
 
 class vmexit_handler;
 
-static constexpr int vcpu_stack_size = 0x8000;
-
 struct interrupt_info_t
 {
   public:
@@ -104,6 +102,47 @@ enum class vcpu_state
   terminated,
 };
 
+//
+// Definition of the stack structure.
+// See vcpu.asm for more details.
+//
+
+static constexpr int vcpu_stack_size = 0x8000;
+
+struct vcpu_stack_t
+{
+  struct machine_frame_t
+  {
+    uint64_t rip;
+    uint64_t cs;
+    uint64_t eflags;
+    uint64_t rsp;
+    uint64_t ss;
+  };
+
+  struct shadow_space_t
+  {
+    uint64_t dummy[4];
+  };
+
+  union
+  {
+    uint8_t data[vcpu_stack_size];
+
+    struct
+    {
+      uint8_t         dummy[vcpu_stack_size
+                            - sizeof(shadow_space_t)
+                            - sizeof(machine_frame_t)];
+      shadow_space_t  shadow_space;
+      machine_frame_t machine_frame;
+    };
+  };
+};
+
+static_assert(sizeof(vcpu_stack_t) == vcpu_stack_size);
+static_assert(sizeof(vcpu_stack_t::shadow_space_t) == 32);
+
 class vcpu_t
 {
   public:
@@ -333,7 +372,7 @@ class vcpu_t
     // If you reorder following three members (stack, guest context and exit
     // context), you have to edit offsets in vcpu.asm.
     //
-    uint8_t            stack_[vcpu_stack_size];
+    vcpu_stack_t       stack_;
     context_t          guest_context_;
     context_t          exit_context_;