Bug 1816953 - Rewrite TestDllInterceptor due to mingwclang failures. …

…r=cmartin,handyman

We were using a friendly high-level syntax for our jumper in
TestDllInterceptor:

  gDetouredCall(aCallee);

We were relying on compiling this to the jumper format recognized by our
hooking code:

  jmp qword ptr [rip + offset gDetouredCall]

Unfortunately depending on compiler options (in particular CFG and
-fno-omit-frame-pointer), the compiler may generate slightly different
code, thus breaking the test.

We now use a less friendly syntax which ensures that our jumper has the
proper format.

Differential Revision: https://phabricator.services.mozilla.com/D175486

toolkit/xre/dllservices/tests/AssemblyPayloads.h

@@ -144,19 +144,16 @@ alignas(uint32_t) uint8_t gDetouredCallUnwindInfo[] = {
 extern decltype(&DetouredCallCode) gDetouredCall;
 
 // This is just a jumper: our hooking code will thus detour the jump target
-// DetouredCall -- it will not detour DetouredCallJumper. We need to do this to
-// point our hooking code to the dynamic code, because our hooking API needs an
+// -- it will not detour DetouredCallJumper. We need to do this to point our
+// hooking code to the dynamic code, because our hooking API works with an
 // exported function name.
-//
-// guard(nocf) ensures that our generated code is a recognized jumper:
-//   jmp qword ptr [rip+offset DetouredCall]
-// rather than:
-//   mov rax, qword ptr [rip+offset DetouredCall]
-//   mov rdx, qword ptr [rip+offset __guard_dispatch_icall_fptr]
-//   jmp rdx
-__declspec(dllexport noinline guard(nocf)) void DetouredCallJumper(
+__attribute__((naked)) __declspec(dllexport noinline) void DetouredCallJumper(
     uintptr_t aCallee) {
-  gDetouredCall(aCallee);
+  // Ideally we would want this to be:
+  //   jmp qword ptr [rip + offset gDetouredCall]
+  // Unfortunately, it is unclear how to do that with inline assembly, so we
+  // use a zero offset and patch it before the test.
+  asm volatile("jmpq *0(%rip)");
 }
 #    endif  // !defined(MOZ_CODE_COVERAGE)
 

toolkit/xre/dllservices/tests/TestDllInterceptor.cpp

@@ -968,6 +968,32 @@ struct DetouredCallChunk {
 // associate it with unwind information.
 decltype(&DetouredCallCode) gDetouredCall =
     []() -> decltype(&DetouredCallCode) {
+  // We first adjust the detoured call jumper from:
+  //   ff 25 00 00 00 00    jmp qword ptr [rip + 0]
+  // to:
+  //   ff 25 XX XX XX XX    jmp qword ptr [rip + offset gDetouredCall]
+  uint8_t bytes[6]{0xff, 0x25, 0, 0, 0, 0};
+  if (0 != memcmp(bytes, reinterpret_cast<void*>(DetouredCallJumper),
+                  sizeof bytes)) {
+    return nullptr;
+  }
+
+  DWORD oldProtect{};
+  if (!VirtualProtect(reinterpret_cast<void*>(DetouredCallJumper), sizeof bytes,
+                      PAGE_READWRITE, &oldProtect)) {
+    return nullptr;
+  }
+
+  *reinterpret_cast<uint32_t*>(&bytes[2]) = static_cast<uint32_t>(
+      reinterpret_cast<uintptr_t>(&gDetouredCall) -
+      (reinterpret_cast<uintptr_t>(DetouredCallJumper) + sizeof bytes));
+  memcpy(reinterpret_cast<void*>(DetouredCallJumper), bytes, sizeof bytes);
+
+  if (!VirtualProtect(reinterpret_cast<void*>(DetouredCallJumper), sizeof bytes,
+                      oldProtect, &oldProtect)) {
+    return nullptr;
+  }
+
   auto detouredCallChunk = reinterpret_cast<DetouredCallChunk*>(
       VirtualAlloc(nullptr, sizeof(DetouredCallChunk), MEM_RESERVE | MEM_COMMIT,
                    PAGE_READWRITE));
@@ -988,7 +1014,6 @@ decltype(&DetouredCallCode) gDetouredCall =
          reinterpret_cast<void*>(DetouredCallCode),
          sizeof(detouredCallChunk->code));
 
-  DWORD oldProtect = 0;
   if (!VirtualProtect(reinterpret_cast<void*>(detouredCallChunk),
                       sizeof(DetouredCallChunk), PAGE_EXECUTE_READ,
                       &oldProtect)) {