Merge "ART: Add instrumentation stubs for ARM64 and X86-64"

7 files changed, 240 insertions, 90 deletions

diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S
index 6057289..9a877f6 100644
--- a/runtime/arch/arm64/quick_entrypoints_arm64.S
+++ b/runtime/arch/arm64/quick_entrypoints_arm64.S
@@ -196,6 +196,11 @@
     .cfi_adjust_cfa_offset -176
 .endm
 
+.macro POP_REF_ONLY_CALLEE_SAVE_FRAME
+    add sp, sp, #176
+    .cfi_adjust_cfa_offset -176
+.endm
+
 .macro RESTORE_REF_ONLY_CALLEE_SAVE_FRAME_AND_RETURN
     RESTORE_REF_ONLY_CALLEE_SAVE_FRAME
     ret
@@ -600,12 +605,12 @@ SAVE_SIZE_AND_METHOD=SAVE_SIZE+STACK_REFERENCE_SIZE
     str x0, [x4]
 
 .Lexit_art_quick_invoke_stub\@:
-    ldp x2, x19, [x29, #32]   // Restore stack pointer and x19.
+    ldp x2, x19, [xFP, #32]   // Restore stack pointer and x19.
     .cfi_restore x19
     mov sp, x2
     .cfi_restore sp
 
-    ldp x29, x30, [x29]    // Restore old frame pointer and link register.
+    ldp xFP, xLR, [xFP]    // Restore old frame pointer and link register.
     .cfi_restore x29
     .cfi_restore x30
 
@@ -1577,9 +1582,74 @@ ENTRY art_quick_to_interpreter_bridge
     RETURN_OR_DELIVER_PENDING_EXCEPTION
 END art_quick_to_interpreter_bridge
 
-UNIMPLEMENTED art_quick_instrumentation_entry
-UNIMPLEMENTED art_quick_instrumentation_exit
-UNIMPLEMENTED art_quick_deoptimize
+
+//
+// Instrumentation-related stubs
+//
+    .extern artInstrumentationMethodEntryFromCode
+ENTRY art_quick_instrumentation_entry
+    SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME
+
+    mov   x19, x0             // Preserve method reference in a callee-save.
+
+    mov   x2, xSELF
+    mov   x3, sp
+    mov   x4, xLR
+    bl    artInstrumentationMethodEntryFromCode  // (Method*, Object*, Thread*, SP, LR)
+
+    mov   x9, x0              // x0 = result of call.
+    mov   x0, x19             // Reload method reference.
+
+    RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME  // Note: will restore xSELF
+    adr   xLR, art_quick_instrumentation_exit
+    br    x9                 // Tail-call method with lr set to art_quick_instrumentation_exit.
+END art_quick_instrumentation_entry
+
+    .extern artInstrumentationMethodExitFromCode
+ENTRY art_quick_instrumentation_exit
+    mov   xLR, #0             // Clobber LR for later checks.
+
+    SETUP_REF_ONLY_CALLEE_SAVE_FRAME
+
+    // We need to save x0 and d0. We could use a callee-save from SETUP_REF_ONLY, but then
+    // we would need to fully restore it. As there are a lot of callee-save registers, it seems
+    // easier to have an extra small stack area.
+
+    str x19, [sp, #-16]!      // Save integer result.
+    .cfi_adjust_cfa_offset 16
+    str d0,  [sp, #8]         // Save floating-point result.
+
+    mov   x0, xSELF           // Pass Thread.
+    add   x1, sp, #16         // Pass SP.
+    mov   x2, x0              // Pass integer result.
+    fmov  x3, d0              // Pass floating-point result.
+    bl   artInstrumentationMethodExitFromCode    // (Thread*, SP, gpr_res, fpr_res)
+
+    mov   x9, x0              // Return address from instrumentation call.
+    mov   xLR, x1             // r1 is holding link register if we're to bounce to deoptimize
+
+    ldr   d0, [sp, #8]        // Restore floating-point result.
+    ldr   x0, [sp], 16        // Restore integer result, and drop stack area.
+    .cfi_adjust_cfa_offset 16
+
+    POP_REF_ONLY_CALLEE_SAVE_FRAME
+
+    br    x9                  // Tail-call out.
+END art_quick_instrumentation_exit
+
+    /*
+     * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization
+     * will long jump to the upcall with a special exception of -1.
+     */
+    .extern artDeoptimize
+ENTRY art_quick_deoptimize
+    SETUP_SAVE_ALL_CALLEE_SAVE_FRAME
+    mov    x0, xSELF          // Pass thread.
+    mov    x1, sp             // Pass SP.
+    bl     artDeoptimize      // artDeoptimize(Thread*, SP)
+END art_quick_deoptimize
+
+
 UNIMPLEMENTED art_quick_indexof
 
    /*
diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index 7474866..c9220c8 100644
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -1293,14 +1293,77 @@ END_FUNCTION art_quick_to_interpreter_bridge
     /*
      * Routine that intercepts method calls and returns.
      */
-UNIMPLEMENTED art_quick_instrumentation_entry
-UNIMPLEMENTED art_quick_instrumentation_exit
+DEFINE_FUNCTION art_quick_instrumentation_entry
+    SETUP_REF_AND_ARGS_CALLEE_SAVE_FRAME
+
+    movq %rdi, %r12               // Preserve method pointer in a callee-save.
+
+    movq %gs:THREAD_SELF_OFFSET, %rdx   // Pass thread.
+    movq %rsp, %rcx                     // Pass SP.
+    movq FRAME_SIZE_REFS_AND_ARGS_CALLEE_SAVE-8(%rsp), %r8   // Pass return PC.
+
+    call PLT_SYMBOL(artInstrumentationMethodEntryFromCode) // (Method*, Object*, Thread*, SP, LR)
+
+                                  // %rax = result of call.
+    movq %r12, %rdi               // Reload method pointer.
+
+    leaq art_quick_instrumentation_exit_local(%rip), %r12   // Set up return through instrumentation
+    movq %r12, FRAME_SIZE_REFS_AND_ARGS_CALLEE_SAVE-8(%rsp) // exit.
+
+    RESTORE_REF_AND_ARGS_CALLEE_SAVE_FRAME
+
+    jmp *%rax                     // Tail call to intended method.
+END_FUNCTION art_quick_instrumentation_entry
+
+DEFINE_FUNCTION art_quick_instrumentation_exit
+    pushq LITERAL(0)          // Push a fake return PC as there will be none on the stack.
+
+    SETUP_REF_ONLY_CALLEE_SAVE_FRAME
+
+    // We need to save rax and xmm0. We could use a callee-save from SETUP_REF_ONLY, but then
+    // we would need to fully restore it. As there are a good number of callee-save registers, it
+    // seems easier to have an extra small stack area. But this should be revisited.
+
+    movq  %rsp, %rsi                          // Pass SP.
+
+    PUSH rax                  // Save integer result.
+    subq LITERAL(8), %rsp     // Save floating-point result.
+    CFI_ADJUST_CFA_OFFSET(8)
+    movd %xmm0, (%rsp)
+
+    movq  %gs:THREAD_SELF_OFFSET, %rdi        // Pass Thread.
+    movq  %rax, %rdx                          // Pass integer result.
+    movq  %xmm0, %rcx                         // Pass floating-point result.
+
+    call PLT_SYMBOL(artInstrumentationMethodExitFromCode)   // (Thread*, SP, gpr_res, fpr_res)
+
+    movq  %rax, %rdi          // Store return PC
+    movq  %rdx, %rsi          // Store second return PC in hidden arg.
+
+    movd (%rsp), %xmm0        // Restore floating-point result.
+    addq LITERAL(8), %rsp
+    CFI_ADJUST_CFA_OFFSET(-8)
+    POP rax                   // Restore integer result.
+
+    addq LITERAL(FRAME_SIZE_REFS_ONLY_CALLEE_SAVE), %rsp   // Drop save frame and fake return pc.
+
+    jmp   *%rdi               // Return.
+END_FUNCTION art_quick_instrumentation_exit
 
     /*
      * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization
      * will long jump to the upcall with a special exception of -1.
      */
-UNIMPLEMENTED art_quick_deoptimize
+DEFINE_FUNCTION art_quick_deoptimize
+    pushq %rsi                     // Fake that we were called. Use hidden arg.
+    SETUP_SAVE_ALL_CALLEE_SAVE_FRAME
+                                   // Stack should be aligned now.
+    movq %rsp, %rsi                           // Pass SP.
+    movq %gs:THREAD_SELF_OFFSET, %rdi         // Pass Thread.
+    call PLT_SYMBOL(artDeoptimize) // artDeoptimize(Thread*, SP)
+    int3                           // Unreachable.
+END_FUNCTION art_quick_deoptimize
+
 
     /*
      * String's compareTo.
diff --git a/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc b/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc
index e0be14e..d161d0b 100644
--- a/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_instrumentation_entrypoints.cc
@@ -15,6 +15,7 @@
  */
 
 #include "callee_save_frame.h"
+#include "instruction_set.h"
 #include "instrumentation.h"
 #include "mirror/art_method-inl.h"
 #include "mirror/object-inl.h"
@@ -40,9 +41,10 @@ extern "C" const void* artInstrumentationMethodEntryFromCode(mirror::ArtMethod*
   return result;
 }
 
-extern "C" uint64_t artInstrumentationMethodExitFromCode(Thread* self,
-                                                         StackReference<mirror::ArtMethod>* sp,
-                                                         uint64_t gpr_result, uint64_t fpr_result)
+extern "C" TwoWordReturn artInstrumentationMethodExitFromCode(Thread* self,
+                                                              StackReference<mirror::ArtMethod>* sp,
+                                                              uint64_t gpr_result,
+                                                              uint64_t fpr_result)
     SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
   // TODO: use FinishCalleeSaveFrameSetup(self, sp, Runtime::kRefsOnly) not the hand inlined below.
   //       We use the hand inline version to ensure the return_pc is assigned before verifying the
@@ -58,9 +60,8 @@ extern "C" uint64_t artInstrumentationMethodExitFromCode(Thread* self,
   self->SetTopOfStack(sp, 0);
   self->VerifyStack();
   instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
-  uint64_t return_or_deoptimize_pc = instrumentation->PopInstrumentationStackFrame(self, return_pc,
-                                                                                   gpr_result,
-                                                                                   fpr_result);
+  TwoWordReturn return_or_deoptimize_pc = instrumentation->PopInstrumentationStackFrame(
+      self, return_pc, gpr_result, fpr_result);
   self->VerifyStack();
   return return_or_deoptimize_pc;
 }
diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
index 5eea252..63010cf 100644
--- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
@@ -20,6 +20,7 @@
 #include "dex_instruction-inl.h"
 #include "entrypoints/entrypoint_utils.h"
 #include "gc/accounting/card_table-inl.h"
+#include "instruction_set.h"
 #include "interpreter/interpreter.h"
 #include "mirror/art_method-inl.h"
 #include "mirror/class-inl.h"
@@ -1615,70 +1616,19 @@ extern "C" uint64_t artQuickGenericJniEndTrampoline(Thread* self,
   }
 }
 
-// The following definitions create return types for two word-sized entities that will be passed
-// in registers so that memory operations for the interface trampolines can be avoided. The entities
-// are the resolved method and the pointer to the code to be invoked.
+// We use TwoWordReturn to optimize scalar returns. We use the hi value for code, and the lo value
+// for the method pointer.
 //
-// On x86, ARM32 and MIPS, this is given for a *scalar* 64bit value. The definition thus *must* be
-// uint64_t or long long int. We use the upper 32b for code, and the lower 32b for the method.
-//
-// On x86_64 and ARM64, structs are decomposed for allocation, so we can create a structs of two
-// size_t-sized values.
-//
-// We need two operations:
-//
-// 1) A flag value that signals failure. The assembly stubs expect the method part to be "0".
-//    GetFailureValue() will return a value that has method == 0.
-//
-// 2) A value that combines a code pointer and a method pointer.
-//    GetSuccessValue() constructs this.
-
-#if defined(__i386__) || defined(__arm__) || defined(__mips__)
-typedef uint64_t MethodAndCode;
-
-// Encodes method_ptr==nullptr and code_ptr==nullptr
-static constexpr MethodAndCode GetFailureValue() {
-  return 0;
-}
-
-// Use the lower 32b for the method pointer and the upper 32b for the code pointer.
-static MethodAndCode GetSuccessValue(const void* code, mirror::ArtMethod* method) {
-  uint32_t method_uint = reinterpret_cast<uint32_t>(method);
-  uint64_t code_uint = reinterpret_cast<uint32_t>(code);
-  return ((code_uint << 32) | method_uint);
-}
-
-#elif defined(__x86_64__) || defined(__aarch64__)
-struct MethodAndCode {
-  uintptr_t method;
-  uintptr_t code;
-};
-
-// Encodes method_ptr==nullptr. Leaves random value in code pointer.
-static MethodAndCode GetFailureValue() {
-  MethodAndCode ret;
-  ret.method = 0;
-  return ret;
-}
-
-// Write values into their respective members.
-static MethodAndCode GetSuccessValue(const void* code, mirror::ArtMethod* method) {
-  MethodAndCode ret;
-  ret.method = reinterpret_cast<uintptr_t>(method);
-  ret.code = reinterpret_cast<uintptr_t>(code);
-  return ret;
-}
-#else
-#error "Unsupported architecture"
-#endif
+// It is valid to use this, as at the usage points here (returns from C functions) we are assuming
+// to hold the mutator lock (see SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) annotations).
 
 template<InvokeType type, bool access_check>
-static MethodAndCode artInvokeCommon(uint32_t method_idx, mirror::Object* this_object,
+static TwoWordReturn artInvokeCommon(uint32_t method_idx, mirror::Object* this_object,
                                      mirror::ArtMethod* caller_method,
                                      Thread* self, StackReference<mirror::ArtMethod>* sp);
 
 template<InvokeType type, bool access_check>
-static MethodAndCode artInvokeCommon(uint32_t method_idx, mirror::Object* this_object,
+static TwoWordReturn artInvokeCommon(uint32_t method_idx, mirror::Object* this_object,
                                      mirror::ArtMethod* caller_method,
                                      Thread* self, StackReference<mirror::ArtMethod>* sp) {
   mirror::ArtMethod* method = FindMethodFast(method_idx, this_object, caller_method, access_check,
@@ -1701,7 +1651,7 @@ static MethodAndCode artInvokeCommon(uint32_t method_idx, mirror::Object* this_o
 
     if (UNLIKELY(method == NULL)) {
       CHECK(self->IsExceptionPending());
-      return GetFailureValue();  // Failure.
+      return GetTwoWordFailureValue();  // Failure.
     }
   }
   DCHECK(!self->IsExceptionPending());
@@ -1711,13 +1661,14 @@ static MethodAndCode artInvokeCommon(uint32_t method_idx, mirror::Object* this_o
   DCHECK(code != nullptr) << "Code was NULL in method: " << PrettyMethod(method) << " location: "
       << MethodHelper(method).GetDexFile().GetLocation();
 
-  return GetSuccessValue(code, method);
+  return GetTwoWordSuccessValue(reinterpret_cast<uintptr_t>(code),
+                                reinterpret_cast<uintptr_t>(method));
 }
 
 // Explicit artInvokeCommon template function declarations to please analysis tool.
 #define EXPLICIT_INVOKE_COMMON_TEMPLATE_DECL(type, access_check)                                \
   template SHARED_LOCKS_REQUIRED(Locks::mutator_lock_)                                          \
-  MethodAndCode artInvokeCommon<type, access_check>(uint32_t method_idx,                        \
+  TwoWordReturn artInvokeCommon<type, access_check>(uint32_t method_idx,                        \
                                                     mirror::Object* this_object,                \
                                                     mirror::ArtMethod* caller_method,           \
                                                     Thread* self,                               \
@@ -1737,7 +1688,7 @@ EXPLICIT_INVOKE_COMMON_TEMPLATE_DECL(kSuper, true);
 
 
 // See comments in runtime_support_asm.S
-extern "C" MethodAndCode artInvokeInterfaceTrampolineWithAccessCheck(uint32_t method_idx,
+extern "C" TwoWordReturn artInvokeInterfaceTrampolineWithAccessCheck(uint32_t method_idx,
     mirror::Object* this_object,
     mirror::ArtMethod* caller_method,
     Thread* self,
@@ -1746,7 +1697,7 @@ extern "C" MethodAndCode artInvokeInterfaceTrampolineWithAccessCheck(uint32_t me
 }
 
 
-extern "C" MethodAndCode artInvokeDirectTrampolineWithAccessCheck(uint32_t method_idx,
+extern "C" TwoWordReturn artInvokeDirectTrampolineWithAccessCheck(uint32_t method_idx,
     mirror::Object* this_object,
     mirror::ArtMethod* caller_method,
     Thread* self,
@@ -1754,7 +1705,7 @@ extern "C" MethodAndCode artInvokeDirectTrampolineWithAccessCheck(uint32_t metho
   return artInvokeCommon<kDirect, true>(method_idx, this_object, caller_method, self, sp);
 }
 
-extern "C" MethodAndCode artInvokeStaticTrampolineWithAccessCheck(uint32_t method_idx,
+extern "C" TwoWordReturn artInvokeStaticTrampolineWithAccessCheck(uint32_t method_idx,
     mirror::Object* this_object,
     mirror::ArtMethod* caller_method,
     Thread* self,
@@ -1762,7 +1713,7 @@ extern "C" MethodAndCode artInvokeStaticTrampolineWithAccessCheck(uint32_t metho
   return artInvokeCommon<kStatic, true>(method_idx, this_object, caller_method, self, sp);
 }
 
-extern "C" MethodAndCode artInvokeSuperTrampolineWithAccessCheck(uint32_t method_idx,
+extern "C" TwoWordReturn artInvokeSuperTrampolineWithAccessCheck(uint32_t method_idx,
     mirror::Object* this_object,
     mirror::ArtMethod* caller_method,
     Thread* self,
@@ -1770,7 +1721,7 @@ extern "C" MethodAndCode artInvokeSuperTrampolineWithAccessCheck(uint32_t method
   return artInvokeCommon<kSuper, true>(method_idx, this_object, caller_method, self, sp);
 }
 
-extern "C" MethodAndCode artInvokeVirtualTrampolineWithAccessCheck(uint32_t method_idx,
+extern "C" TwoWordReturn artInvokeVirtualTrampolineWithAccessCheck(uint32_t method_idx,
     mirror::Object* this_object,
     mirror::ArtMethod* caller_method,
     Thread* self,
@@ -1779,7 +1730,7 @@ extern "C" MethodAndCode artInvokeVirtualTrampolineWithAccessCheck(uint32_t meth
 }
 
 // Determine target of interface dispatch. This object is known non-null.
-extern "C" MethodAndCode artInvokeInterfaceTrampoline(mirror::ArtMethod* interface_method,
+extern "C" TwoWordReturn artInvokeInterfaceTrampoline(mirror::ArtMethod* interface_method,
                                                       mirror::Object* this_object,
                                                       mirror::ArtMethod* caller_method,
                                                       Thread* self,
@@ -1792,7 +1743,7 @@ extern "C" MethodAndCode artInvokeInterfaceTrampoline(mirror::ArtMethod* interfa
       FinishCalleeSaveFrameSetup(self, sp, Runtime::kRefsAndArgs);
       ThrowIncompatibleClassChangeErrorClassForInterfaceDispatch(interface_method, this_object,
                                                                  caller_method);
-      return GetFailureValue();  // Failure.
+      return GetTwoWordFailureValue();  // Failure.
     }
   } else {
     FinishCalleeSaveFrameSetup(self, sp, Runtime::kRefsAndArgs);
@@ -1835,7 +1786,7 @@ extern "C" MethodAndCode artInvokeInterfaceTrampoline(mirror::ArtMethod* interfa
 
     if (UNLIKELY(method == nullptr)) {
       CHECK(self->IsExceptionPending());
-      return GetFailureValue();  // Failure.
+      return GetTwoWordFailureValue();  // Failure.
     }
   }
   const void* code = method->GetEntryPointFromQuickCompiledCode();
@@ -1844,7 +1795,8 @@ extern "C" MethodAndCode artInvokeInterfaceTrampoline(mirror::ArtMethod* interfa
   DCHECK(code != nullptr) << "Code was NULL in method: " << PrettyMethod(method) << " location: "
       << MethodHelper(method).GetDexFile().GetLocation();
 
-  return GetSuccessValue(code, method);
+  return GetTwoWordSuccessValue(reinterpret_cast<uintptr_t>(code),
+                                reinterpret_cast<uintptr_t>(method));
 }
 
 }  // namespace art
diff --git a/runtime/instruction_set.h b/runtime/instruction_set.h
index 7b49b47..67e7100 100644
--- a/runtime/instruction_set.h
+++ b/runtime/instruction_set.h
@@ -216,6 +216,68 @@ class PACKED(4) InstructionSetFeatures {
   uint32_t mask_;
 };
 
+// The following definitions create return types for two word-sized entities that will be passed
+// in registers so that memory operations for the interface trampolines can be avoided. The entities
+// are the resolved method and the pointer to the code to be invoked.
+//
+// On x86, ARM32 and MIPS, this is given for a *scalar* 64bit value. The definition thus *must* be
+// uint64_t or long long int.
+//
+// On x86_64 and ARM64, structs are decomposed for allocation, so we can create a structs of two
+// size_t-sized values.
+//
+// We need two operations:
+//
+// 1) A flag value that signals failure. The assembly stubs expect the lower part to be "0".
+//    GetTwoWordFailureValue() will return a value that has lower part == 0.
+//
+// 2) A value that combines two word-sized values.
+//    GetTwoWordSuccessValue() constructs this.
+//
+// IMPORTANT: If you use this to transfer object pointers, it is your responsibility to ensure
+//            that the object does not move or the value is updated. Simple use of this is NOT SAFE
+//            when the garbage collector can move objects concurrently. Ensure that required locks
+//            are held when using!
+
+#if defined(__i386__) || defined(__arm__) || defined(__mips__)
+typedef uint64_t TwoWordReturn;
+
+// Encodes method_ptr==nullptr and code_ptr==nullptr
+static inline constexpr TwoWordReturn GetTwoWordFailureValue() {
+  return 0;
+}
+
+// Use the lower 32b for the method pointer and the upper 32b for the code pointer.
+static inline TwoWordReturn GetTwoWordSuccessValue(uintptr_t hi, uintptr_t lo) {
+  uint32_t lo32 = static_cast<uint32_t>(lo);
+  uint64_t hi64 = static_cast<uint64_t>(hi);
+  return ((hi64 << 32) | lo32);
+}
+
+#elif defined(__x86_64__) || defined(__aarch64__)
+struct TwoWordReturn {
+  uintptr_t lo;
+  uintptr_t hi;
+};
+
+// Encodes method_ptr==nullptr. Leaves random value in code pointer.
+static inline TwoWordReturn GetTwoWordFailureValue() {
+  TwoWordReturn ret;
+  ret.lo = 0;
+  return ret;
+}
+
+// Write values into their respective members.
+static inline TwoWordReturn GetTwoWordSuccessValue(uintptr_t hi, uintptr_t lo) {
+  TwoWordReturn ret;
+  ret.lo = lo;
+  ret.hi = hi;
+  return ret;
+}
+#else
+#error "Unsupported architecture"
+#endif
+
 }  // namespace art
 
 #endif  // ART_RUNTIME_INSTRUCTION_SET_H_
diff --git a/runtime/instrumentation.cc b/runtime/instrumentation.cc
index 194cb18..a407c55 100644
--- a/runtime/instrumentation.cc
+++ b/runtime/instrumentation.cc
@@ -846,8 +846,9 @@ void Instrumentation::PushInstrumentationStackFrame(Thread* self, mirror::Object
   MethodEnterEvent(self, this_object, method, 0);
 }
 
-uint64_t Instrumentation::PopInstrumentationStackFrame(Thread* self, uintptr_t* return_pc,
-                                                       uint64_t gpr_result, uint64_t fpr_result) {
+TwoWordReturn Instrumentation::PopInstrumentationStackFrame(Thread* self, uintptr_t* return_pc,
+                                                            uint64_t gpr_result,
+                                                            uint64_t fpr_result) {
   // Do the pop.
   std::deque<instrumentation::InstrumentationStackFrame>* stack = self->GetInstrumentationStack();
   CHECK_GT(stack->size(), 0U);
@@ -889,14 +890,14 @@ uint64_t Instrumentation::PopInstrumentationStackFrame(Thread* self, uintptr_t*
                 << " result is " << std::hex << return_value.GetJ();
     }
     self->SetDeoptimizationReturnValue(return_value);
-    return static_cast<uint64_t>(GetQuickDeoptimizationEntryPoint()) |
-        (static_cast<uint64_t>(*return_pc) << 32);
+    return GetTwoWordSuccessValue(*return_pc,
+                                  reinterpret_cast<uintptr_t>(GetQuickDeoptimizationEntryPoint()));
   } else {
     if (kVerboseInstrumentation) {
       LOG(INFO) << "Returning from " << PrettyMethod(method)
                 << " to PC " << reinterpret_cast<void*>(*return_pc);
     }
-    return *return_pc;
+    return GetTwoWordSuccessValue(0, *return_pc);
   }
 }
 
diff --git a/runtime/instrumentation.h b/runtime/instrumentation.h
index 2dd2cd7..6625801 100644
--- a/runtime/instrumentation.h
+++ b/runtime/instrumentation.h
@@ -22,6 +22,7 @@
 #include <list>
 
 #include "atomic.h"
+#include "instruction_set.h"
 #include "base/macros.h"
 #include "base/mutex.h"
 #include "object_callbacks.h"
@@ -311,8 +312,8 @@ class Instrumentation {
 
   // Called when an instrumented method is exited. Removes the pushed instrumentation frame
   // returning the intended link register. Generates method exit events.
-  uint64_t PopInstrumentationStackFrame(Thread* self, uintptr_t* return_pc, uint64_t gpr_result,
-                                        uint64_t fpr_result)
+  TwoWordReturn PopInstrumentationStackFrame(Thread* self, uintptr_t* return_pc,
+                                             uint64_t gpr_result, uint64_t fpr_result)
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
   // Pops an instrumentation frame from the current thread and generate an unwind event.