Quick: Clean up temp use counting.

For the boot image on arm64 and x86-64 we're using true
PC-relative addressing, so pc_rel_temp_ is nullptr and
CanUsePcRelDexCacheArrayLoad() returns true, but we're not
actually using the ArtMethod* so fix the AnalyzeMIR() to
take it into account.

Also don't count intrinsic invokes towards ArtMethod* uses.
To avoid repeated method inliner inquiries about whether a
method is intrinsic or special (requiring lock acquisition),
cache that information in MirMethodLoweringInfo. As part of
that cleanup, take quickened invokes into account for
suspend check elimination.

Change-Id: I5b4ec124221c0db1314c8e72675976c110ebe7ca

10 files changed, 75 insertions, 40 deletions

diff --git a/compiler/dex/bb_optimizations.h b/compiler/dex/bb_optimizations.h
index 93d83c6..0850f42 100644
--- a/compiler/dex/bb_optimizations.h
+++ b/compiler/dex/bb_optimizations.h
@@ -403,13 +403,6 @@ class SuspendCheckElimination : public PassME {
     DCHECK(bb != nullptr);
     return c_unit->mir_graph->EliminateSuspendChecks(bb);
   }
-
-  void End(PassDataHolder* data) const {
-    DCHECK(data != nullptr);
-    CompilationUnit* c_unit = down_cast<const PassMEDataHolder*>(data)->c_unit;
-    DCHECK(c_unit != nullptr);
-    c_unit->mir_graph->EliminateSuspendChecksEnd();
-  }
 };
 
 }  // namespace art
diff --git a/compiler/dex/mir_graph.h b/compiler/dex/mir_graph.h
index d4a9eb9..d6c4b64 100644
--- a/compiler/dex/mir_graph.h
+++ b/compiler/dex/mir_graph.h
@@ -1085,7 +1085,6 @@ class MIRGraph {
   void EliminateDeadCodeEnd();
   bool EliminateSuspendChecksGate();
   bool EliminateSuspendChecks(BasicBlock* bb);
-  void EliminateSuspendChecksEnd();
 
   uint16_t GetGvnIFieldId(MIR* mir) const {
     DCHECK(IsInstructionIGetOrIPut(mir->dalvikInsn.opcode));
@@ -1408,10 +1407,6 @@ class MIRGraph {
       uint16_t* sfield_ids;  // Ditto.
       GvnDeadCodeElimination* dce;
     } gvn;
-    // Suspend check elimination.
-    struct {
-      DexFileMethodInliner* inliner;
-    } sce;
   } temp_;
   static const int kInvalidEntry = -1;
   ArenaVector<BasicBlock*> block_list_;
diff --git a/compiler/dex/mir_method_info.cc b/compiler/dex/mir_method_info.cc
index 831ad42..0c84b82 100644
--- a/compiler/dex/mir_method_info.cc
+++ b/compiler/dex/mir_method_info.cc
@@ -16,6 +16,8 @@
 
 # include "mir_method_info.h"
 
+#include "dex/quick/dex_file_method_inliner.h"
+#include "dex/quick/dex_file_to_method_inliner_map.h"
 #include "dex/verified_method.h"
 #include "driver/compiler_driver.h"
 #include "driver/dex_compilation_unit.h"
@@ -64,6 +66,9 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver,
   const DexFile* const dex_file = mUnit->GetDexFile();
   const bool use_jit = runtime->UseJit();
   const VerifiedMethod* const verified_method = mUnit->GetVerifiedMethod();
+  DexFileToMethodInlinerMap* inliner_map = compiler_driver->GetMethodInlinerMap();
+  DexFileMethodInliner* default_inliner =
+      (inliner_map != nullptr) ? inliner_map->GetMethodInliner(dex_file) : nullptr;
 
   for (auto it = method_infos, end = method_infos + count; it != end; ++it) {
     // For quickened invokes, the dex method idx is actually the mir offset.
@@ -122,6 +127,7 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver,
     if (UNLIKELY(resolved_method == nullptr)) {
       continue;
     }
+
     compiler_driver->GetResolvedMethodDexFileLocation(resolved_method,
         &it->declaring_dex_file_, &it->declaring_class_idx_, &it->declaring_method_idx_);
     if (!it->IsQuickened()) {
@@ -133,6 +139,7 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver,
       it->vtable_idx_ =
           compiler_driver->GetResolvedMethodVTableIndex(resolved_method, invoke_type);
     }
+
     MethodReference target_method(it->target_dex_file_, it->target_method_idx_);
     int fast_path_flags = compiler_driver->IsFastInvoke(
         soa, current_dex_cache, class_loader, mUnit, referrer_class.Get(), resolved_method,
@@ -140,10 +147,23 @@ void MirMethodLoweringInfo::Resolve(CompilerDriver* compiler_driver,
     const bool is_referrers_class = referrer_class.Get() == resolved_method->GetDeclaringClass();
     const bool is_class_initialized =
         compiler_driver->IsMethodsClassInitialized(referrer_class.Get(), resolved_method);
+
+    // Check if the target method is intrinsic or special.
+    InlineMethodFlags is_intrinsic_or_special = kNoInlineMethodFlags;
+    if (inliner_map != nullptr) {
+      auto* inliner = (target_method.dex_file == dex_file)
+          ? default_inliner
+          : inliner_map->GetMethodInliner(target_method.dex_file);
+      is_intrinsic_or_special = inliner->IsIntrinsicOrSpecial(target_method.dex_method_index);
+    }
+
     uint16_t other_flags = it->flags_ &
-        ~(kFlagFastPath | kFlagClassIsInitialized | (kInvokeTypeMask << kBitSharpTypeBegin));
+        ~(kFlagFastPath | kFlagIsIntrinsic | kFlagIsSpecial | kFlagClassIsInitialized |
+            (kInvokeTypeMask << kBitSharpTypeBegin));
     it->flags_ = other_flags |
         (fast_path_flags != 0 ? kFlagFastPath : 0u) |
+        ((is_intrinsic_or_special & kInlineIntrinsic) != 0 ? kFlagIsIntrinsic : 0u) |
+        ((is_intrinsic_or_special & kInlineSpecial) != 0 ? kFlagIsSpecial : 0u) |
         (static_cast<uint16_t>(invoke_type) << kBitSharpTypeBegin) |
         (is_referrers_class ? kFlagIsReferrersClass : 0u) |
         (is_class_initialized ? kFlagClassIsInitialized : 0u);
diff --git a/compiler/dex/mir_method_info.h b/compiler/dex/mir_method_info.h
index e131c96..7230c46 100644
--- a/compiler/dex/mir_method_info.h
+++ b/compiler/dex/mir_method_info.h
@@ -127,6 +127,14 @@ class MirMethodLoweringInfo : public MirMethodInfo {
     return (flags_ & kFlagFastPath) != 0u;
   }
 
+  bool IsIntrinsic() const {
+    return (flags_ & kFlagIsIntrinsic) != 0u;
+  }
+
+  bool IsSpecial() const {
+    return (flags_ & kFlagIsSpecial) != 0u;
+  }
+
   bool IsReferrersClass() const {
     return (flags_ & kFlagIsReferrersClass) != 0;
   }
@@ -188,9 +196,11 @@ class MirMethodLoweringInfo : public MirMethodInfo {
  private:
   enum {
     kBitFastPath = kMethodInfoBitEnd,
+    kBitIsIntrinsic,
+    kBitIsSpecial,
     kBitInvokeTypeBegin,
     kBitInvokeTypeEnd = kBitInvokeTypeBegin + 3,  // 3 bits for invoke type.
-    kBitSharpTypeBegin,
+    kBitSharpTypeBegin = kBitInvokeTypeEnd,
     kBitSharpTypeEnd = kBitSharpTypeBegin + 3,  // 3 bits for sharp type.
     kBitIsReferrersClass = kBitSharpTypeEnd,
     kBitClassIsInitialized,
@@ -199,6 +209,8 @@ class MirMethodLoweringInfo : public MirMethodInfo {
   };
   static_assert(kMethodLoweringInfoBitEnd <= 16, "Too many flags");
   static constexpr uint16_t kFlagFastPath = 1u << kBitFastPath;
+  static constexpr uint16_t kFlagIsIntrinsic = 1u << kBitIsIntrinsic;
+  static constexpr uint16_t kFlagIsSpecial = 1u << kBitIsSpecial;
   static constexpr uint16_t kFlagIsReferrersClass = 1u << kBitIsReferrersClass;
   static constexpr uint16_t kFlagClassIsInitialized = 1u << kBitClassIsInitialized;
   static constexpr uint16_t kFlagQuickened = 1u << kBitQuickened;
diff --git a/compiler/dex/mir_optimization.cc b/compiler/dex/mir_optimization.cc
index 5dcc903..9d7b4b4 100644
--- a/compiler/dex/mir_optimization.cc
+++ b/compiler/dex/mir_optimization.cc
@@ -1517,7 +1517,7 @@ void MIRGraph::InlineSpecialMethods(BasicBlock* bb) {
       continue;
     }
     const MirMethodLoweringInfo& method_info = GetMethodLoweringInfo(mir);
-    if (!method_info.FastPath()) {
+    if (!method_info.FastPath() || !method_info.IsSpecial()) {
       continue;
     }
 
@@ -1659,10 +1659,6 @@ bool MIRGraph::EliminateSuspendChecksGate() {
       !HasInvokes()) {               // No invokes to actually eliminate any suspend checks.
     return false;
   }
-  if (cu_->compiler_driver != nullptr && cu_->compiler_driver->GetMethodInlinerMap() != nullptr) {
-    temp_.sce.inliner =
-        cu_->compiler_driver->GetMethodInlinerMap()->GetMethodInliner(cu_->dex_file);
-  }
   suspend_checks_in_loops_ = arena_->AllocArray<uint32_t>(GetNumBlocks(), kArenaAllocMisc);
   return true;
 }
@@ -1680,9 +1676,9 @@ bool MIRGraph::EliminateSuspendChecks(BasicBlock* bb) {
   uint32_t suspend_checks_in_loops = (1u << bb->nesting_depth) - 1u;  // Start with all loop heads.
   bool found_invoke = false;
   for (MIR* mir = bb->first_mir_insn; mir != nullptr; mir = mir->next) {
-    if (IsInstructionInvoke(mir->dalvikInsn.opcode) &&
-        (temp_.sce.inliner == nullptr ||
-         !temp_.sce.inliner->IsIntrinsic(mir->dalvikInsn.vB, nullptr))) {
+    if ((IsInstructionInvoke(mir->dalvikInsn.opcode) ||
+        IsInstructionQuickInvoke(mir->dalvikInsn.opcode)) &&
+        !GetMethodLoweringInfo(mir).IsIntrinsic()) {
       // Non-intrinsic invoke, rely on a suspend point in the invoked method.
       found_invoke = true;
       break;
@@ -1745,10 +1741,6 @@ bool MIRGraph::EliminateSuspendChecks(BasicBlock* bb) {
   return true;
 }
 
-void MIRGraph::EliminateSuspendChecksEnd() {
-  temp_.sce.inliner = nullptr;
-}
-
 bool MIRGraph::CanThrow(MIR* mir) const {
   if ((mir->dalvikInsn.FlagsOf() & Instruction::kThrow) == 0) {
     return false;
diff --git a/compiler/dex/mir_optimization_test.cc b/compiler/dex/mir_optimization_test.cc
index 9ce5ebb..10a4337 100644
--- a/compiler/dex/mir_optimization_test.cc
+++ b/compiler/dex/mir_optimization_test.cc
@@ -474,7 +474,6 @@ class SuspendCheckEliminationTest : public MirOptimizationTest {
     for (BasicBlock* bb = iterator.Next(change); bb != nullptr; bb = iterator.Next(change)) {
       change = cu_.mir_graph->EliminateSuspendChecks(bb);
     }
-    cu_.mir_graph->EliminateSuspendChecksEnd();
   }
 
   SuspendCheckEliminationTest()
diff --git a/compiler/dex/quick/dex_file_method_inliner.cc b/compiler/dex/quick/dex_file_method_inliner.cc
index 8e3f4ef..4ac6c0c 100644
--- a/compiler/dex/quick/dex_file_method_inliner.cc
+++ b/compiler/dex/quick/dex_file_method_inliner.cc
@@ -413,6 +413,17 @@ bool DexFileMethodInliner::AnalyseMethodCode(verifier::MethodVerifier* verifier)
   return success && AddInlineMethod(verifier->GetMethodReference().dex_method_index, method);
 }
 
+InlineMethodFlags DexFileMethodInliner::IsIntrinsicOrSpecial(uint32_t method_index) {
+  ReaderMutexLock mu(Thread::Current(), lock_);
+  auto it = inline_methods_.find(method_index);
+  if (it != inline_methods_.end()) {
+    DCHECK_NE(it->second.flags & (kInlineIntrinsic | kInlineSpecial), 0);
+    return it->second.flags;
+  } else {
+    return kNoInlineMethodFlags;
+  }
+}
+
 bool DexFileMethodInliner::IsIntrinsic(uint32_t method_index, InlineMethod* intrinsic) {
   ReaderMutexLock mu(Thread::Current(), lock_);
   auto it = inline_methods_.find(method_index);
diff --git a/compiler/dex/quick/dex_file_method_inliner.h b/compiler/dex/quick/dex_file_method_inliner.h
index cb521da..d1e5621 100644
--- a/compiler/dex/quick/dex_file_method_inliner.h
+++ b/compiler/dex/quick/dex_file_method_inliner.h
@@ -65,6 +65,11 @@ class DexFileMethodInliner {
         SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) LOCKS_EXCLUDED(lock_);
 
     /**
+     * Check whether a particular method index corresponds to an intrinsic or special function.
+     */
+    InlineMethodFlags IsIntrinsicOrSpecial(uint32_t method_index) LOCKS_EXCLUDED(lock_);
+
+    /**
      * Check whether a particular method index corresponds to an intrinsic function.
      */
     bool IsIntrinsic(uint32_t method_index, InlineMethod* intrinsic) LOCKS_EXCLUDED(lock_);
diff --git a/compiler/dex/quick/gen_invoke.cc b/compiler/dex/quick/gen_invoke.cc
index e747239..db7095d 100755
--- a/compiler/dex/quick/gen_invoke.cc
+++ b/compiler/dex/quick/gen_invoke.cc
@@ -1435,10 +1435,12 @@ bool Mir2Lir::GenInlinedUnsafePut(CallInfo* info, bool is_long,
 
 void Mir2Lir::GenInvoke(CallInfo* info) {
   DCHECK(cu_->compiler_driver->GetMethodInlinerMap() != nullptr);
-  const DexFile* dex_file = info->method_ref.dex_file;
-  if (cu_->compiler_driver->GetMethodInlinerMap()->GetMethodInliner(dex_file)
-      ->GenIntrinsic(this, info)) {
-    return;
+  if (mir_graph_->GetMethodLoweringInfo(info->mir).IsIntrinsic()) {
+    const DexFile* dex_file = info->method_ref.dex_file;
+    auto* inliner = cu_->compiler_driver->GetMethodInlinerMap()->GetMethodInliner(dex_file);
+    if (inliner->GenIntrinsic(this, info)) {
+      return;
+    }
   }
   GenInvokeNoInline(info);
 }
diff --git a/compiler/dex/quick/ralloc_util.cc b/compiler/dex/quick/ralloc_util.cc
index 487d31c..e779479 100644
--- a/compiler/dex/quick/ralloc_util.cc
+++ b/compiler/dex/quick/ralloc_util.cc
@@ -1156,7 +1156,7 @@ void Mir2Lir::AnalyzeMIR(RefCounts* core_counts, MIR* mir, uint32_t weight) {
               mir_graph_->GetCurrentDexCompilationUnit(), mir->offset)) {
         break;  // No code generated.
       }
-      if (!needs_access_check && !use_declaring_class && pc_rel_temp_ != nullptr) {
+      if (!needs_access_check && !use_declaring_class && CanUseOpPcRelDexCacheArrayLoad()) {
         uses_pc_rel_load = true;  // And ignore method use in slow path.
         dex_cache_array_offset = dex_cache_arrays_layout_.TypeOffset(type_idx);
       } else {
@@ -1166,7 +1166,7 @@ void Mir2Lir::AnalyzeMIR(RefCounts* core_counts, MIR* mir, uint32_t weight) {
     }
 
     case Instruction::CONST_CLASS:
-      if (pc_rel_temp_ != nullptr &&
+      if (CanUseOpPcRelDexCacheArrayLoad() &&
           cu_->compiler_driver->CanAccessTypeWithoutChecks(cu_->method_idx, *cu_->dex_file,
                                                            mir->dalvikInsn.vB)) {
         uses_pc_rel_load = true;  // And ignore method use in slow path.
@@ -1178,7 +1178,7 @@ void Mir2Lir::AnalyzeMIR(RefCounts* core_counts, MIR* mir, uint32_t weight) {
 
     case Instruction::CONST_STRING:
     case Instruction::CONST_STRING_JUMBO:
-      if (pc_rel_temp_ != nullptr) {
+      if (CanUseOpPcRelDexCacheArrayLoad()) {
         uses_pc_rel_load = true;  // And ignore method use in slow path.
         dex_cache_array_offset = dex_cache_arrays_layout_.StringOffset(mir->dalvikInsn.vB);
       } else {
@@ -1200,11 +1200,13 @@ void Mir2Lir::AnalyzeMIR(RefCounts* core_counts, MIR* mir, uint32_t weight) {
     case Instruction::INVOKE_VIRTUAL_RANGE_QUICK: {
       const MirMethodLoweringInfo& info = mir_graph_->GetMethodLoweringInfo(mir);
       InvokeType sharp_type = info.GetSharpType();
-      if (!info.FastPath() || (sharp_type != kStatic && sharp_type != kDirect)) {
+      if (info.IsIntrinsic()) {
+        // Nothing to do, if an intrinsic uses ArtMethod* it's in the slow-path - don't count it.
+      } else if (!info.FastPath() || (sharp_type != kStatic && sharp_type != kDirect)) {
         // Nothing to do, the generated code or entrypoint uses method from the stack.
       } else if (info.DirectCode() != 0 && info.DirectMethod() != 0) {
         // Nothing to do, the generated code uses method from the stack.
-      } else if (pc_rel_temp_ != nullptr) {
+      } else if (CanUseOpPcRelDexCacheArrayLoad()) {
         uses_pc_rel_load = true;
         dex_cache_array_offset = dex_cache_arrays_layout_.MethodOffset(mir->dalvikInsn.vB);
       } else {
@@ -1245,7 +1247,7 @@ void Mir2Lir::AnalyzeMIR(RefCounts* core_counts, MIR* mir, uint32_t weight) {
           ? field_info.FastGet()
           : field_info.FastPut();
       if (fast && (cu_->enable_debug & (1 << kDebugSlowFieldPath)) == 0) {
-        if (!field_info.IsReferrersClass() && pc_rel_temp_ != nullptr) {
+        if (!field_info.IsReferrersClass() && CanUseOpPcRelDexCacheArrayLoad()) {
           uses_pc_rel_load = true;  // And ignore method use in slow path.
           dex_cache_array_offset = dex_cache_arrays_layout_.TypeOffset(field_info.StorageIndex());
         } else {
@@ -1264,9 +1266,13 @@ void Mir2Lir::AnalyzeMIR(RefCounts* core_counts, MIR* mir, uint32_t weight) {
     core_counts[SRegToPMap(mir_graph_->GetMethodLoc().s_reg_low)].count += weight;
   }
   if (uses_pc_rel_load) {
-    core_counts[SRegToPMap(pc_rel_temp_->s_reg_low)].count += weight;
-    DCHECK_NE(dex_cache_array_offset, std::numeric_limits<uint32_t>::max());
-    dex_cache_arrays_min_offset_ = std::min(dex_cache_arrays_min_offset_, dex_cache_array_offset);
+    if (pc_rel_temp_ != nullptr) {
+      core_counts[SRegToPMap(pc_rel_temp_->s_reg_low)].count += weight;
+      DCHECK_NE(dex_cache_array_offset, std::numeric_limits<uint32_t>::max());
+      dex_cache_arrays_min_offset_ = std::min(dex_cache_arrays_min_offset_, dex_cache_array_offset);
+    } else {
+      // Nothing to do, using PC-relative addressing without promoting base PC to register.
+    }
   }
 }