Lazy pushing shadow frame.

Function:
aget v0, v1, v2
return v0

Original:
push shadow frame;
if (null pointer)        { throw; pop shadow frame; unwind; }
if (index out of bounds) { throw; pop shadow frame; unwind; }
load from array;
pop shadow frame;
return;

New:
if (null pointer)        { push shadow frame; throw; pop shadow frame; unwind; }
if (index out of bounds) { push shadow frame; throw; pop shadow frame; unwind; }
load from array;
return;

Change-Id: I7fc0ff12b9a5454f8e1491b9ce1cdef3afcbed23

9 files changed, 150 insertions, 41 deletions

diff --git a/src/compiler_llvm/art_module.ll b/src/compiler_llvm/art_module.ll
index e7738ad..cebc76c 100644
--- a/src/compiler_llvm/art_module.ll
+++ b/src/compiler_llvm/art_module.ll
@@ -43,7 +43,8 @@ declare void @art_unlock_object_from_code(%JavaObject*, %JavaObject*)
 
 declare void @art_test_suspend_from_code(%JavaObject*)
 
-declare %ShadowFrame* @art_push_shadow_frame_from_code(%ShadowFrame*)
+declare %ShadowFrame* @art_push_shadow_frame_from_code(%ShadowFrame*, %JavaObject*, i32)
+declare %ShadowFrame* @art_push_shadow_frame_noinline_from_code(%ShadowFrame*, %JavaObject*, i32)
 declare void @art_pop_shadow_frame_from_code(%ShadowFrame*)
 
 
diff --git a/src/compiler_llvm/generated/art_module.cc b/src/compiler_llvm/generated/art_module.cc
index f25a88b..1f88e8d 100644
--- a/src/compiler_llvm/generated/art_module.cc
+++ b/src/compiler_llvm/generated/art_module.cc
@@ -84,6 +84,8 @@ FunctionType* FuncTy_5 = FunctionType::get(
 
 std::vector<Type*>FuncTy_6_args;
 FuncTy_6_args.push_back(PointerTy_2);
+FuncTy_6_args.push_back(PointerTy_1);
+FuncTy_6_args.push_back(IntegerType::get(mod->getContext(), 32));
 FunctionType* FuncTy_6 = FunctionType::get(
  /*Result=*/PointerTy_2,
  /*Params=*/FuncTy_6_args,
@@ -423,6 +425,17 @@ func_art_push_shadow_frame_from_code->setCallingConv(CallingConv::C);
 AttrListPtr func_art_push_shadow_frame_from_code_PAL;
 func_art_push_shadow_frame_from_code->setAttributes(func_art_push_shadow_frame_from_code_PAL);
 
+Function* func_art_push_shadow_frame_noinline_from_code = mod->getFunction("art_push_shadow_frame_noinline_from_code");
+if (!func_art_push_shadow_frame_noinline_from_code) {
+func_art_push_shadow_frame_noinline_from_code = Function::Create(
+ /*Type=*/FuncTy_6,
+ /*Linkage=*/GlobalValue::ExternalLinkage,
+ /*Name=*/"art_push_shadow_frame_noinline_from_code", mod); // (external, no body)
+func_art_push_shadow_frame_noinline_from_code->setCallingConv(CallingConv::C);
+}
+AttrListPtr func_art_push_shadow_frame_noinline_from_code_PAL;
+func_art_push_shadow_frame_noinline_from_code->setAttributes(func_art_push_shadow_frame_noinline_from_code_PAL);
+
 Function* func_art_pop_shadow_frame_from_code = mod->getFunction("art_pop_shadow_frame_from_code");
 if (!func_art_pop_shadow_frame_from_code) {
 func_art_pop_shadow_frame_from_code = Function::Create(
diff --git a/src/compiler_llvm/jni_compiler.cc b/src/compiler_llvm/jni_compiler.cc
index c76c80d..e66674c 100644
--- a/src/compiler_llvm/jni_compiler.cc
+++ b/src/compiler_llvm/jni_compiler.cc
@@ -115,29 +115,17 @@ CompiledMethod* JniCompiler::Compile() {
   llvm::StructType* shadow_frame_type = irb_.getShadowFrameTy(sirt_size);
   llvm::AllocaInst* shadow_frame_ = irb_.CreateAlloca(shadow_frame_type);
 
-  // Store the method pointer
-  llvm::Value* method_field_addr =
-    irb_.CreatePtrDisp(shadow_frame_,
-                       irb_.getPtrEquivInt(ShadowFrame::MethodOffset()),
-                       irb_.getJObjectTy()->getPointerTo());
-  irb_.CreateStore(method_object_addr, method_field_addr, kTBAAShadowFrame);
-
   // Store the dex pc
   irb_.StoreToObjectOffset(shadow_frame_,
                            ShadowFrame::DexPCOffset(),
                            irb_.getInt32(0),
                            kTBAAShadowFrame);
 
-  // Store the number of the pointer slots
-  irb_.StoreToObjectOffset(shadow_frame_,
-                           ShadowFrame::NumberOfReferencesOffset(),
-                           irb_.getInt32(sirt_size),
-                           kTBAAShadowFrame);
-
   // Push the shadow frame
   llvm::Value* shadow_frame_upcast = irb_.CreateConstGEP2_32(shadow_frame_, 0, 0);
   llvm::Value* old_shadow_frame =
-      irb_.CreateCall(irb_.GetRuntime(PushShadowFrame), shadow_frame_upcast);
+      irb_.CreateCall3(irb_.GetRuntime(PushShadowFrame),
+                       shadow_frame_upcast, method_object_addr, irb_.getInt32(sirt_size));
 
   // Get JNIEnv
   llvm::Value* jni_env_object_addr =
diff --git a/src/compiler_llvm/method_compiler.cc b/src/compiler_llvm/method_compiler.cc
index 0612855..bdad82b 100644
--- a/src/compiler_llvm/method_compiler.cc
+++ b/src/compiler_llvm/method_compiler.cc
@@ -73,6 +73,7 @@ MethodCompiler::MethodCompiler(CompilationUnit* cunit,
     basic_block_landing_pads_(code_item_->tries_size_, NULL),
     basic_block_unwind_(NULL), basic_block_unreachable_(NULL),
     shadow_frame_(NULL), jvalue_temp_(NULL), old_shadow_frame_(NULL),
+    already_pushed_shadow_frame_(NULL), shadow_frame_size_(0),
     elf_func_idx_(cunit_->AcquireUniqueElfFuncIndex()) {
 }
 
@@ -277,18 +278,21 @@ void MethodCompiler::EmitPrologueAllocShadowFrame() {
   irb_.SetInsertPoint(basic_block_alloca_);
 
   // Allocate the shadow frame now!
-  uint32_t sirt_size = 0;
+  shadow_frame_size_ = 0;
   if (method_info_.need_shadow_frame_entry) {
     for (uint32_t i = 0, num_of_regs = code_item_->registers_size_; i < num_of_regs; ++i) {
       if (IsRegCanBeObject(i)) {
-        reg_to_shadow_frame_index_[i] = sirt_size++;
+        reg_to_shadow_frame_index_[i] = shadow_frame_size_++;
       }
     }
   }
 
-  llvm::StructType* shadow_frame_type = irb_.getShadowFrameTy(sirt_size);
+  llvm::StructType* shadow_frame_type = irb_.getShadowFrameTy(shadow_frame_size_);
   shadow_frame_ = irb_.CreateAlloca(shadow_frame_type);
 
+  // Alloca a pointer to old shadow frame
+  old_shadow_frame_ = irb_.CreateAlloca(shadow_frame_type->getElementType(0)->getPointerTo());
+
   irb_.SetInsertPoint(basic_block_shadow_frame_);
 
   // Zero-initialization of the shadow frame table
@@ -300,26 +304,16 @@ void MethodCompiler::EmitPrologueAllocShadowFrame() {
 
   irb_.CreateStore(zero_initializer, shadow_frame_table, kTBAAShadowFrame);
 
-  // Get method object
-  llvm::Value* method_object_addr = EmitLoadMethodObjectAddr();
-
-  // Store the method pointer
-  irb_.StoreToObjectOffset(shadow_frame_,
-                           ShadowFrame::MethodOffset(),
-                           method_object_addr,
-                           kTBAAShadowFrame);
-
-  // Store the number of the pointer slots
-  irb_.StoreToObjectOffset(shadow_frame_,
-                           ShadowFrame::NumberOfReferencesOffset(),
-                           irb_.getJInt(sirt_size),
-                           kTBAAShadowFrame);
-
-  // Push the shadow frame
-  llvm::Value* shadow_frame_upcast =
-    irb_.CreateConstGEP2_32(shadow_frame_, 0, 0);
+  // Lazy pushing shadow frame
+  if (method_info_.lazy_push_shadow_frame) {
+    irb_.SetInsertPoint(basic_block_alloca_);
+    already_pushed_shadow_frame_ = irb_.CreateAlloca(irb_.getInt1Ty());
+    irb_.SetInsertPoint(basic_block_shadow_frame_);
+    irb_.CreateStore(irb_.getFalse(), already_pushed_shadow_frame_, kTBAARegister);
+    return;
+  }
 
-  old_shadow_frame_ = irb_.CreateCall(irb_.GetRuntime(PushShadowFrame), shadow_frame_upcast);
+  EmitPushShadowFrame(true);
 }
 
 
@@ -3952,12 +3946,50 @@ llvm::Value* MethodCompiler::GetShadowFrameEntry(uint32_t reg_idx) {
 }
 
 
+void MethodCompiler::EmitPushShadowFrame(bool is_inline) {
+  if (!method_info_.need_shadow_frame) {
+    return;
+  }
+  DCHECK(shadow_frame_ != NULL);
+  DCHECK(old_shadow_frame_ != NULL);
+
+  // Get method object
+  llvm::Value* method_object_addr = EmitLoadMethodObjectAddr();
+
+  // Push the shadow frame
+  llvm::Value* shadow_frame_upcast =
+    irb_.CreateConstGEP2_32(shadow_frame_, 0, 0);
+
+  llvm::Value* result =
+      irb_.CreateCall3(irb_.GetRuntime(is_inline ? PushShadowFrame : PushShadowFrameNoInline),
+                       shadow_frame_upcast, method_object_addr, irb_.getJInt(shadow_frame_size_));
+  irb_.CreateStore(result, old_shadow_frame_, kTBAARegister);
+}
+
+
 void MethodCompiler::EmitPopShadowFrame() {
   if (!method_info_.need_shadow_frame) {
     return;
   }
   DCHECK(old_shadow_frame_ != NULL);
-  irb_.CreateCall(irb_.GetRuntime(PopShadowFrame), old_shadow_frame_);
+
+  if (method_info_.lazy_push_shadow_frame) {
+    llvm::BasicBlock* bb_pop = llvm::BasicBlock::Create(*context_, "pop", func_);
+    llvm::BasicBlock* bb_cont = llvm::BasicBlock::Create(*context_, "cont", func_);
+
+    llvm::Value* need_pop = irb_.CreateLoad(already_pushed_shadow_frame_, kTBAARegister);
+    irb_.CreateCondBr(need_pop, bb_pop, bb_cont, kUnlikely);
+
+    irb_.SetInsertPoint(bb_pop);
+    irb_.CreateCall(irb_.GetRuntime(PopShadowFrame),
+                    irb_.CreateLoad(old_shadow_frame_, kTBAARegister));
+    irb_.CreateBr(bb_cont);
+
+    irb_.SetInsertPoint(bb_cont);
+  } else {
+    irb_.CreateCall(irb_.GetRuntime(PopShadowFrame),
+                    irb_.CreateLoad(old_shadow_frame_, kTBAARegister));
+  }
 }
 
 
@@ -3969,6 +4001,21 @@ void MethodCompiler::EmitUpdateDexPC(uint32_t dex_pc) {
                            ShadowFrame::DexPCOffset(),
                            irb_.getInt32(dex_pc),
                            kTBAAShadowFrame);
+  // Lazy pushing shadow frame
+  if (method_info_.lazy_push_shadow_frame) {
+    llvm::BasicBlock* bb_push = CreateBasicBlockWithDexPC(dex_pc, "push");
+    llvm::BasicBlock* bb_cont = CreateBasicBlockWithDexPC(dex_pc, "cont");
+
+    llvm::Value* no_need_push = irb_.CreateLoad(already_pushed_shadow_frame_, kTBAARegister);
+    irb_.CreateCondBr(no_need_push, bb_cont, bb_push, kLikely);
+
+    irb_.SetInsertPoint(bb_push);
+    EmitPushShadowFrame(false);
+    irb_.CreateStore(irb_.getTrue(), already_pushed_shadow_frame_, kTBAARegister);
+    irb_.CreateBr(bb_cont);
+
+    irb_.SetInsertPoint(bb_cont);
+  }
 }
 
 
@@ -4611,6 +4658,10 @@ void MethodCompiler::ComputeMethodInfo() {
   method_info_.need_shadow_frame_entry = has_invoke || may_have_loop;
   // If this method may throw an exception, we need a shadow frame for stack trace (dexpc).
   method_info_.need_shadow_frame = method_info_.need_shadow_frame_entry || may_throw_exception;
+  // If can only throw exception, but can't suspend check (no loop, no invoke),
+  // then there is no shadow frame entry. Only Shadow frame is needed.
+  method_info_.lazy_push_shadow_frame =
+      method_info_.need_shadow_frame && !method_info_.need_shadow_frame_entry;
 }
 
 
diff --git a/src/compiler_llvm/method_compiler.h b/src/compiler_llvm/method_compiler.h
index 9e4c81b..44ab863 100644
--- a/src/compiler_llvm/method_compiler.h
+++ b/src/compiler_llvm/method_compiler.h
@@ -264,6 +264,7 @@ class MethodCompiler {
   void EmitMarkGCCard(llvm::Value* value, llvm::Value* target_addr);
 
   // Shadow frame helper function
+  void EmitPushShadowFrame(bool is_inline);
   void EmitPopShadowFrame();
   void EmitUpdateDexPC(uint32_t dex_pc);
 
@@ -430,6 +431,7 @@ class MethodCompiler {
     bool has_invoke;
     bool need_shadow_frame_entry;
     bool need_shadow_frame;
+    bool lazy_push_shadow_frame;
   };
   MethodInfo method_info_;
 
@@ -474,6 +476,9 @@ class MethodCompiler {
   llvm::AllocaInst* jvalue_temp_;
   llvm::Value* old_shadow_frame_;
 
+  llvm::Value* already_pushed_shadow_frame_;
+  uint32_t shadow_frame_size_;
+
   uint16_t elf_func_idx_;
 };
 
diff --git a/src/compiler_llvm/runtime_support_builder.cc b/src/compiler_llvm/runtime_support_builder.cc
index 3bcd212..2e86938 100644
--- a/src/compiler_llvm/runtime_support_builder.cc
+++ b/src/compiler_llvm/runtime_support_builder.cc
@@ -80,7 +80,10 @@ void RuntimeSupportBuilder::OptimizeRuntimeSupport() {
 
     Function* get_thread = GetRuntimeSupportFunction(GetCurrentThread);
     Value* thread = irb_.CreateCall(get_thread);
-    Value* new_shadow_frame = func->arg_begin();
+    Function::arg_iterator arg_iter = func->arg_begin();
+    Value* new_shadow_frame = arg_iter++;
+    Value* method_object_addr = arg_iter++;
+    Value* shadow_frame_size = arg_iter++;
     Value* old_shadow_frame = irb_.LoadFromObjectOffset(thread,
                                                         Thread::TopShadowFrameOffset().Int32Value(),
                                                         irb_.getArtFrameTy()->getPointerTo(),
@@ -89,10 +92,48 @@ void RuntimeSupportBuilder::OptimizeRuntimeSupport() {
                              Thread::TopShadowFrameOffset().Int32Value(),
                              new_shadow_frame,
                              kTBAARuntimeInfo);
+
+    // Store the method pointer
+    irb_.StoreToObjectOffset(new_shadow_frame,
+                             ShadowFrame::MethodOffset(),
+                             method_object_addr,
+                             kTBAAShadowFrame);
+
+    // Store the number of the pointer slots
+    irb_.StoreToObjectOffset(new_shadow_frame,
+                             ShadowFrame::NumberOfReferencesOffset(),
+                             shadow_frame_size,
+                             kTBAAShadowFrame);
+
+    // Store the link to previous shadow frame
     irb_.StoreToObjectOffset(new_shadow_frame,
                              ShadowFrame::LinkOffset(),
                              old_shadow_frame,
                              kTBAAShadowFrame);
+
+    irb_.CreateRet(old_shadow_frame);
+
+    VERIFY_LLVM_FUNCTION(*func);
+  }
+
+  if (!target_runtime_support_func_[PushShadowFrameNoInline]) {
+    Function* func = GetRuntimeSupportFunction(PushShadowFrameNoInline);
+
+    func->setLinkage(GlobalValue::PrivateLinkage);
+    func->addFnAttr(Attribute::NoInline);
+
+    BasicBlock* basic_block = BasicBlock::Create(context_, "entry", func);
+    irb_.SetInsertPoint(basic_block);
+
+    Function::arg_iterator arg_iter = func->arg_begin();
+    Value* new_shadow_frame = arg_iter++;
+    Value* method_object_addr = arg_iter++;
+    Value* shadow_frame_size = arg_iter++;
+
+    // Call inline version
+    Value* old_shadow_frame =
+      irb_.CreateCall3(GetRuntimeSupportFunction(PushShadowFrame),
+                       new_shadow_frame, method_object_addr, shadow_frame_size);
     irb_.CreateRet(old_shadow_frame);
 
     VERIFY_LLVM_FUNCTION(*func);
diff --git a/src/compiler_llvm/runtime_support_func_list.h b/src/compiler_llvm/runtime_support_func_list.h
index 62729c6..74af5da 100644
--- a/src/compiler_llvm/runtime_support_func_list.h
+++ b/src/compiler_llvm/runtime_support_func_list.h
@@ -20,6 +20,7 @@
   V(GetCurrentThread, art_get_current_thread_from_code) \
   V(SetCurrentThread, art_set_current_thread_from_code) \
   V(PushShadowFrame, art_push_shadow_frame_from_code) \
+  V(PushShadowFrameNoInline, art_push_shadow_frame_noinline_from_code) \
   V(PopShadowFrame, art_pop_shadow_frame_from_code) \
   V(TestSuspend, art_test_suspend_from_code) \
   V(ThrowException, art_throw_exception_from_code) \
diff --git a/src/compiler_llvm/runtime_support_llvm.cc b/src/compiler_llvm/runtime_support_llvm.cc
index 86bf954..08e5921 100644
--- a/src/compiler_llvm/runtime_support_llvm.cc
+++ b/src/compiler_llvm/runtime_support_llvm.cc
@@ -80,7 +80,13 @@ void art_test_suspend_from_code(Thread* thread) {
   Runtime::Current()->GetThreadList()->FullSuspendCheck(thread);
 }
 
-void* art_push_shadow_frame_from_code(void* new_shadow_frame) {
+void* art_push_shadow_frame_from_code(void* new_shadow_frame, Object* method, uint32_t size) {
+  LOG(FATAL) << "Implemented by IRBuilder.";
+  return NULL;
+}
+
+void* art_push_shadow_frame_noinline_from_code(void* new_shadow_frame,
+                                               Object* method, uint32_t size) {
   LOG(FATAL) << "Implemented by IRBuilder.";
   return NULL;
 }
diff --git a/src/compiler_llvm/runtime_support_llvm.h b/src/compiler_llvm/runtime_support_llvm.h
index b7f0655..5647c54 100644
--- a/src/compiler_llvm/runtime_support_llvm.h
+++ b/src/compiler_llvm/runtime_support_llvm.h
@@ -36,7 +36,10 @@ class Object;
 // Thread
 //----------------------------------------------------------------------------
 
-void* art_push_shadow_frame_from_code(void* new_shadow_frame);
+void* art_push_shadow_frame_from_code(void* new_shadow_frame, Object* method, uint32_t size);
+
+void* art_push_shadow_frame_noinline_from_code(void* new_shadow_frame,
+                                               Object* method, uint32_t size);
 
 void art_pop_shadow_frame_from_code(void*);