Set vtable in class object to null after linking.

This is follow-up work of embedding imt and vtable for
faster interface/virtual call dispatching.
Once vtable becomes embedded, the original vtable is nulled.

Change-Id: I307696657d1e283654169dbecb8f7815c42bbabc

9 files changed, 123 insertions, 49 deletions

diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 6c5679e..c40424b 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -3570,9 +3570,9 @@ bool ClassLinker::ValidateSuperClassDescriptors(Handle<mirror::Class> klass) {
   MethodHelper super_mh(hs.NewHandle<mirror::ArtMethod>(nullptr));
   if (klass->HasSuperClass() &&
       klass->GetClassLoader() != klass->GetSuperClass()->GetClassLoader()) {
-    for (int i = klass->GetSuperClass()->GetVTable()->GetLength() - 1; i >= 0; --i) {
-      mh.ChangeMethod(klass->GetVTable()->GetWithoutChecks(i));
-      super_mh.ChangeMethod(klass->GetSuperClass()->GetVTable()->GetWithoutChecks(i));
+    for (int i = klass->GetSuperClass()->GetVTableLength() - 1; i >= 0; --i) {
+      mh.ChangeMethod(klass->GetVTableEntry(i));
+      super_mh.ChangeMethod(klass->GetSuperClass()->GetVTableEntry(i));
       if (mh.GetMethod() != super_mh.GetMethod() &&
           !mh.HasSameSignatureWithDifferentClassLoaders(&super_mh)) {
         ThrowLinkageError(klass.Get(),
@@ -3730,10 +3730,6 @@ bool ClassLinker::LinkClass(Thread* self, const char* descriptor, Handle<mirror:
     // This will notify waiters on new_class that saw the not yet resolved
     // class in the class_table_ during EnsureResolved.
     new_class_h->SetStatus(mirror::Class::kStatusResolved, self);
-
-    // Only embedded imt should be used from this point.
-    new_class_h->SetImTable(NULL);
-    // TODO: remove vtable and only use embedded vtable.
   }
   return true;
 }
@@ -3866,17 +3862,31 @@ bool ClassLinker::LinkMethods(Thread* self, Handle<mirror::Class> klass,
 bool ClassLinker::LinkVirtualMethods(Thread* self, Handle<mirror::Class> klass) {
   if (klass->HasSuperClass()) {
     uint32_t max_count = klass->NumVirtualMethods() +
-        klass->GetSuperClass()->GetVTable()->GetLength();
-    size_t actual_count = klass->GetSuperClass()->GetVTable()->GetLength();
+        klass->GetSuperClass()->GetVTableLength();
+    size_t actual_count = klass->GetSuperClass()->GetVTableLength();
     CHECK_LE(actual_count, max_count);
-    // TODO: do not assign to the vtable field until it is fully constructed.
     StackHandleScope<3> hs(self);
-    Handle<mirror::ObjectArray<mirror::ArtMethod>> vtable(
-        hs.NewHandle(klass->GetSuperClass()->GetVTable()->CopyOf(self, max_count)));
-    if (UNLIKELY(vtable.Get() == NULL)) {
-      CHECK(self->IsExceptionPending());  // OOME.
-      return false;
+    Handle<mirror::ObjectArray<mirror::ArtMethod>> vtable;
+    mirror::Class* super_class = klass->GetSuperClass();
+    if (super_class->ShouldHaveEmbeddedImtAndVTable()) {
+      vtable = hs.NewHandle(AllocArtMethodArray(self, max_count));
+      if (UNLIKELY(vtable.Get() == nullptr)) {
+        CHECK(self->IsExceptionPending());  // OOME.
+        return false;
+      }
+      int len = super_class->GetVTableLength();
+      for (int i=0; i<len; i++) {
+        vtable->Set<false>(i, super_class->GetVTableEntry(i));
+      }
+    } else {
+      CHECK(super_class->GetVTable() != nullptr) << PrettyClass(super_class);
+      vtable = hs.NewHandle(super_class->GetVTable()->CopyOf(self, max_count));
+      if (UNLIKELY(vtable.Get() == nullptr)) {
+        CHECK(self->IsExceptionPending());  // OOME.
+        return false;
+      }
     }
+
     // See if any of our virtual methods override the superclass.
     MethodHelper local_mh(hs.NewHandle<mirror::ArtMethod>(nullptr));
     MethodHelper super_mh(hs.NewHandle<mirror::ArtMethod>(nullptr));
diff --git a/runtime/class_linker_test.cc b/runtime/class_linker_test.cc
index 21fe006..8e16d9b 100644
--- a/runtime/class_linker_test.cc
+++ b/runtime/class_linker_test.cc
@@ -91,7 +91,7 @@ class ClassLinkerTest : public CommonRuntimeTest {
     EXPECT_EQ(0U, primitive->NumInstanceFields());
     EXPECT_EQ(0U, primitive->NumStaticFields());
     EXPECT_EQ(0U, primitive->NumDirectInterfaces());
-    EXPECT_TRUE(primitive->GetVTable() == NULL);
+    EXPECT_FALSE(primitive->HasVTable());
     EXPECT_EQ(0, primitive->GetIfTableCount());
     EXPECT_TRUE(primitive->GetIfTable() == NULL);
     EXPECT_EQ(kAccPublic | kAccFinal | kAccAbstract, primitive->GetAccessFlags());
@@ -143,7 +143,7 @@ class ClassLinkerTest : public CommonRuntimeTest {
     EXPECT_EQ(0U, array->NumInstanceFields());
     EXPECT_EQ(0U, array->NumStaticFields());
     EXPECT_EQ(2U, array->NumDirectInterfaces());
-    EXPECT_TRUE(array->GetVTable() != NULL);
+    EXPECT_TRUE(array->ShouldHaveEmbeddedImtAndVTable());
     EXPECT_EQ(2, array->GetIfTableCount());
     ASSERT_TRUE(array->GetIfTable() != NULL);
     mirror::Class* direct_interface0 = mirror::Class::GetDirectInterface(self, array, 0);
@@ -216,7 +216,7 @@ class ClassLinkerTest : public CommonRuntimeTest {
         EXPECT_NE(0U, klass->NumDirectMethods());
       }
     }
-    EXPECT_EQ(klass->IsInterface(), klass->GetVTable() == NULL);
+    EXPECT_EQ(klass->IsInterface(), !klass->HasVTable());
     mirror::IfTable* iftable = klass->GetIfTable();
     for (int i = 0; i < klass->GetIfTableCount(); i++) {
       mirror::Class* interface = iftable->GetInterface(i);
diff --git a/runtime/entrypoints/entrypoint_utils-inl.h b/runtime/entrypoints/entrypoint_utils-inl.h
index 90c8fcf..cb0be04 100644
--- a/runtime/entrypoints/entrypoint_utils-inl.h
+++ b/runtime/entrypoints/entrypoint_utils-inl.h
@@ -390,26 +390,26 @@ static inline mirror::ArtMethod* FindMethodFromCode(uint32_t method_idx,
     case kDirect:
       return resolved_method;
     case kVirtual: {
-      mirror::ObjectArray<mirror::ArtMethod>* vtable = (*this_object)->GetClass()->GetVTable();
+      mirror::Class* klass = (*this_object)->GetClass();
       uint16_t vtable_index = resolved_method->GetMethodIndex();
       if (access_check &&
-          (vtable == nullptr || vtable_index >= static_cast<uint32_t>(vtable->GetLength()))) {
+          (!klass->HasVTable() ||
+           vtable_index >= static_cast<uint32_t>(klass->GetVTableLength()))) {
         // Behavior to agree with that of the verifier.
         ThrowNoSuchMethodError(type, resolved_method->GetDeclaringClass(),
                                resolved_method->GetName(), resolved_method->GetSignature());
         return nullptr;  // Failure.
       }
-      DCHECK(vtable != nullptr);
-      return vtable->GetWithoutChecks(vtable_index);
+      DCHECK(klass->HasVTable()) << PrettyClass(klass);
+      return klass->GetVTableEntry(vtable_index);
     }
     case kSuper: {
       mirror::Class* super_class = (*referrer)->GetDeclaringClass()->GetSuperClass();
       uint16_t vtable_index = resolved_method->GetMethodIndex();
-      mirror::ObjectArray<mirror::ArtMethod>* vtable;
       if (access_check) {
         // Check existence of super class.
-        vtable = (super_class != nullptr) ? super_class->GetVTable() : nullptr;
-        if (vtable == nullptr || vtable_index >= static_cast<uint32_t>(vtable->GetLength())) {
+        if (super_class == nullptr || !super_class->HasVTable() ||
+            vtable_index >= static_cast<uint32_t>(super_class->GetVTableLength())) {
           // Behavior to agree with that of the verifier.
           ThrowNoSuchMethodError(type, resolved_method->GetDeclaringClass(),
                                  resolved_method->GetName(), resolved_method->GetSignature());
@@ -418,10 +418,9 @@ static inline mirror::ArtMethod* FindMethodFromCode(uint32_t method_idx,
       } else {
         // Super class must exist.
         DCHECK(super_class != nullptr);
-        vtable = super_class->GetVTable();
       }
-      DCHECK(vtable != nullptr);
-      return vtable->GetWithoutChecks(vtable_index);
+      DCHECK(super_class->HasVTable());
+      return super_class->GetVTableEntry(vtable_index);
     }
     case kInterface: {
       uint32_t imt_index = resolved_method->GetDexMethodIndex() % mirror::Class::kImtSize;
@@ -555,11 +554,11 @@ static inline mirror::ArtMethod* FindMethodFast(uint32_t method_idx,
   } else if (is_direct) {
     return resolved_method;
   } else if (type == kSuper) {
-    return referrer->GetDeclaringClass()->GetSuperClass()->GetVTable()->
-        Get(resolved_method->GetMethodIndex());
+    return referrer->GetDeclaringClass()->GetSuperClass()
+                   ->GetVTableEntry(resolved_method->GetMethodIndex());
   } else {
     DCHECK(type == kVirtual);
-    return this_object->GetClass()->GetVTable()->Get(resolved_method->GetMethodIndex());
+    return this_object->GetClass()->GetVTableEntry(resolved_method->GetMethodIndex());
   }
 }
 
diff --git a/runtime/interpreter/interpreter_common.h b/runtime/interpreter/interpreter_common.h
index 1bcd27e..5a1d01e 100644
--- a/runtime/interpreter/interpreter_common.h
+++ b/runtime/interpreter/interpreter_common.h
@@ -140,7 +140,8 @@ static inline bool DoInvokeVirtualQuick(Thread* self, ShadowFrame& shadow_frame,
     return false;
   }
   const uint32_t vtable_idx = (is_range) ? inst->VRegB_3rc() : inst->VRegB_35c();
-  ArtMethod* const method = receiver->GetClass()->GetVTable()->GetWithoutChecks(vtable_idx);
+  CHECK(receiver->GetClass()->ShouldHaveEmbeddedImtAndVTable());
+  ArtMethod* const method = receiver->GetClass()->GetEmbeddedVTableEntry(vtable_idx);
   if (UNLIKELY(method == nullptr)) {
     CHECK(self->IsExceptionPending());
     result->SetJ(0);
diff --git a/runtime/mirror/art_method.cc b/runtime/mirror/art_method.cc
index 167f848..211ba1d 100644
--- a/runtime/mirror/art_method.cc
+++ b/runtime/mirror/art_method.cc
@@ -130,12 +130,11 @@ ArtMethod* ArtMethod::FindOverriddenMethod() {
   Class* declaring_class = GetDeclaringClass();
   Class* super_class = declaring_class->GetSuperClass();
   uint16_t method_index = GetMethodIndex();
-  ObjectArray<ArtMethod>* super_class_vtable = super_class->GetVTable();
   ArtMethod* result = NULL;
   // Did this method override a super class method? If so load the result from the super class'
   // vtable
-  if (super_class_vtable != NULL && method_index < super_class_vtable->GetLength()) {
-    result = super_class_vtable->Get(method_index);
+  if (super_class->HasVTable() && method_index < super_class->GetVTableLength()) {
+    result = super_class->GetVTableEntry(method_index);
   } else {
     // Method didn't override superclass method so search interfaces
     if (IsProxyMethod()) {
diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index 329a984..c3754d7 100644
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -161,6 +161,37 @@ inline void Class::SetEmbeddedImTableEntry(uint32_t i, ArtMethod* method) {
   CHECK(method == GetImTable()->Get(i));
 }
 
+inline bool Class::HasVTable() {
+  return (GetVTable() != nullptr) || ShouldHaveEmbeddedImtAndVTable();
+}
+
+inline int32_t Class::GetVTableLength() {
+  if (ShouldHaveEmbeddedImtAndVTable()) {
+    return GetEmbeddedVTableLength();
+  }
+  return (GetVTable() != nullptr) ? GetVTable()->GetLength() : 0;
+}
+
+inline ArtMethod* Class::GetVTableEntry(uint32_t i) {
+  if (ShouldHaveEmbeddedImtAndVTable()) {
+    return GetEmbeddedVTableEntry(i);
+  }
+  return (GetVTable() != nullptr) ? GetVTable()->Get(i) : nullptr;
+}
+
+inline int32_t Class::GetEmbeddedVTableLength() {
+  return GetField32(EmbeddedVTableLengthOffset());
+}
+
+inline void Class::SetEmbeddedVTableLength(int32_t len) {
+  SetField32<false>(EmbeddedVTableLengthOffset(), len);
+}
+
+inline ArtMethod* Class::GetEmbeddedVTableEntry(uint32_t i) {
+  uint32_t offset = EmbeddedVTableOffset().Uint32Value() + i * sizeof(VTableEntry);
+  return GetFieldObject<mirror::ArtMethod>(MemberOffset(offset));
+}
+
 inline void Class::SetEmbeddedVTableEntry(uint32_t i, ArtMethod* method) {
   uint32_t offset = EmbeddedVTableOffset().Uint32Value() + i * sizeof(VTableEntry);
   SetFieldObject<false>(MemberOffset(offset), method);
@@ -340,12 +371,12 @@ inline ArtMethod* Class::FindVirtualMethodForVirtual(ArtMethod* method) {
   DCHECK(!method->GetDeclaringClass()->IsInterface() || method->IsMiranda());
   // The argument method may from a super class.
   // Use the index to a potentially overridden one for this instance's class.
-  return GetVTable()->Get(method->GetMethodIndex());
+  return GetVTableEntry(method->GetMethodIndex());
 }
 
 inline ArtMethod* Class::FindVirtualMethodForSuper(ArtMethod* method) {
   DCHECK(!method->GetDeclaringClass()->IsInterface());
-  return GetSuperClass()->GetVTable()->Get(method->GetMethodIndex());
+  return GetSuperClass()->GetVTableEntry(method->GetMethodIndex());
 }
 
 inline ArtMethod* Class::FindVirtualMethodForVirtualOrInterface(ArtMethod* method) {
@@ -534,13 +565,19 @@ inline uint32_t Class::ComputeClassSize(bool has_embedded_tables,
   if (has_embedded_tables) {
     uint32_t embedded_imt_size = kImtSize * sizeof(ImTableEntry);
     uint32_t embedded_vtable_size = num_vtable_entries * sizeof(VTableEntry);
-    size += embedded_imt_size + embedded_vtable_size;
+    size += embedded_imt_size +
+            sizeof(int32_t) /* vtable len */ +
+            embedded_vtable_size;
   }
   // Space used by reference statics.
   size +=  num_ref_static_fields * sizeof(HeapReference<Object>);
   // Possible pad for alignment.
-  if (((size & 7) != 0) && (num_64bit_static_fields > 0) && (num_32bit_static_fields == 0)) {
+  if (((size & 7) != 0) && (num_64bit_static_fields > 0)) {
     size += sizeof(uint32_t);
+    if (num_32bit_static_fields != 0) {
+      // Shuffle one 32 bit static field forward.
+      num_32bit_static_fields--;
+    }
   }
   // Space used for primitive static fields.
   size += (num_32bit_static_fields * sizeof(uint32_t)) +
@@ -574,7 +611,10 @@ inline void Class::VisitEmbeddedImtAndVTable(const Visitor& visitor) {
     pos += sizeof(ImTableEntry);
   }
 
-  count = ((GetVTable() != NULL) ? GetVTable()->GetLength() : 0);
+  // Skip vtable length.
+  pos += sizeof(int32_t);
+
+  count = GetEmbeddedVTableLength();
   for (size_t i = 0; i < count; ++i) {
     MemberOffset offset = MemberOffset(pos);
     visitor(this, offset, true);
diff --git a/runtime/mirror/class.cc b/runtime/mirror/class.cc
index 189c537..a218b1c 100644
--- a/runtime/mirror/class.cc
+++ b/runtime/mirror/class.cc
@@ -827,10 +827,18 @@ void Class::PopulateEmbeddedImtAndVTable() SHARED_LOCKS_REQUIRED(Locks::mutator_
   }
 
   table = GetVTableDuringLinking();
-  CHECK(table != nullptr);
+  CHECK(table != nullptr) << PrettyClass(this);
+  SetEmbeddedVTableLength(table->GetLength());
   for (int32_t i = 0; i < table->GetLength(); i++) {
     SetEmbeddedVTableEntry(i, table->Get(i));
   }
+
+  SetImTable(nullptr);
+  // Keep java.lang.Object class's vtable around for since it's easier
+  // to be reused by array classes during their linking.
+  if (!IsObjectClass()) {
+    SetVTable(nullptr);
+  }
 }
 
 // The pre-fence visitor for Class::CopyOf().
diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index 648bdde..0525abf 100644
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -692,18 +692,34 @@ class MANAGED Class FINAL : public Object {
     return MemberOffset(sizeof(Class));
   }
 
-  static MemberOffset EmbeddedVTableOffset() {
+  static MemberOffset EmbeddedVTableLengthOffset() {
     return MemberOffset(sizeof(Class) + kImtSize * sizeof(mirror::Class::ImTableEntry));
   }
 
+  static MemberOffset EmbeddedVTableOffset() {
+    return MemberOffset(sizeof(Class) + kImtSize * sizeof(mirror::Class::ImTableEntry) + sizeof(int32_t));
+  }
+
   bool ShouldHaveEmbeddedImtAndVTable() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) {
     return IsInstantiable();
   }
 
+  bool HasVTable() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
   ArtMethod* GetEmbeddedImTableEntry(uint32_t i) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
   void SetEmbeddedImTableEntry(uint32_t i, ArtMethod* method) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
+  int32_t GetVTableLength() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
+  ArtMethod* GetVTableEntry(uint32_t i) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
+  int32_t GetEmbeddedVTableLength() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
+  void SetEmbeddedVTableLength(int32_t len) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
+  ArtMethod* GetEmbeddedVTableEntry(uint32_t i) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
   void SetEmbeddedVTableEntry(uint32_t i, ArtMethod* method) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
   void PopulateEmbeddedImtAndVTable() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc
index fbd710c..82c196d 100644
--- a/runtime/verifier/method_verifier.cc
+++ b/runtime/verifier/method_verifier.cc
@@ -3253,7 +3253,7 @@ mirror::ArtMethod* MethodVerifier::VerifyInvocationArgs(const Instruction* inst,
       return nullptr;
     }
     mirror::Class* super_klass = super.GetClass();
-    if (res_method->GetMethodIndex() >= super_klass->GetVTable()->GetLength()) {
+    if (res_method->GetMethodIndex() >= super_klass->GetVTableLength()) {
       Fail(VERIFY_ERROR_NO_METHOD) << "invalid invoke-super from "
                                    << PrettyMethod(dex_method_idx_, *dex_file_)
                                    << " to super " << super
@@ -3278,20 +3278,21 @@ mirror::ArtMethod* MethodVerifier::GetQuickInvokedMethod(const Instruction* inst
     VLOG(verifier) << "Failed to get mirror::Class* from '" << actual_arg_type << "'";
     return nullptr;
   }
-  mirror::ObjectArray<mirror::ArtMethod>* vtable = nullptr;
   mirror::Class* klass = actual_arg_type.GetClass();
+  mirror::Class* dispatch_class;
   if (klass->IsInterface()) {
     // Derive Object.class from Class.class.getSuperclass().
     mirror::Class* object_klass = klass->GetClass()->GetSuperClass();
     CHECK(object_klass->IsObjectClass());
-    vtable = object_klass->GetVTable();
+    dispatch_class = object_klass;
   } else {
-    vtable = klass->GetVTable();
+    dispatch_class = klass;
   }
-  CHECK(vtable != nullptr) << PrettyDescriptor(klass);
+  CHECK(dispatch_class->HasVTable()) << PrettyDescriptor(dispatch_class);
   uint16_t vtable_index = is_range ? inst->VRegB_3rc() : inst->VRegB_35c();
-  CHECK_LT(static_cast<int32_t>(vtable_index), vtable->GetLength()) << PrettyDescriptor(klass);
-  mirror::ArtMethod* res_method = vtable->Get(vtable_index);
+  CHECK_LT(static_cast<int32_t>(vtable_index), dispatch_class->GetVTableLength())
+      << PrettyDescriptor(klass);
+  mirror::ArtMethod* res_method = dispatch_class->GetVTableEntry(vtable_index);
   CHECK(!Thread::Current()->IsExceptionPending());
   return res_method;
 }