Write out image bitmap inside of image file.

We now create the image bitmap when we generate the image. The image
bitmap is written after the image inside of the image file. This
speeds up dex2oat by making walking the image during heap creation
unnecessary. This should also help memory pressure by enabling the
image bitmap to be swappable.

Bug: 10432288

Change-Id: Idebf459ed15edbb41a7d9b9b353934155bce2f19

5 files changed, 64 insertions, 59 deletions

diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index 702e162..63b24ff 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -45,11 +45,19 @@ std::string SpaceBitmap::Dump() const {
 }
 
 void SpaceSetMap::Walk(SpaceBitmap::Callback* callback, void* arg) {
-  for (Objects::iterator it = contained_.begin(); it != contained_.end(); ++it) {
-    callback(const_cast<mirror::Object*>(*it), arg);
+  for (const mirror::Object* obj : contained_) {
+    callback(const_cast<mirror::Object*>(obj), arg);
   }
 }
 
+SpaceBitmap* SpaceBitmap::CreateFromMemMap(const std::string& name, MemMap* mem_map,
+                                           byte* heap_begin, size_t heap_capacity) {
+  CHECK(mem_map != nullptr);
+  word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
+  size_t bitmap_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize;
+  return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin);
+}
+
 SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size_t heap_capacity) {
   CHECK(heap_begin != NULL);
   // Round up since heap_capacity is not necessarily a multiple of kAlignment * kBitsPerWord.
@@ -59,8 +67,7 @@ SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size
     LOG(ERROR) << "Failed to allocate bitmap " << name;
     return NULL;
   }
-  word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
-  return new SpaceBitmap(name, mem_map.release(), bitmap_begin, bitmap_size, heap_begin);
+  return CreateFromMemMap(name, mem_map.release(), heap_begin, heap_capacity);
 }
 
 // Clean up any resources associated with the bitmap.
@@ -74,14 +81,11 @@ void SpaceBitmap::SetHeapLimit(uintptr_t new_end) {
   }
   // Not sure if doing this trim is necessary, since nothing past the end of the heap capacity
   // should be marked.
-  // TODO: Fix this code is, it broken and causes rare heap corruption!
-  // mem_map_->Trim(reinterpret_cast<byte*>(heap_begin_ + bitmap_size_));
 }
 
 void SpaceBitmap::Clear() {
   if (bitmap_begin_ != NULL) {
-    // This returns the memory to the system.  Successive page faults
-    // will return zeroed memory.
+    // This returns the memory to the system.  Successive page faults will return zeroed memory.
     int result = madvise(bitmap_begin_, bitmap_size_, MADV_DONTNEED);
     if (result == -1) {
       PLOG(FATAL) << "madvise failed";
diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h
index 26ab1de..f975692 100644
--- a/runtime/gc/accounting/space_bitmap.h
+++ b/runtime/gc/accounting/space_bitmap.h
@@ -48,10 +48,16 @@ class SpaceBitmap {
 
   typedef void SweepCallback(size_t ptr_count, mirror::Object** ptrs, void* arg);
 
-  // Initialize a HeapBitmap so that it points to a bitmap large enough to cover a heap at
+  // Initialize a space bitmap so that it points to a bitmap large enough to cover a heap at
   // heap_begin of heap_capacity bytes, where objects are guaranteed to be kAlignment-aligned.
   static SpaceBitmap* Create(const std::string& name, byte* heap_begin, size_t heap_capacity);
 
+  // Initialize a space bitmap using the provided mem_map as the live bits. Takes ownership of the
+  // mem map. The address range covered starts at heap_begin and is of size equal to heap_capacity.
+  // Objects are kAlignement-aligned.
+  static SpaceBitmap* CreateFromMemMap(const std::string& name, MemMap* mem_map,
+                                       byte* heap_begin, size_t heap_capacity);
+
   ~SpaceBitmap();
 
   // <offset> is the difference from .base to a pointer address.
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index e20c2c5..e0048a0 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -187,14 +187,6 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max
     heap_capacity += continuous_spaces_.back()->AsDlMallocSpace()->NonGrowthLimitCapacity();
   }
 
-  // Mark image objects in the live bitmap.
-  for (const auto& space : continuous_spaces_) {
-    if (space->IsImageSpace()) {
-      space::ImageSpace* image_space = space->AsImageSpace();
-      image_space->RecordImageAllocations(image_space->GetLiveBitmap());
-    }
-  }
-
   // Allocate the card table.
   card_table_.reset(accounting::CardTable::Create(heap_begin, heap_capacity));
   CHECK(card_table_.get() != NULL) << "Failed to create card table";
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index f959cff..e2e5bf7 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -35,15 +35,13 @@ namespace art {
 namespace gc {
 namespace space {
 
-size_t ImageSpace::bitmap_index_ = 0;
-
-ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map)
-: MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) {
-  const size_t bitmap_index = bitmap_index_++;
-  live_bitmap_.reset(accounting::SpaceBitmap::Create(
-      StringPrintf("imagespace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
-      Begin(), Capacity()));
-  DCHECK(live_bitmap_.get() != NULL) << "could not create imagespace live bitmap #" << bitmap_index;
+AtomicInteger ImageSpace::bitmap_index_(0);
+
+ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map,
+                       accounting::SpaceBitmap* live_bitmap)
+    : MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) {
+  DCHECK(live_bitmap != NULL);
+  live_bitmap_.reset(live_bitmap);
 }
 
 static bool GenerateImage(const std::string& image_file_name) {
@@ -151,6 +149,17 @@ ImageSpace* ImageSpace::Create(const std::string& original_image_file_name) {
   return space::ImageSpace::Init(image_file_name, true);
 }
 
+void ImageSpace::VerifyImageAllocations() {
+  byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
+  while (current < End()) {
+    DCHECK_ALIGNED(current, kObjectAlignment);
+    const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current);
+    CHECK(live_bitmap_->Test(obj));
+    CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class";
+    current += RoundUp(obj->SizeOf(), kObjectAlignment);
+  }
+}
+
 ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_oat_file) {
   CHECK(!image_file_name.empty());
 
@@ -171,8 +180,10 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o
     LOG(ERROR) << "Invalid image header " << image_file_name;
     return NULL;
   }
+
+  // Note: The image header is part of the image due to mmap page alignment required of offset.
   UniquePtr<MemMap> map(MemMap::MapFileAtAddress(image_header.GetImageBegin(),
-                                                 file->GetLength(),
+                                                 image_header.GetImageSize(),
                                                  PROT_READ | PROT_WRITE,
                                                  MAP_PRIVATE | MAP_FIXED,
                                                  file->Fd(),
@@ -185,6 +196,20 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o
   CHECK_EQ(image_header.GetImageBegin(), map->Begin());
   DCHECK_EQ(0, memcmp(&image_header, map->Begin(), sizeof(ImageHeader)));
 
+  UniquePtr<MemMap> image_map(MemMap::MapFileAtAddress(nullptr, image_header.GetImageBitmapSize(),
+                                                       PROT_READ, MAP_PRIVATE,
+                                                       file->Fd(), image_header.GetBitmapOffset(),
+                                                       false));
+  CHECK(image_map.get() != nullptr) << "failed to map image bitmap";
+  size_t bitmap_index = bitmap_index_.fetch_add(1);
+  std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_file_name.c_str(),
+                                       bitmap_index));
+  UniquePtr<accounting::SpaceBitmap> bitmap(
+      accounting::SpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(),
+                                                reinterpret_cast<byte*>(map->Begin()),
+                                                map->Size()));
+  CHECK(bitmap.get() != nullptr) << "could not create " << bitmap_name;
+
   Runtime* runtime = Runtime::Current();
   mirror::Object* resolution_method = image_header.GetImageRoot(ImageHeader::kResolutionMethod);
   runtime->SetResolutionMethod(down_cast<mirror::ArtMethod*>(resolution_method));
@@ -196,7 +221,10 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o
   callee_save_method = image_header.GetImageRoot(ImageHeader::kRefsAndArgsSaveMethod);
   runtime->SetCalleeSaveMethod(down_cast<mirror::ArtMethod*>(callee_save_method), Runtime::kRefsAndArgs);
 
-  UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release()));
+  UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release(), bitmap.release()));
+  if (kIsDebugBuild) {
+    space->VerifyImageAllocations();
+  }
 
   space->oat_file_.reset(space->OpenOatFile());
   if (space->oat_file_.get() == NULL) {
@@ -245,9 +273,7 @@ OatFile* ImageSpace::OpenOatFile() const {
 
 bool ImageSpace::ValidateOatFile() const {
   CHECK(oat_file_.get() != NULL);
-  std::vector<const OatFile::OatDexFile*> oat_dex_files = oat_file_->GetOatDexFiles();
-  for (size_t i = 0; i < oat_dex_files.size(); i++) {
-    const OatFile::OatDexFile* oat_dex_file = oat_dex_files[i];
+  for (const OatFile::OatDexFile* oat_dex_file : oat_file_->GetOatDexFiles()) {
     const std::string& dex_file_location = oat_dex_file->GetDexFileLocation();
     uint32_t dex_file_location_checksum;
     if (!DexFile::GetChecksum(dex_file_location.c_str(), dex_file_location_checksum)) {
@@ -270,28 +296,6 @@ OatFile& ImageSpace::ReleaseOatFile() {
   return *oat_file_.release();
 }
 
-void ImageSpace::RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const {
-  uint64_t start_time = 0;
-  if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
-    LOG(INFO) << "ImageSpace::RecordImageAllocations entering";
-    start_time = NanoTime();
-  }
-  DCHECK(!Runtime::Current()->IsStarted());
-  CHECK(live_bitmap != NULL);
-  byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
-  byte* end = End();
-  while (current < end) {
-    DCHECK_ALIGNED(current, kObjectAlignment);
-    const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current);
-    live_bitmap->Set(obj);
-    current += RoundUp(obj->SizeOf(), kObjectAlignment);
-  }
-  if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
-    LOG(INFO) << "ImageSpace::RecordImageAllocations exiting ("
-        << PrettyDuration(NanoTime() - start_time) << ")";
-  }
-}
-
 void ImageSpace::Dump(std::ostream& os) const {
   os << GetType()
       << "begin=" << reinterpret_cast<void*>(Begin())
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index bdda9fa..381a98e 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -53,6 +53,9 @@ class ImageSpace : public MemMapSpace {
   OatFile& ReleaseOatFile()
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
+  void VerifyImageAllocations()
+      SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
   const ImageHeader& GetImageHeader() const {
     return *reinterpret_cast<ImageHeader*>(Begin());
   }
@@ -61,10 +64,6 @@ class ImageSpace : public MemMapSpace {
     return GetName();
   }
 
-  // Mark the objects defined in this space in the given live bitmap
-  void RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const
-      SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
-
   accounting::SpaceBitmap* GetLiveBitmap() const {
     return live_bitmap_.get();
   }
@@ -96,11 +95,11 @@ class ImageSpace : public MemMapSpace {
 
   friend class Space;
 
-  static size_t bitmap_index_;
+  static AtomicInteger bitmap_index_;
 
   UniquePtr<accounting::SpaceBitmap> live_bitmap_;
 
-  ImageSpace(const std::string& name, MemMap* mem_map);
+  ImageSpace(const std::string& name, MemMap* mem_map, accounting::SpaceBitmap* live_bitmap);
 
   // The OatFile associated with the image during early startup to
   // reserve space contiguous to the image. It is later released to