diff options
54 files changed, 654 insertions, 588 deletions
diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 0405198..f76587a 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -235,8 +235,8 @@ bool ImageWriter::AllocMemory() { } // Create the image bitmap. - image_bitmap_.reset(gc::accounting::SpaceBitmap::Create("image bitmap", image_->Begin(), - length)); + image_bitmap_.reset(gc::accounting::ContinuousSpaceBitmap::Create("image bitmap", image_->Begin(), + length)); if (image_bitmap_.get() == nullptr) { LOG(ERROR) << "Failed to allocate memory for image bitmap"; return false; @@ -525,7 +525,7 @@ void ImageWriter::CalculateNewObjectOffsets(size_t oat_loaded_size, size_t oat_d // Return to write header at start of image with future location of image_roots. At this point, // image_end_ is the size of the image (excluding bitmaps). - const size_t heap_bytes_per_bitmap_byte = kBitsPerByte * gc::accounting::SpaceBitmap::kAlignment; + const size_t heap_bytes_per_bitmap_byte = kBitsPerByte * kObjectAlignment; const size_t bitmap_bytes = RoundUp(image_end_, heap_bytes_per_bitmap_byte) / heap_bytes_per_bitmap_byte; ImageHeader image_header(PointerToLowMemUInt32(image_begin_), diff --git a/compiler/image_writer.h b/compiler/image_writer.h index 92b24f6..ee241cb 100644 --- a/compiler/image_writer.h +++ b/compiler/image_writer.h @@ -173,7 +173,7 @@ class ImageWriter { const byte* oat_data_begin_; // Image bitmap which lets us know where the objects inside of the image reside. - UniquePtr<gc::accounting::SpaceBitmap> image_bitmap_; + UniquePtr<gc::accounting::ContinuousSpaceBitmap> image_bitmap_; // Offset from oat_data_begin_ to the stubs. uint32_t interpreter_to_interpreter_bridge_offset_; diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index 336a0cc..fd2cfeb 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -561,8 +561,8 @@ DEFINE_FUNCTION art_quick_unlock_object jz .Lslow_unlock movl LOCK_WORD_OFFSET(%eax), %ecx // ecx := lock word movl %fs:THREAD_ID_OFFSET, %edx // edx := thread id - test %ecx, %ecx - jb .Lslow_unlock // lock word contains a monitor + test LITERAL(0xC0000000), %ecx + jnz .Lslow_unlock // lock word contains a monitor cmpw %cx, %dx // does the thread id match? jne .Lslow_unlock cmpl LITERAL(65536), %ecx diff --git a/runtime/debugger.cc b/runtime/debugger.cc index 6dd2493..c52a588 100644 --- a/runtime/debugger.cc +++ b/runtime/debugger.cc @@ -245,6 +245,14 @@ void DebugInvokeReq::VisitRoots(RootCallback* callback, void* arg, uint32_t tid, } } +void DebugInvokeReq::Clear() { + invoke_needed = false; + receiver = nullptr; + thread = nullptr; + klass = nullptr; + method = nullptr; +} + void SingleStepControl::VisitRoots(RootCallback* callback, void* arg, uint32_t tid, RootType root_type) { if (method != nullptr) { @@ -252,6 +260,16 @@ void SingleStepControl::VisitRoots(RootCallback* callback, void* arg, uint32_t t } } +bool SingleStepControl::ContainsDexPc(uint32_t dex_pc) const { + return dex_pcs.find(dex_pc) == dex_pcs.end(); +} + +void SingleStepControl::Clear() { + is_active = false; + method = nullptr; + dex_pcs.clear(); +} + void DeoptimizationRequest::VisitRoots(RootCallback* callback, void* arg) { if (method != nullptr) { callback(reinterpret_cast<mirror::Object**>(&method), arg, 0, kRootDebugger); @@ -2591,7 +2609,7 @@ void Dbg::UpdateDebugger(Thread* thread, mirror::Object* this_object, } else if (single_step_control->step_size == JDWP::SS_MIN) { event_flags |= kSingleStep; VLOG(jdwp) << "SS new instruction"; - } else if (single_step_control->dex_pcs.find(dex_pc) == single_step_control->dex_pcs.end()) { + } else if (single_step_control->ContainsDexPc(dex_pc)) { event_flags |= kSingleStep; VLOG(jdwp) << "SS new line"; } @@ -2613,7 +2631,7 @@ void Dbg::UpdateDebugger(Thread* thread, mirror::Object* this_object, if (single_step_control->step_size == JDWP::SS_MIN) { event_flags |= kSingleStep; VLOG(jdwp) << "SS new instruction"; - } else if (single_step_control->dex_pcs.find(dex_pc) == single_step_control->dex_pcs.end()) { + } else if (single_step_control->ContainsDexPc(dex_pc)) { event_flags |= kSingleStep; VLOG(jdwp) << "SS new line"; } @@ -2980,8 +2998,9 @@ JDWP::JdwpError Dbg::ConfigureStep(JDWP::ObjectId thread_id, JDWP::JdwpStepSize // struct DebugCallbackContext { - explicit DebugCallbackContext(SingleStepControl* single_step_control, int32_t line_number) - : single_step_control_(single_step_control), line_number_(line_number), + explicit DebugCallbackContext(SingleStepControl* single_step_control, int32_t line_number, + const DexFile::CodeItem* code_item) + : single_step_control_(single_step_control), line_number_(line_number), code_item_(code_item), last_pc_valid(false), last_pc(0) { } @@ -3008,7 +3027,7 @@ JDWP::JdwpError Dbg::ConfigureStep(JDWP::ObjectId thread_id, JDWP::JdwpStepSize ~DebugCallbackContext() { // If the line number was the last in the position table... if (last_pc_valid) { - size_t end = MethodHelper(single_step_control_->method).GetCodeItem()->insns_size_in_code_units_; + size_t end = code_item_->insns_size_in_code_units_; for (uint32_t dex_pc = last_pc; dex_pc < end; ++dex_pc) { single_step_control_->dex_pcs.insert(dex_pc); } @@ -3017,15 +3036,17 @@ JDWP::JdwpError Dbg::ConfigureStep(JDWP::ObjectId thread_id, JDWP::JdwpStepSize SingleStepControl* const single_step_control_; const int32_t line_number_; + const DexFile::CodeItem* const code_item_; bool last_pc_valid; uint32_t last_pc; }; single_step_control->dex_pcs.clear(); mirror::ArtMethod* m = single_step_control->method; if (!m->IsNative()) { - DebugCallbackContext context(single_step_control, line_number); MethodHelper mh(m); - mh.GetDexFile().DecodeDebugInfo(mh.GetCodeItem(), m->IsStatic(), m->GetDexMethodIndex(), + const DexFile::CodeItem* const code_item = mh.GetCodeItem(); + DebugCallbackContext context(single_step_control, line_number, code_item); + mh.GetDexFile().DecodeDebugInfo(code_item, m->IsStatic(), m->GetDexMethodIndex(), DebugCallbackContext::Callback, NULL, &context); } @@ -3045,8 +3066,8 @@ JDWP::JdwpError Dbg::ConfigureStep(JDWP::ObjectId thread_id, JDWP::JdwpStepSize VLOG(jdwp) << "Single-step current line: " << line_number; VLOG(jdwp) << "Single-step current stack depth: " << single_step_control->stack_depth; VLOG(jdwp) << "Single-step dex_pc values:"; - for (std::set<uint32_t>::iterator it = single_step_control->dex_pcs.begin(); it != single_step_control->dex_pcs.end(); ++it) { - VLOG(jdwp) << StringPrintf(" %#x", *it); + for (uint32_t dex_pc : single_step_control->dex_pcs) { + VLOG(jdwp) << StringPrintf(" %#x", dex_pc); } } @@ -3061,8 +3082,7 @@ void Dbg::UnconfigureStep(JDWP::ObjectId thread_id) { if (error == JDWP::ERR_NONE) { SingleStepControl* single_step_control = thread->GetSingleStepControl(); DCHECK(single_step_control != nullptr); - single_step_control->is_active = false; - single_step_control->dex_pcs.clear(); + single_step_control->Clear(); } } diff --git a/runtime/debugger.h b/runtime/debugger.h index 6126ddc..b3e94c3 100644 --- a/runtime/debugger.h +++ b/runtime/debugger.h @@ -86,6 +86,8 @@ struct DebugInvokeReq { void VisitRoots(RootCallback* callback, void* arg, uint32_t tid, RootType root_type) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + void Clear(); + private: DISALLOW_COPY_AND_ASSIGN(DebugInvokeReq); }; @@ -119,6 +121,10 @@ struct SingleStepControl { void VisitRoots(RootCallback* callback, void* arg, uint32_t tid, RootType root_type) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + bool ContainsDexPc(uint32_t dex_pc) const; + + void Clear(); + private: DISALLOW_COPY_AND_ASSIGN(SingleStepControl); }; diff --git a/runtime/gc/accounting/card_table-inl.h b/runtime/gc/accounting/card_table-inl.h index 564168e..a1d001e 100644 --- a/runtime/gc/accounting/card_table-inl.h +++ b/runtime/gc/accounting/card_table-inl.h @@ -43,7 +43,7 @@ static inline bool byte_cas(byte old_value, byte new_value, byte* address) { } template <typename Visitor> -inline size_t CardTable::Scan(SpaceBitmap* bitmap, byte* scan_begin, byte* scan_end, +inline size_t CardTable::Scan(ContinuousSpaceBitmap* bitmap, byte* scan_begin, byte* scan_end, const Visitor& visitor, const byte minimum_age) const { DCHECK(bitmap->HasAddress(scan_begin)); DCHECK(bitmap->HasAddress(scan_end - 1)); // scan_end is the byte after the last byte we scan. diff --git a/runtime/gc/accounting/card_table.h b/runtime/gc/accounting/card_table.h index 8b7bfd3..8d5dc07 100644 --- a/runtime/gc/accounting/card_table.h +++ b/runtime/gc/accounting/card_table.h @@ -38,7 +38,7 @@ class Heap; namespace accounting { -class SpaceBitmap; +template<size_t kAlignment> class SpaceBitmap; // Maintain a card table from the the write barrier. All writes of // non-NULL values to heap addresses should go through an entry in @@ -102,7 +102,8 @@ class CardTable { // For every dirty at least minumum age between begin and end invoke the visitor with the // specified argument. Returns how many cards the visitor was run on. template <typename Visitor> - size_t Scan(SpaceBitmap* bitmap, byte* scan_begin, byte* scan_end, const Visitor& visitor, + size_t Scan(SpaceBitmap<kObjectAlignment>* bitmap, byte* scan_begin, byte* scan_end, + const Visitor& visitor, const byte minimum_age = kCardDirty) const EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); diff --git a/runtime/gc/accounting/heap_bitmap-inl.h b/runtime/gc/accounting/heap_bitmap-inl.h index 04e85d2..ed7b427 100644 --- a/runtime/gc/accounting/heap_bitmap-inl.h +++ b/runtime/gc/accounting/heap_bitmap-inl.h @@ -37,16 +37,16 @@ inline void HeapBitmap::Visit(const Visitor& visitor) { } inline bool HeapBitmap::Test(const mirror::Object* obj) { - SpaceBitmap* bitmap = GetContinuousSpaceBitmap(obj); + ContinuousSpaceBitmap* bitmap = GetContinuousSpaceBitmap(obj); if (LIKELY(bitmap != nullptr)) { return bitmap->Test(obj); } else { - return GetDiscontinuousSpaceObjectSet(obj) != NULL; + return GetDiscontinuousSpaceObjectSet(obj) != nullptr; } } inline void HeapBitmap::Clear(const mirror::Object* obj) { - SpaceBitmap* bitmap = GetContinuousSpaceBitmap(obj); + ContinuousSpaceBitmap* bitmap = GetContinuousSpaceBitmap(obj); if (LIKELY(bitmap != nullptr)) { bitmap->Clear(obj); } else { @@ -57,7 +57,7 @@ inline void HeapBitmap::Clear(const mirror::Object* obj) { } inline void HeapBitmap::Set(const mirror::Object* obj) { - SpaceBitmap* bitmap = GetContinuousSpaceBitmap(obj); + ContinuousSpaceBitmap* bitmap = GetContinuousSpaceBitmap(obj); if (LIKELY(bitmap != NULL)) { bitmap->Set(obj); } else { @@ -67,7 +67,7 @@ inline void HeapBitmap::Set(const mirror::Object* obj) { } } -inline SpaceBitmap* HeapBitmap::GetContinuousSpaceBitmap(const mirror::Object* obj) const { +inline ContinuousSpaceBitmap* HeapBitmap::GetContinuousSpaceBitmap(const mirror::Object* obj) const { for (const auto& bitmap : continuous_space_bitmaps_) { if (bitmap->HasAddress(obj)) { return bitmap; diff --git a/runtime/gc/accounting/heap_bitmap.cc b/runtime/gc/accounting/heap_bitmap.cc index f94cf24..1db886c 100644 --- a/runtime/gc/accounting/heap_bitmap.cc +++ b/runtime/gc/accounting/heap_bitmap.cc @@ -16,13 +16,15 @@ #include "heap_bitmap.h" +#include "gc/accounting/space_bitmap-inl.h" #include "gc/space/space.h" namespace art { namespace gc { namespace accounting { -void HeapBitmap::ReplaceBitmap(SpaceBitmap* old_bitmap, SpaceBitmap* new_bitmap) { +void HeapBitmap::ReplaceBitmap(ContinuousSpaceBitmap* old_bitmap, + ContinuousSpaceBitmap* new_bitmap) { for (auto& bitmap : continuous_space_bitmaps_) { if (bitmap == old_bitmap) { bitmap = new_bitmap; @@ -42,7 +44,7 @@ void HeapBitmap::ReplaceObjectSet(ObjectSet* old_set, ObjectSet* new_set) { LOG(FATAL) << "object set " << static_cast<const void*>(old_set) << " not found"; } -void HeapBitmap::AddContinuousSpaceBitmap(accounting::SpaceBitmap* bitmap) { +void HeapBitmap::AddContinuousSpaceBitmap(accounting::ContinuousSpaceBitmap* bitmap) { DCHECK(bitmap != NULL); // Check for interval overlap. @@ -55,14 +57,14 @@ void HeapBitmap::AddContinuousSpaceBitmap(accounting::SpaceBitmap* bitmap) { continuous_space_bitmaps_.push_back(bitmap); } -void HeapBitmap::RemoveContinuousSpaceBitmap(accounting::SpaceBitmap* bitmap) { +void HeapBitmap::RemoveContinuousSpaceBitmap(accounting::ContinuousSpaceBitmap* bitmap) { auto it = std::find(continuous_space_bitmaps_.begin(), continuous_space_bitmaps_.end(), bitmap); DCHECK(it != continuous_space_bitmaps_.end()); continuous_space_bitmaps_.erase(it); } void HeapBitmap::AddDiscontinuousObjectSet(ObjectSet* set) { - DCHECK(set != NULL); + DCHECK(set != nullptr); discontinuous_space_sets_.push_back(set); } diff --git a/runtime/gc/accounting/heap_bitmap.h b/runtime/gc/accounting/heap_bitmap.h index f729c0e..61a2429 100644 --- a/runtime/gc/accounting/heap_bitmap.h +++ b/runtime/gc/accounting/heap_bitmap.h @@ -34,7 +34,7 @@ class HeapBitmap { bool Test(const mirror::Object* obj) SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_); void Clear(const mirror::Object* obj) EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_); void Set(const mirror::Object* obj) EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_); - SpaceBitmap* GetContinuousSpaceBitmap(const mirror::Object* obj) const; + ContinuousSpaceBitmap* GetContinuousSpaceBitmap(const mirror::Object* obj) const; ObjectSet* GetDiscontinuousSpaceObjectSet(const mirror::Object* obj) const; void Walk(ObjectCallback* callback, void* arg) @@ -46,7 +46,7 @@ class HeapBitmap { SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); // Find and replace a bitmap pointer, this is used by for the bitmap swapping in the GC. - void ReplaceBitmap(SpaceBitmap* old_bitmap, SpaceBitmap* new_bitmap) + void ReplaceBitmap(ContinuousSpaceBitmap* old_bitmap, ContinuousSpaceBitmap* new_bitmap) EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_); // Find and replace a object set pointer, this is used by for the bitmap swapping in the GC. @@ -58,13 +58,14 @@ class HeapBitmap { private: const Heap* const heap_; - void AddContinuousSpaceBitmap(SpaceBitmap* bitmap); - void RemoveContinuousSpaceBitmap(SpaceBitmap* bitmap); + void AddContinuousSpaceBitmap(ContinuousSpaceBitmap* bitmap); + void RemoveContinuousSpaceBitmap(ContinuousSpaceBitmap* bitmap); void AddDiscontinuousObjectSet(ObjectSet* set); void RemoveDiscontinuousObjectSet(ObjectSet* set); // Bitmaps covering continuous spaces. - std::vector<SpaceBitmap*, GcAllocator<SpaceBitmap*>> continuous_space_bitmaps_; + std::vector<ContinuousSpaceBitmap*, GcAllocator<ContinuousSpaceBitmap*>> + continuous_space_bitmaps_; // Sets covering discontinuous spaces. std::vector<ObjectSet*, GcAllocator<ObjectSet*>> discontinuous_space_sets_; diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc index 34ca654..d744dee 100644 --- a/runtime/gc/accounting/mod_union_table.cc +++ b/runtime/gc/accounting/mod_union_table.cc @@ -19,6 +19,7 @@ #include "base/stl_util.h" #include "card_table-inl.h" #include "heap_bitmap.h" +#include "gc/accounting/space_bitmap-inl.h" #include "gc/collector/mark_sweep.h" #include "gc/collector/mark_sweep-inl.h" #include "gc/heap.h" @@ -222,7 +223,7 @@ void ModUnionTableReferenceCache::Verify() { // Check the references of each clean card which is also in the mod union table. CardTable* card_table = heap_->GetCardTable(); - SpaceBitmap* live_bitmap = space_->GetLiveBitmap(); + ContinuousSpaceBitmap* live_bitmap = space_->GetLiveBitmap(); for (const auto& ref_pair : references_) { const byte* card = ref_pair.first; if (*card == CardTable::kCardClean) { @@ -272,7 +273,7 @@ void ModUnionTableReferenceCache::UpdateAndMarkReferences(MarkHeapReferenceCallb uintptr_t end = start + CardTable::kCardSize; auto* space = heap_->FindContinuousSpaceFromObject(reinterpret_cast<Object*>(start), false); DCHECK(space != nullptr); - SpaceBitmap* live_bitmap = space->GetLiveBitmap(); + ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); live_bitmap->VisitMarkedRange(start, end, add_visitor); // Update the corresponding references for the card. @@ -312,7 +313,7 @@ void ModUnionTableCardCache::UpdateAndMarkReferences(MarkHeapReferenceCallback* void* arg) { CardTable* card_table = heap_->GetCardTable(); ModUnionScanImageRootVisitor scan_visitor(callback, arg); - SpaceBitmap* bitmap = space_->GetLiveBitmap(); + ContinuousSpaceBitmap* bitmap = space_->GetLiveBitmap(); for (const byte* card_addr : cleared_cards_) { uintptr_t start = reinterpret_cast<uintptr_t>(card_table->AddrFromCard(card_addr)); DCHECK(space_->HasAddress(reinterpret_cast<Object*>(start))); diff --git a/runtime/gc/accounting/mod_union_table.h b/runtime/gc/accounting/mod_union_table.h index c3a90e2..5ae7c77 100644 --- a/runtime/gc/accounting/mod_union_table.h +++ b/runtime/gc/accounting/mod_union_table.h @@ -44,7 +44,6 @@ class Heap; namespace accounting { -class SpaceBitmap; class HeapBitmap; // The mod-union table is the union of modified cards. It is used to allow the card table to be diff --git a/runtime/gc/accounting/remembered_set.cc b/runtime/gc/accounting/remembered_set.cc index 56f7caa..044216e 100644 --- a/runtime/gc/accounting/remembered_set.cc +++ b/runtime/gc/accounting/remembered_set.cc @@ -112,7 +112,7 @@ void RememberedSet::UpdateAndMarkReferences(MarkHeapReferenceCallback* callback, bool contains_reference_to_target_space = false; RememberedSetObjectVisitor obj_visitor(callback, target_space, &contains_reference_to_target_space, arg); - SpaceBitmap* bitmap = space_->GetLiveBitmap(); + ContinuousSpaceBitmap* bitmap = space_->GetLiveBitmap(); CardSet remove_card_set; for (byte* const card_addr : dirty_cards_) { contains_reference_to_target_space = false; diff --git a/runtime/gc/accounting/space_bitmap-inl.h b/runtime/gc/accounting/space_bitmap-inl.h index 880ff1f..ed140e0 100644 --- a/runtime/gc/accounting/space_bitmap-inl.h +++ b/runtime/gc/accounting/space_bitmap-inl.h @@ -17,14 +17,26 @@ #ifndef ART_RUNTIME_GC_ACCOUNTING_SPACE_BITMAP_INL_H_ #define ART_RUNTIME_GC_ACCOUNTING_SPACE_BITMAP_INL_H_ +#include "space_bitmap.h" + #include "base/logging.h" +#include "dex_file-inl.h" +#include "heap_bitmap.h" +#include "mirror/art_field-inl.h" +#include "mirror/class-inl.h" +#include "mirror/object-inl.h" +#include "mirror/object_array-inl.h" +#include "object_utils.h" +#include "space_bitmap-inl.h" +#include "UniquePtr.h" #include "utils.h" namespace art { namespace gc { namespace accounting { -inline bool SpaceBitmap::AtomicTestAndSet(const mirror::Object* obj) { +template<size_t kAlignment> +inline bool SpaceBitmap<kAlignment>::AtomicTestAndSet(const mirror::Object* obj) { uintptr_t addr = reinterpret_cast<uintptr_t>(obj); DCHECK_GE(addr, heap_begin_); const uintptr_t offset = addr - heap_begin_; @@ -45,7 +57,8 @@ inline bool SpaceBitmap::AtomicTestAndSet(const mirror::Object* obj) { return false; } -inline bool SpaceBitmap::Test(const mirror::Object* obj) const { +template<size_t kAlignment> +inline bool SpaceBitmap<kAlignment>::Test(const mirror::Object* obj) const { uintptr_t addr = reinterpret_cast<uintptr_t>(obj); DCHECK(HasAddress(obj)) << obj; DCHECK(bitmap_begin_ != NULL); @@ -54,9 +67,9 @@ inline bool SpaceBitmap::Test(const mirror::Object* obj) const { return (bitmap_begin_[OffsetToIndex(offset)] & OffsetToMask(offset)) != 0; } -template <typename Visitor> -void SpaceBitmap::VisitMarkedRange(uintptr_t visit_begin, uintptr_t visit_end, - const Visitor& visitor) const { +template<size_t kAlignment> template<typename Visitor> +inline void SpaceBitmap<kAlignment>::VisitMarkedRange(uintptr_t visit_begin, uintptr_t visit_end, + const Visitor& visitor) const { DCHECK_LT(visit_begin, visit_end); #if 0 for (uintptr_t i = visit_begin; i < visit_end; i += kAlignment) { @@ -148,7 +161,8 @@ void SpaceBitmap::VisitMarkedRange(uintptr_t visit_begin, uintptr_t visit_end, #endif } -inline bool SpaceBitmap::Modify(const mirror::Object* obj, bool do_set) { +template<size_t kAlignment> template<bool kSetBit> +inline bool SpaceBitmap<kAlignment>::Modify(const mirror::Object* obj) { uintptr_t addr = reinterpret_cast<uintptr_t>(obj); DCHECK_GE(addr, heap_begin_); const uintptr_t offset = addr - heap_begin_; @@ -157,15 +171,24 @@ inline bool SpaceBitmap::Modify(const mirror::Object* obj, bool do_set) { DCHECK_LT(index, bitmap_size_ / kWordSize) << " bitmap_size_ = " << bitmap_size_; uword* address = &bitmap_begin_[index]; uword old_word = *address; - if (do_set) { + if (kSetBit) { *address = old_word | mask; } else { *address = old_word & ~mask; } - DCHECK_EQ(Test(obj), do_set); + DCHECK_EQ(Test(obj), kSetBit); return (old_word & mask) != 0; } +template<size_t kAlignment> +inline std::ostream& operator << (std::ostream& stream, const SpaceBitmap<kAlignment>& bitmap) { + return stream + << bitmap.GetName() << "[" + << "begin=" << reinterpret_cast<const void*>(bitmap.HeapBegin()) + << ",end=" << reinterpret_cast<const void*>(bitmap.HeapLimit()) + << "]"; +} + } // namespace accounting } // namespace gc } // namespace art diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc index 1957c21..7eed05a 100644 --- a/runtime/gc/accounting/space_bitmap.cc +++ b/runtime/gc/accounting/space_bitmap.cc @@ -14,51 +14,24 @@ * limitations under the License. */ -#include "base/logging.h" -#include "dex_file-inl.h" -#include "heap_bitmap.h" -#include "mirror/art_field-inl.h" -#include "mirror/class-inl.h" -#include "mirror/object-inl.h" -#include "mirror/object_array-inl.h" -#include "object_utils.h" #include "space_bitmap-inl.h" -#include "UniquePtr.h" -#include "utils.h" namespace art { namespace gc { namespace accounting { -std::string SpaceBitmap::GetName() const { - return name_; -} - -void SpaceBitmap::SetName(const std::string& name) { - name_ = name; -} - -std::string SpaceBitmap::Dump() const { - return StringPrintf("%s: %p-%p", name_.c_str(), - reinterpret_cast<void*>(HeapBegin()), - reinterpret_cast<void*>(HeapLimit())); -} - -void ObjectSet::Walk(ObjectCallback* callback, void* arg) { - for (const mirror::Object* obj : contained_) { - callback(const_cast<mirror::Object*>(obj), arg); - } -} - -SpaceBitmap* SpaceBitmap::CreateFromMemMap(const std::string& name, MemMap* mem_map, - byte* heap_begin, size_t heap_capacity) { +template<size_t kAlignment> +SpaceBitmap<kAlignment>* SpaceBitmap<kAlignment>::CreateFromMemMap( + const std::string& name, MemMap* mem_map, byte* heap_begin, size_t heap_capacity) { CHECK(mem_map != nullptr); uword* bitmap_begin = reinterpret_cast<uword*>(mem_map->Begin()); size_t bitmap_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize; return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin); } -SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size_t heap_capacity) { +template<size_t kAlignment> +SpaceBitmap<kAlignment>* SpaceBitmap<kAlignment>::Create( + const std::string& name, byte* heap_begin, size_t heap_capacity) { CHECK(heap_begin != NULL); // Round up since heap_capacity is not necessarily a multiple of kAlignment * kBitsPerWord. size_t bitmap_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize; @@ -72,10 +45,8 @@ SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size return CreateFromMemMap(name, mem_map.release(), heap_begin, heap_capacity); } -// Clean up any resources associated with the bitmap. -SpaceBitmap::~SpaceBitmap() {} - -void SpaceBitmap::SetHeapLimit(uintptr_t new_end) { +template<size_t kAlignment> +void SpaceBitmap<kAlignment>::SetHeapLimit(uintptr_t new_end) { DCHECK(IsAligned<kBitsPerWord * kAlignment>(new_end)); size_t new_size = OffsetToIndex(new_end - heap_begin_) * kWordSize; if (new_size < bitmap_size_) { @@ -85,7 +56,8 @@ void SpaceBitmap::SetHeapLimit(uintptr_t new_end) { // should be marked. } -void SpaceBitmap::Clear() { +template<size_t kAlignment> +void SpaceBitmap<kAlignment>::Clear() { if (bitmap_begin_ != NULL) { // This returns the memory to the system. Successive page faults will return zeroed memory. int result = madvise(bitmap_begin_, bitmap_size_, MADV_DONTNEED); @@ -95,14 +67,14 @@ void SpaceBitmap::Clear() { } } -void SpaceBitmap::CopyFrom(SpaceBitmap* source_bitmap) { +template<size_t kAlignment> +inline void SpaceBitmap<kAlignment>::CopyFrom(SpaceBitmap* source_bitmap) { DCHECK_EQ(Size(), source_bitmap->Size()); std::copy(source_bitmap->Begin(), source_bitmap->Begin() + source_bitmap->Size() / kWordSize, Begin()); } -// Visits set bits in address order. The callback is not permitted to -// change the bitmap bits or max during the traversal. -void SpaceBitmap::Walk(ObjectCallback* callback, void* arg) { +template<size_t kAlignment> +inline void SpaceBitmap<kAlignment>::Walk(ObjectCallback* callback, void* arg) { CHECK(bitmap_begin_ != NULL); CHECK(callback != NULL); @@ -122,15 +94,11 @@ void SpaceBitmap::Walk(ObjectCallback* callback, void* arg) { } } -// Walk through the bitmaps in increasing address order, and find the -// object pointers that correspond to garbage objects. Call -// <callback> zero or more times with lists of these object pointers. -// -// The callback is not permitted to increase the max of either bitmap. -void SpaceBitmap::SweepWalk(const SpaceBitmap& live_bitmap, - const SpaceBitmap& mark_bitmap, - uintptr_t sweep_begin, uintptr_t sweep_end, - SpaceBitmap::SweepCallback* callback, void* arg) { +template<size_t kAlignment> +void SpaceBitmap<kAlignment>::SweepWalk(const SpaceBitmap<kAlignment>& live_bitmap, + const SpaceBitmap<kAlignment>& mark_bitmap, + uintptr_t sweep_begin, uintptr_t sweep_end, + SpaceBitmap::SweepCallback* callback, void* arg) { CHECK(live_bitmap.bitmap_begin_ != NULL); CHECK(mark_bitmap.bitmap_begin_ != NULL); CHECK_EQ(live_bitmap.heap_begin_, mark_bitmap.heap_begin_); @@ -174,13 +142,10 @@ void SpaceBitmap::SweepWalk(const SpaceBitmap& live_bitmap, } } -static void WalkFieldsInOrder(SpaceBitmap* visited, ObjectCallback* callback, mirror::Object* obj, - void* arg); - -// Walk instance fields of the given Class. Separate function to allow recursion on the super -// class. -static void WalkInstanceFields(SpaceBitmap* visited, ObjectCallback* callback, mirror::Object* obj, - mirror::Class* klass, void* arg) +template<size_t kAlignment> +void SpaceBitmap<kAlignment>::WalkInstanceFields(SpaceBitmap<kAlignment>* visited, + ObjectCallback* callback, mirror::Object* obj, + mirror::Class* klass, void* arg) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { // Visit fields of parent classes first. mirror::Class* super = klass->GetSuperClass(); @@ -203,10 +168,10 @@ static void WalkInstanceFields(SpaceBitmap* visited, ObjectCallback* callback, m } } -// For an unvisited object, visit it then all its children found via fields. -static void WalkFieldsInOrder(SpaceBitmap* visited, ObjectCallback* callback, mirror::Object* obj, - void* arg) - SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { +template<size_t kAlignment> +void SpaceBitmap<kAlignment>::WalkFieldsInOrder(SpaceBitmap<kAlignment>* visited, + ObjectCallback* callback, + mirror::Object* obj, void* arg) { if (visited->Test(obj)) { return; } @@ -244,14 +209,13 @@ static void WalkFieldsInOrder(SpaceBitmap* visited, ObjectCallback* callback, mi } } -// Visits set bits with an in order traversal. The callback is not permitted to change the bitmap -// bits or max during the traversal. -void SpaceBitmap::InOrderWalk(ObjectCallback* callback, void* arg) { - UniquePtr<SpaceBitmap> visited(Create("bitmap for in-order walk", - reinterpret_cast<byte*>(heap_begin_), - IndexToOffset(bitmap_size_ / kWordSize))); - CHECK(bitmap_begin_ != NULL); - CHECK(callback != NULL); +template<size_t kAlignment> +void SpaceBitmap<kAlignment>::InOrderWalk(ObjectCallback* callback, void* arg) { + UniquePtr<SpaceBitmap<kAlignment>> visited( + Create("bitmap for in-order walk", reinterpret_cast<byte*>(heap_begin_), + IndexToOffset(bitmap_size_ / kWordSize))); + CHECK(bitmap_begin_ != nullptr); + CHECK(callback != nullptr); uintptr_t end = Size() / kWordSize; for (uintptr_t i = 0; i < end; ++i) { // Need uint for unsigned shift. @@ -268,14 +232,15 @@ void SpaceBitmap::InOrderWalk(ObjectCallback* callback, void* arg) { } } -std::ostream& operator << (std::ostream& stream, const SpaceBitmap& bitmap) { - return stream - << bitmap.GetName() << "[" - << "begin=" << reinterpret_cast<const void*>(bitmap.HeapBegin()) - << ",end=" << reinterpret_cast<const void*>(bitmap.HeapLimit()) - << "]"; +void ObjectSet::Walk(ObjectCallback* callback, void* arg) { + for (const mirror::Object* obj : contained_) { + callback(const_cast<mirror::Object*>(obj), arg); + } } +template class SpaceBitmap<kObjectAlignment>; +template class SpaceBitmap<kPageSize>; + } // namespace accounting } // namespace gc } // namespace art diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h index a88f3e4..b90a799 100644 --- a/runtime/gc/accounting/space_bitmap.h +++ b/runtime/gc/accounting/space_bitmap.h @@ -38,11 +38,9 @@ namespace mirror { namespace gc { namespace accounting { +template<size_t kAlignment> class SpaceBitmap { public: - // Alignment of objects within spaces. - static const size_t kAlignment = 8; - typedef void ScanCallback(mirror::Object* obj, void* finger, void* arg); typedef void SweepCallback(size_t ptr_count, mirror::Object** ptrs, void* arg); @@ -57,30 +55,31 @@ class SpaceBitmap { static SpaceBitmap* CreateFromMemMap(const std::string& name, MemMap* mem_map, byte* heap_begin, size_t heap_capacity); - ~SpaceBitmap(); + ~SpaceBitmap() { + } // <offset> is the difference from .base to a pointer address. // <index> is the index of .bits that contains the bit representing // <offset>. - static size_t OffsetToIndex(size_t offset) { + static size_t OffsetToIndex(size_t offset) ALWAYS_INLINE { return offset / kAlignment / kBitsPerWord; } - static uintptr_t IndexToOffset(size_t index) { + static uintptr_t IndexToOffset(size_t index) ALWAYS_INLINE { return static_cast<uintptr_t>(index * kAlignment * kBitsPerWord); } // Bits are packed in the obvious way. - static uword OffsetToMask(uintptr_t offset) { + static uword OffsetToMask(uintptr_t offset) ALWAYS_INLINE { return (static_cast<size_t>(1)) << ((offset / kAlignment) % kBitsPerWord); } - inline bool Set(const mirror::Object* obj) { - return Modify(obj, true); + bool Set(const mirror::Object* obj) ALWAYS_INLINE { + return Modify<true>(obj); } - inline bool Clear(const mirror::Object* obj) { - return Modify(obj, false); + bool Clear(const mirror::Object* obj) ALWAYS_INLINE { + return Modify<false>(obj); } // Returns true if the object was previously marked. @@ -123,20 +122,26 @@ class SpaceBitmap { } } - /** - * Visit the live objects in the range [visit_begin, visit_end). - */ + // Visit the live objects in the range [visit_begin, visit_end). + // TODO: Use lock annotations when clang is fixed. + // EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); template <typename Visitor> void VisitMarkedRange(uintptr_t visit_begin, uintptr_t visit_end, const Visitor& visitor) const - EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) - SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + NO_THREAD_SAFETY_ANALYSIS; + // Visits set bits in address order. The callback is not permitted to change the bitmap bits or + // max during the traversal. void Walk(ObjectCallback* callback, void* arg) SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_); + // Visits set bits with an in order traversal. The callback is not permitted to change the bitmap + // bits or max during the traversal. void InOrderWalk(ObjectCallback* callback, void* arg) SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_, Locks::mutator_lock_); + // Walk through the bitmaps in increasing address order, and find the object pointers that + // correspond to garbage objects. Call <callback> zero or more times with lists of these object + // pointers. The callback is not permitted to increase the max of either bitmap. static void SweepWalk(const SpaceBitmap& live, const SpaceBitmap& mark, uintptr_t base, uintptr_t max, SweepCallback* thunk, void* arg); @@ -169,10 +174,18 @@ class SpaceBitmap { // Set the max address which can covered by the bitmap. void SetHeapLimit(uintptr_t new_end); - std::string GetName() const; - void SetName(const std::string& name); + std::string GetName() const { + return name_; + } - std::string Dump() const; + void SetName(const std::string& name) { + name_ = name; + } + + std::string Dump() const { + return StringPrintf("%s: %p-%p", name_.c_str(), reinterpret_cast<void*>(HeapBegin()), + reinterpret_cast<void*>(HeapLimit())); + } const void* GetObjectWordAddress(const mirror::Object* obj) const { uintptr_t addr = reinterpret_cast<uintptr_t>(obj); @@ -190,7 +203,17 @@ class SpaceBitmap { heap_begin_(reinterpret_cast<uintptr_t>(heap_begin)), name_(name) {} - bool Modify(const mirror::Object* obj, bool do_set); + template<bool kSetBit> + bool Modify(const mirror::Object* obj); + + // For an unvisited object, visit it then all its children found via fields. + static void WalkFieldsInOrder(SpaceBitmap* visited, ObjectCallback* callback, mirror::Object* obj, + void* arg) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + // Walk instance fields of the given Class. Separate function to allow recursion on the super + // class. + static void WalkInstanceFields(SpaceBitmap<kAlignment>* visited, ObjectCallback* callback, + mirror::Object* obj, mirror::Class* klass, void* arg) + SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); // Backing storage for bitmap. UniquePtr<MemMap> mem_map_; @@ -272,7 +295,12 @@ class ObjectSet { Objects contained_; }; -std::ostream& operator << (std::ostream& stream, const SpaceBitmap& bitmap); +typedef SpaceBitmap<kObjectAlignment> ContinuousSpaceBitmap; +// TODO: Replace usage of ObjectSet with LargeObjectBitmap. +typedef SpaceBitmap<kLargeObjectAlignment> LargeObjectBitmap; + +template<size_t kAlignment> +std::ostream& operator << (std::ostream& stream, const SpaceBitmap<kAlignment>& bitmap); } // namespace accounting } // namespace gc diff --git a/runtime/gc/accounting/space_bitmap_test.cc b/runtime/gc/accounting/space_bitmap_test.cc index 68994a8..7c18052 100644 --- a/runtime/gc/accounting/space_bitmap_test.cc +++ b/runtime/gc/accounting/space_bitmap_test.cc @@ -32,14 +32,15 @@ class SpaceBitmapTest : public CommonRuntimeTest {}; TEST_F(SpaceBitmapTest, Init) { byte* heap_begin = reinterpret_cast<byte*>(0x10000000); size_t heap_capacity = 16 * MB; - UniquePtr<SpaceBitmap> space_bitmap(SpaceBitmap::Create("test bitmap", - heap_begin, heap_capacity)); + UniquePtr<ContinuousSpaceBitmap> space_bitmap( + ContinuousSpaceBitmap::Create("test bitmap", heap_begin, heap_capacity)); EXPECT_TRUE(space_bitmap.get() != NULL); } class BitmapVerify { public: - BitmapVerify(SpaceBitmap* bitmap, const mirror::Object* begin, const mirror::Object* end) + BitmapVerify(ContinuousSpaceBitmap* bitmap, const mirror::Object* begin, + const mirror::Object* end) : bitmap_(bitmap), begin_(begin), end_(end) {} @@ -50,7 +51,7 @@ class BitmapVerify { EXPECT_EQ(bitmap_->Test(obj), ((reinterpret_cast<uintptr_t>(obj) & 0xF) != 0)); } - SpaceBitmap* bitmap_; + ContinuousSpaceBitmap* bitmap_; const mirror::Object* begin_; const mirror::Object* end_; }; @@ -59,14 +60,14 @@ TEST_F(SpaceBitmapTest, ScanRange) { byte* heap_begin = reinterpret_cast<byte*>(0x10000000); size_t heap_capacity = 16 * MB; - UniquePtr<SpaceBitmap> space_bitmap(SpaceBitmap::Create("test bitmap", - heap_begin, heap_capacity)); + UniquePtr<ContinuousSpaceBitmap> space_bitmap( + ContinuousSpaceBitmap::Create("test bitmap", heap_begin, heap_capacity)); EXPECT_TRUE(space_bitmap.get() != NULL); // Set all the odd bits in the first BitsPerWord * 3 to one. for (size_t j = 0; j < kBitsPerWord * 3; ++j) { const mirror::Object* obj = - reinterpret_cast<mirror::Object*>(heap_begin + j * SpaceBitmap::kAlignment); + reinterpret_cast<mirror::Object*>(heap_begin + j * kObjectAlignment); if (reinterpret_cast<uintptr_t>(obj) & 0xF) { space_bitmap->Set(obj); } @@ -77,10 +78,10 @@ TEST_F(SpaceBitmapTest, ScanRange) { // words. for (size_t i = 0; i < static_cast<size_t>(kBitsPerWord); ++i) { mirror::Object* start = - reinterpret_cast<mirror::Object*>(heap_begin + i * SpaceBitmap::kAlignment); + reinterpret_cast<mirror::Object*>(heap_begin + i * kObjectAlignment); for (size_t j = 0; j < static_cast<size_t>(kBitsPerWord * 2); ++j) { mirror::Object* end = - reinterpret_cast<mirror::Object*>(heap_begin + (i + j) * SpaceBitmap::kAlignment); + reinterpret_cast<mirror::Object*>(heap_begin + (i + j) * kObjectAlignment); BitmapVerify(space_bitmap.get(), start, end); } } @@ -118,8 +119,8 @@ void compat_test() NO_THREAD_SAFETY_ANALYSIS { for (int i = 0; i < 5 ; ++i) { - UniquePtr<SpaceBitmap> space_bitmap(SpaceBitmap::Create("test bitmap", - heap_begin, heap_capacity)); + UniquePtr<ContinuousSpaceBitmap> space_bitmap( + ContinuousSpaceBitmap::Create("test bitmap", heap_begin, heap_capacity)); for (int j = 0; j < 10000; ++j) { size_t offset = (r.next() % heap_capacity) & ~(0x7); diff --git a/runtime/gc/collector/garbage_collector.cc b/runtime/gc/collector/garbage_collector.cc index a700c73..d99136a 100644 --- a/runtime/gc/collector/garbage_collector.cc +++ b/runtime/gc/collector/garbage_collector.cc @@ -174,8 +174,8 @@ void GarbageCollector::SwapBitmaps() { if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect || (gc_type == kGcTypeFull && space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect)) { - accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); - accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); if (live_bitmap != nullptr && live_bitmap != mark_bitmap) { heap_->GetLiveBitmap()->ReplaceBitmap(live_bitmap, mark_bitmap); heap_->GetMarkBitmap()->ReplaceBitmap(mark_bitmap, live_bitmap); diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc index bb41b57..f07e6f1 100644 --- a/runtime/gc/collector/mark_sweep.cc +++ b/runtime/gc/collector/mark_sweep.cc @@ -123,7 +123,6 @@ void MarkSweep::InitializePhase() { mark_immune_count_ = 0; mark_fastpath_count_ = 0; mark_slowpath_count_ = 0; - FindDefaultSpaceBitmap(); { // TODO: I don't think we should need heap bitmap lock to get the mark bitmap. ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); @@ -293,7 +292,7 @@ void MarkSweep::ReclaimPhase() { void MarkSweep::FindDefaultSpaceBitmap() { TimingLogger::ScopedSplit split("FindDefaultMarkBitmap", &timings_); for (const auto& space : GetHeap()->GetContinuousSpaces()) { - accounting::SpaceBitmap* bitmap = space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* bitmap = space->GetMarkBitmap(); if (bitmap != nullptr && space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect) { current_space_bitmap_ = bitmap; @@ -359,7 +358,7 @@ inline void MarkSweep::MarkObjectNonNull(Object* obj) { } // Try to take advantage of locality of references within a space, failing this find the space // the hard way. - accounting::SpaceBitmap* object_bitmap = current_space_bitmap_; + accounting::ContinuousSpaceBitmap* object_bitmap = current_space_bitmap_; if (UNLIKELY(!object_bitmap->HasAddress(obj))) { object_bitmap = mark_bitmap_->GetContinuousSpaceBitmap(obj); if (kCountMarkedObjects) { @@ -428,9 +427,9 @@ inline bool MarkSweep::MarkObjectParallel(const Object* obj) { } // Try to take advantage of locality of references within a space, failing this find the space // the hard way. - accounting::SpaceBitmap* object_bitmap = current_space_bitmap_; + accounting::ContinuousSpaceBitmap* object_bitmap = current_space_bitmap_; if (UNLIKELY(!object_bitmap->HasAddress(obj))) { - accounting::SpaceBitmap* new_bitmap = mark_bitmap_->GetContinuousSpaceBitmap(obj); + accounting::ContinuousSpaceBitmap* new_bitmap = mark_bitmap_->GetContinuousSpaceBitmap(obj); if (new_bitmap != NULL) { object_bitmap = new_bitmap; } else { @@ -476,7 +475,7 @@ void MarkSweep::VerifyRootCallback(const Object* root, void* arg, size_t vreg, void MarkSweep::VerifyRoot(const Object* root, size_t vreg, const StackVisitor* visitor, RootType root_type) { // See if the root is on any space bitmap. - if (GetHeap()->GetLiveBitmap()->GetContinuousSpaceBitmap(root) == nullptr) { + if (heap_->GetLiveBitmap()->GetContinuousSpaceBitmap(root) == nullptr) { space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); if (!large_object_space->Contains(root)) { LOG(ERROR) << "Found invalid root: " << root << " with type " << root_type; @@ -686,7 +685,8 @@ class MarkStackTask : public Task { class CardScanTask : public MarkStackTask<false> { public: - CardScanTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, accounting::SpaceBitmap* bitmap, + CardScanTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, + accounting::ContinuousSpaceBitmap* bitmap, byte* begin, byte* end, byte minimum_age, size_t mark_stack_size, Object** mark_stack_obj) : MarkStackTask<false>(thread_pool, mark_sweep, mark_stack_size, mark_stack_obj), @@ -697,7 +697,7 @@ class CardScanTask : public MarkStackTask<false> { } protected: - accounting::SpaceBitmap* const bitmap_; + accounting::ContinuousSpaceBitmap* const bitmap_; byte* const begin_; byte* const end_; const byte minimum_age_; @@ -820,7 +820,7 @@ void MarkSweep::ScanGrayObjects(bool paused, byte minimum_age) { class RecursiveMarkTask : public MarkStackTask<false> { public: RecursiveMarkTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, - accounting::SpaceBitmap* bitmap, uintptr_t begin, uintptr_t end) + accounting::ContinuousSpaceBitmap* bitmap, uintptr_t begin, uintptr_t end) : MarkStackTask<false>(thread_pool, mark_sweep, 0, NULL), bitmap_(bitmap), begin_(begin), @@ -828,7 +828,7 @@ class RecursiveMarkTask : public MarkStackTask<false> { } protected: - accounting::SpaceBitmap* const bitmap_; + accounting::ContinuousSpaceBitmap* const bitmap_; const uintptr_t begin_; const uintptr_t end_; @@ -1045,8 +1045,8 @@ void MarkSweep::SweepArray(accounting::ObjectStack* allocations, bool swap_bitma // Start by sweeping the continuous spaces. for (space::ContinuousSpace* space : sweep_spaces) { space::AllocSpace* alloc_space = space->AsAllocSpace(); - accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); - accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); if (swap_bitmaps) { std::swap(live_bitmap, mark_bitmap); } diff --git a/runtime/gc/collector/mark_sweep.h b/runtime/gc/collector/mark_sweep.h index d49e427..6dbb270 100644 --- a/runtime/gc/collector/mark_sweep.h +++ b/runtime/gc/collector/mark_sweep.h @@ -22,6 +22,7 @@ #include "base/macros.h" #include "base/mutex.h" #include "garbage_collector.h" +#include "gc/accounting/space_bitmap.h" #include "immune_region.h" #include "object_callbacks.h" #include "offsets.h" @@ -45,7 +46,6 @@ class Heap; namespace accounting { template<typename T> class AtomicStack; typedef AtomicStack<mirror::Object*> ObjectStack; - class SpaceBitmap; } // namespace accounting namespace collector { @@ -283,7 +283,7 @@ class MarkSweep : public GarbageCollector { // Current space, we check this space first to avoid searching for the appropriate space for an // object. - accounting::SpaceBitmap* current_space_bitmap_; + accounting::ContinuousSpaceBitmap* current_space_bitmap_; // Cache the heap's mark bitmap to prevent having to do 2 loads during slow path marking. accounting::HeapBitmap* mark_bitmap_; diff --git a/runtime/gc/collector/semi_space-inl.h b/runtime/gc/collector/semi_space-inl.h index df731ff..8a9611f 100644 --- a/runtime/gc/collector/semi_space-inl.h +++ b/runtime/gc/collector/semi_space-inl.h @@ -65,7 +65,7 @@ inline void SemiSpace::MarkObject( } obj_ptr->Assign(forward_address); } else { - accounting::SpaceBitmap* object_bitmap = + accounting::ContinuousSpaceBitmap* object_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); if (LIKELY(object_bitmap != nullptr)) { if (generational_) { diff --git a/runtime/gc/collector/semi_space.cc b/runtime/gc/collector/semi_space.cc index e82d533..c0e172e 100644 --- a/runtime/gc/collector/semi_space.cc +++ b/runtime/gc/collector/semi_space.cc @@ -63,7 +63,6 @@ namespace gc { namespace collector { static constexpr bool kProtectFromSpace = true; -static constexpr bool kClearFromSpace = true; static constexpr bool kStoreStackTraces = false; static constexpr bool kUseBytesPromoted = true; static constexpr size_t kBytesPromotedThreshold = 4 * MB; @@ -122,6 +121,7 @@ void SemiSpace::InitializePhase() { // Do any pre GC verification. timings_.NewSplit("PreGcVerification"); heap_->PreGcVerification(this); + CHECK(from_space_->CanMoveObjects()) << "Attempting to move from " << *from_space_; // Set the initial bitmap. to_space_live_bitmap_ = to_space_->GetLiveBitmap(); } @@ -182,9 +182,6 @@ void SemiSpace::MarkingPhase() { Locks::mutator_lock_->AssertExclusiveHeld(self_); TimingLogger::ScopedSplit split("MarkingPhase", &timings_); - // Need to do this with mutators paused so that somebody doesn't accidentally allocate into the - // wrong space. - heap_->SwapSemiSpaces(); if (generational_) { // If last_gc_to_space_end_ is out of the bounds of the from-space // (the to-space from last GC), then point it to the beginning of @@ -336,7 +333,7 @@ void SemiSpace::MarkReachableObjects() { // remain in the space, that is, the remembered set (and the // card table) didn't miss any from-space references in the // space. - accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); SemiSpaceVerifyNoFromSpaceReferencesObjectVisitor visitor(this); live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), reinterpret_cast<uintptr_t>(space->End()), @@ -344,7 +341,7 @@ void SemiSpace::MarkReachableObjects() { } } else { DCHECK(rem_set == nullptr); - accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); SemiSpaceScanObjectVisitor visitor(this); live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), reinterpret_cast<uintptr_t>(space->End()), @@ -396,10 +393,10 @@ void SemiSpace::ReclaimPhase() { // Note: Freed bytes can be negative if we copy form a compacted space to a free-list backed // space. heap_->RecordFree(freed_objects, freed_bytes); + timings_.StartSplit("PreSweepingGcVerification"); heap_->PreSweepingGcVerification(this); timings_.EndSplit(); - { WriterMutexLock mu(self_, *Locks::heap_bitmap_lock_); // Reclaim unmarked objects. @@ -414,11 +411,9 @@ void SemiSpace::ReclaimPhase() { TimingLogger::ScopedSplit split("UnBindBitmaps", &timings_); GetHeap()->UnBindBitmaps(); } - if (kClearFromSpace) { - // Release the memory used by the from space. - from_space_->Clear(); - } - from_space_->Reset(); + // TODO: Do this before doing verification since the from space may have objects which weren't + // moved and point to dead objects. + from_space_->Clear(); // Protect the from space. VLOG(heap) << "Protecting space " << *from_space_; if (kProtectFromSpace) { @@ -540,9 +535,9 @@ mirror::Object* SemiSpace::MarkNonForwardedObject(mirror::Object* obj) { // space. GetHeap()->WriteBarrierEveryFieldOf(forward_address); // Handle the bitmaps marking. - accounting::SpaceBitmap* live_bitmap = promo_dest_space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = promo_dest_space->GetLiveBitmap(); DCHECK(live_bitmap != nullptr); - accounting::SpaceBitmap* mark_bitmap = promo_dest_space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = promo_dest_space->GetMarkBitmap(); DCHECK(mark_bitmap != nullptr); DCHECK(!live_bitmap->Test(forward_address)); if (!whole_heap_collection_) { @@ -715,8 +710,8 @@ void SemiSpace::ScanObject(Object* obj) { // Scan anything that's on the mark stack. void SemiSpace::ProcessMarkStack() { - space::MallocSpace* promo_dest_space = NULL; - accounting::SpaceBitmap* live_bitmap = NULL; + space::MallocSpace* promo_dest_space = nullptr; + accounting::ContinuousSpaceBitmap* live_bitmap = nullptr; if (generational_ && !whole_heap_collection_) { // If a bump pointer space only collection (and the promotion is // enabled,) we delay the live-bitmap marking of promoted objects @@ -724,7 +719,7 @@ void SemiSpace::ProcessMarkStack() { promo_dest_space = GetHeap()->GetPrimaryFreeListSpace(); live_bitmap = promo_dest_space->GetLiveBitmap(); DCHECK(live_bitmap != nullptr); - accounting::SpaceBitmap* mark_bitmap = promo_dest_space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = promo_dest_space->GetMarkBitmap(); DCHECK(mark_bitmap != nullptr); DCHECK_EQ(live_bitmap, mark_bitmap); } diff --git a/runtime/gc/collector/semi_space.h b/runtime/gc/collector/semi_space.h index 3442751..4169ca9 100644 --- a/runtime/gc/collector/semi_space.h +++ b/runtime/gc/collector/semi_space.h @@ -21,6 +21,7 @@ #include "base/macros.h" #include "base/mutex.h" #include "garbage_collector.h" +#include "gc/accounting/space_bitmap.h" #include "immune_region.h" #include "object_callbacks.h" #include "offsets.h" @@ -42,7 +43,6 @@ class Heap; namespace accounting { template <typename T> class AtomicStack; typedef AtomicStack<mirror::Object*> ObjectStack; - class SpaceBitmap; } // namespace accounting namespace space { @@ -198,7 +198,8 @@ class SemiSpace : public GarbageCollector { // Destination and source spaces (can be any type of ContinuousMemMapAllocSpace which either has // a live bitmap or doesn't). space::ContinuousMemMapAllocSpace* to_space_; - accounting::SpaceBitmap* to_space_live_bitmap_; // Cached live bitmap as an optimization. + // Cached live bitmap as an optimization. + accounting::ContinuousSpaceBitmap* to_space_live_bitmap_; space::ContinuousMemMapAllocSpace* from_space_; Thread* self_; diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index fcf9fe9..0c09b47 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -81,10 +81,15 @@ static constexpr size_t kMaxConcurrentRemainingBytes = 512 * KB; // relative to partial/full GC. This is desirable since sticky GCs interfere less with mutator // threads (lower pauses, use less memory bandwidth). static constexpr double kStickyGcThroughputAdjustment = 1.25; +// Whether or not we use the free list large object space. +static constexpr bool kUseFreeListSpaceForLOS = false; +// Whtehr or not we compact the zygote in PreZygoteFork. +static constexpr bool kCompactZygote = kMovingCollector; +static constexpr size_t kNonMovingSpaceCapacity = 64 * MB; Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max_free, double target_utilization, size_t capacity, const std::string& image_file_name, - CollectorType post_zygote_collector_type, CollectorType background_collector_type, + CollectorType foreground_collector_type, CollectorType background_collector_type, size_t parallel_gc_threads, size_t conc_gc_threads, bool low_memory_mode, size_t long_pause_log_threshold, size_t long_gc_log_threshold, bool ignore_max_footprint, bool use_tlab, bool verify_pre_gc_heap, @@ -95,9 +100,9 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max dlmalloc_space_(nullptr), main_space_(nullptr), collector_type_(kCollectorTypeNone), - post_zygote_collector_type_(post_zygote_collector_type), + foreground_collector_type_(foreground_collector_type), background_collector_type_(background_collector_type), - desired_collector_type_(collector_type_), + desired_collector_type_(foreground_collector_type_), heap_trim_request_lock_(nullptr), last_trim_time_(0), heap_transition_target_time_(0), @@ -162,15 +167,11 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max // If we aren't the zygote, switch to the default non zygote allocator. This may update the // entrypoints. if (!is_zygote) { - desired_collector_type_ = post_zygote_collector_type_; large_object_threshold_ = kDefaultLargeObjectThreshold; - } else { - if (kMovingCollector) { - // We are the zygote, use bump pointer allocation + semi space collector. - bool generational = post_zygote_collector_type_ == kCollectorTypeGSS; - desired_collector_type_ = generational ? kCollectorTypeGSS : kCollectorTypeSS; - } else { - desired_collector_type_ = post_zygote_collector_type_; + // Background compaction is currently not supported for command line runs. + if (background_collector_type_ != foreground_collector_type_) { + LOG(WARNING) << "Disabling background compaction for non zygote"; + background_collector_type_ = foreground_collector_type_; } } ChangeCollector(desired_collector_type_); @@ -187,73 +188,61 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max // isn't going to get in the middle byte* oat_file_end_addr = image_space->GetImageHeader().GetOatFileEnd(); CHECK_GT(oat_file_end_addr, image_space->End()); - if (oat_file_end_addr > requested_alloc_space_begin) { - requested_alloc_space_begin = AlignUp(oat_file_end_addr, kPageSize); - } + requested_alloc_space_begin = AlignUp(oat_file_end_addr, kPageSize); } - MemMap* malloc_space_mem_map = nullptr; - const char* malloc_space_name = is_zygote ? "zygote space" : "alloc space"; if (is_zygote) { - // Allocate a single mem map that is split into the malloc space - // and the post zygote non-moving space to put them adjacent. - size_t post_zygote_non_moving_space_size = 64 * MB; - size_t non_moving_spaces_size = capacity + post_zygote_non_moving_space_size; + // Reserve the address range before we create the non moving space to make sure bitmaps don't + // take it. std::string error_str; - malloc_space_mem_map = MemMap::MapAnonymous(malloc_space_name, requested_alloc_space_begin, - non_moving_spaces_size, PROT_READ | PROT_WRITE, - true, &error_str); - CHECK(malloc_space_mem_map != nullptr) << error_str; - post_zygote_non_moving_space_mem_map_.reset(malloc_space_mem_map->RemapAtEnd( - malloc_space_mem_map->Begin() + capacity, "post zygote non-moving space", - PROT_READ | PROT_WRITE, &error_str)); - CHECK(post_zygote_non_moving_space_mem_map_.get() != nullptr) << error_str; - VLOG(heap) << "malloc space mem map : " << malloc_space_mem_map; - VLOG(heap) << "post zygote non-moving space mem map : " - << post_zygote_non_moving_space_mem_map_.get(); + MemMap* mem_map = MemMap::MapAnonymous( + "main space", requested_alloc_space_begin + kNonMovingSpaceCapacity, capacity, + PROT_READ | PROT_WRITE, true, &error_str); + CHECK(mem_map != nullptr) << error_str; + // Non moving space is always dlmalloc since we currently don't have support for multiple + // rosalloc spaces. + non_moving_space_ = space::DlMallocSpace::Create( + "zygote / non moving space", initial_size, kNonMovingSpaceCapacity, kNonMovingSpaceCapacity, + requested_alloc_space_begin, false); + non_moving_space_->SetFootprintLimit(non_moving_space_->Capacity()); + CreateMainMallocSpace(mem_map, initial_size, growth_limit, capacity); } else { - // Allocate a mem map for the malloc space. std::string error_str; - malloc_space_mem_map = MemMap::MapAnonymous(malloc_space_name, requested_alloc_space_begin, - capacity, PROT_READ | PROT_WRITE, true, &error_str); - CHECK(malloc_space_mem_map != nullptr) << error_str; - VLOG(heap) << "malloc space mem map : " << malloc_space_mem_map; - } - CHECK(malloc_space_mem_map != nullptr); - space::MallocSpace* malloc_space; - if (kUseRosAlloc) { - malloc_space = space::RosAllocSpace::CreateFromMemMap(malloc_space_mem_map, malloc_space_name, - kDefaultStartingSize, initial_size, - growth_limit, capacity, low_memory_mode_); - CHECK(malloc_space != nullptr) << "Failed to create rosalloc space"; - } else { - malloc_space = space::DlMallocSpace::CreateFromMemMap(malloc_space_mem_map, malloc_space_name, - kDefaultStartingSize, initial_size, - growth_limit, capacity); - CHECK(malloc_space != nullptr) << "Failed to create dlmalloc space"; - } - VLOG(heap) << "malloc_space : " << malloc_space; + MemMap* mem_map = MemMap::MapAnonymous("main/non-moving space", requested_alloc_space_begin, + capacity, PROT_READ | PROT_WRITE, true, &error_str); + CHECK(mem_map != nullptr) << error_str; + // Create the main free list space, which doubles as the non moving space. We can do this since + // non zygote means that we won't have any background compaction. + CreateMainMallocSpace(mem_map, initial_size, growth_limit, capacity); + non_moving_space_ = main_space_; + } + CHECK(non_moving_space_ != nullptr); + + // We need to create the bump pointer if the foreground collector is a compacting GC. We only + // create the bump pointer space if we are not a moving foreground collector but have a moving + // background collector since the heap transition code will create the temp space by recycling + // the bitmap from the main space. if (kMovingCollector) { // TODO: Place bump-pointer spaces somewhere to minimize size of card table. - // TODO: Having 3+ spaces as big as the large heap size can cause virtual memory fragmentation - // issues. - const size_t bump_pointer_space_size = std::min(malloc_space->Capacity(), 128 * MB); + // TODO: Not create all the bump pointer spaces if not necessary (currently only GSS needs all + // 2 of bump pointer spaces + main space) b/14059466. Divide by 2 for a temporary fix. + const size_t bump_pointer_space_capacity = capacity / 2; bump_pointer_space_ = space::BumpPointerSpace::Create("Bump pointer space", - bump_pointer_space_size, nullptr); + bump_pointer_space_capacity, nullptr); CHECK(bump_pointer_space_ != nullptr) << "Failed to create bump pointer space"; AddSpace(bump_pointer_space_); - temp_space_ = space::BumpPointerSpace::Create("Bump pointer space 2", bump_pointer_space_size, - nullptr); + temp_space_ = space::BumpPointerSpace::Create("Bump pointer space 2", + bump_pointer_space_capacity, nullptr); CHECK(temp_space_ != nullptr) << "Failed to create bump pointer space"; AddSpace(temp_space_); - VLOG(heap) << "bump_pointer_space : " << bump_pointer_space_; - VLOG(heap) << "temp_space : " << temp_space_; } - non_moving_space_ = malloc_space; - malloc_space->SetFootprintLimit(malloc_space->Capacity()); - AddSpace(malloc_space); + if (non_moving_space_ != main_space_) { + AddSpace(non_moving_space_); + } + if (main_space_ != nullptr) { + AddSpace(main_space_); + } // Allocate the large object space. - constexpr bool kUseFreeListSpaceForLOS = false; if (kUseFreeListSpaceForLOS) { large_object_space_ = space::FreeListSpace::Create("large object space", nullptr, capacity); } else { @@ -268,11 +257,6 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max // Relies on the spaces being sorted. byte* heap_begin = continuous_spaces_.front()->Begin(); byte* heap_end = continuous_spaces_.back()->Limit(); - if (is_zygote) { - CHECK(post_zygote_non_moving_space_mem_map_.get() != nullptr); - heap_begin = std::min(post_zygote_non_moving_space_mem_map_->Begin(), heap_begin); - heap_end = std::max(post_zygote_non_moving_space_mem_map_->End(), heap_end); - } size_t heap_capacity = heap_end - heap_begin; // Allocate the card table. @@ -292,6 +276,12 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max new accounting::RememberedSet("Non-moving space remembered set", this, non_moving_space_); CHECK(non_moving_space_rem_set != nullptr) << "Failed to create non-moving space remembered set"; AddRememberedSet(non_moving_space_rem_set); + if (main_space_ != nullptr && main_space_ != non_moving_space_) { + accounting::RememberedSet* main_space_rem_set = + new accounting::RememberedSet("Main space remembered set", this, main_space_); + CHECK(main_space_rem_set != nullptr) << "Failed to create main space remembered set"; + AddRememberedSet(main_space_rem_set); + } } // TODO: Count objects in the image space here. @@ -329,7 +319,7 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max } if (kMovingCollector) { // TODO: Clean this up. - bool generational = post_zygote_collector_type_ == kCollectorTypeGSS; + bool generational = foreground_collector_type_ == kCollectorTypeGSS; semi_space_collector_ = new collector::SemiSpace(this, generational, generational ? "generational" : ""); garbage_collectors_.push_back(semi_space_collector_); @@ -347,6 +337,37 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max } } +void Heap::CreateMainMallocSpace(MemMap* mem_map, size_t initial_size, size_t growth_limit, + size_t capacity) { + // Is background compaction is enabled? + bool can_move_objects = IsMovingGc(background_collector_type_) != + IsMovingGc(foreground_collector_type_); + // If we are the zygote and don't yet have a zygote space, it means that the zygote fork will + // happen in the future. If this happens and we have kCompactZygote enabled we wish to compact + // from the main space to the zygote space. If background compaction is enabled, always pass in + // that we can move objets. + if (kCompactZygote && Runtime::Current()->IsZygote() && !can_move_objects) { + // After the zygote we want this to be false if we don't have background compaction enabled so + // that getting primitive array elements is faster. + can_move_objects = !have_zygote_space_; + } + if (kUseRosAlloc) { + main_space_ = space::RosAllocSpace::CreateFromMemMap(mem_map, "main rosalloc space", + kDefaultStartingSize, initial_size, + growth_limit, capacity, low_memory_mode_, + can_move_objects); + CHECK(main_space_ != nullptr) << "Failed to create rosalloc space"; + } else { + main_space_ = space::DlMallocSpace::CreateFromMemMap(mem_map, "main dlmalloc space", + kDefaultStartingSize, initial_size, + growth_limit, capacity, + can_move_objects); + CHECK(main_space_ != nullptr) << "Failed to create dlmalloc space"; + } + main_space_->SetFootprintLimit(main_space_->Capacity()); + VLOG(heap) << "Created main space " << main_space_; +} + void Heap::ChangeAllocator(AllocatorType allocator) { if (current_allocator_ != allocator) { // These two allocators are only used internally and don't have any entrypoints. @@ -360,13 +381,13 @@ void Heap::ChangeAllocator(AllocatorType allocator) { } void Heap::DisableCompaction() { - if (IsCompactingGC(post_zygote_collector_type_)) { - post_zygote_collector_type_ = kCollectorTypeCMS; + if (IsMovingGc(foreground_collector_type_)) { + foreground_collector_type_ = kCollectorTypeCMS; } - if (IsCompactingGC(background_collector_type_)) { - background_collector_type_ = post_zygote_collector_type_; + if (IsMovingGc(background_collector_type_)) { + background_collector_type_ = foreground_collector_type_; } - TransitionCollector(post_zygote_collector_type_); + TransitionCollector(foreground_collector_type_); } std::string Heap::SafeGetClassDescriptor(mirror::Class* klass) { @@ -428,14 +449,6 @@ void Heap::DumpObject(std::ostream& stream, mirror::Object* obj) { break; } } - if (space == nullptr) { - if (allocator_mem_map_.get() == nullptr || !allocator_mem_map_->HasAddress(obj)) { - stream << "obj " << obj << " not a valid heap address"; - return; - } else if (allocator_mem_map_.get() != nullptr) { - allocator_mem_map_->Protect(PROT_READ | PROT_WRITE); - } - } // Unprotect all the spaces. for (const auto& space : continuous_spaces_) { mprotect(space->Begin(), space->Capacity(), PROT_READ | PROT_WRITE); @@ -478,7 +491,7 @@ void Heap::IncrementDisableMovingGC(Thread* self) { ScopedThreadStateChange tsc(self, kWaitingForGcToComplete); MutexLock mu(self, *gc_complete_lock_); ++disable_moving_gc_count_; - if (IsCompactingGC(collector_type_running_)) { + if (IsMovingGc(collector_type_running_)) { WaitForGcToCompleteLocked(self); } } @@ -496,12 +509,12 @@ void Heap::UpdateProcessState(ProcessState process_state) { // Start at index 1 to avoid "is always false" warning. // Have iteration 1 always transition the collector. TransitionCollector((((i & 1) == 1) == (process_state_ == kProcessStateJankPerceptible)) - ? post_zygote_collector_type_ : background_collector_type_); + ? foreground_collector_type_ : background_collector_type_); usleep(kCollectorTransitionStressWait); } if (process_state_ == kProcessStateJankPerceptible) { // Transition back to foreground right away to prevent jank. - RequestCollectorTransition(post_zygote_collector_type_, 0); + RequestCollectorTransition(foreground_collector_type_, 0); } else { // Don't delay for debug builds since we may want to stress test the GC. RequestCollectorTransition(background_collector_type_, kIsDebugBuild ? 0 : @@ -568,8 +581,8 @@ void Heap::AddSpace(space::Space* space, bool set_as_default) { DCHECK(!space->IsDiscontinuousSpace()); space::ContinuousSpace* continuous_space = space->AsContinuousSpace(); // Continuous spaces don't necessarily have bitmaps. - accounting::SpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap(); - accounting::SpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap(); if (live_bitmap != nullptr) { DCHECK(mark_bitmap != nullptr); live_bitmap_->AddContinuousSpaceBitmap(live_bitmap); @@ -609,8 +622,8 @@ void Heap::RemoveSpace(space::Space* space) { DCHECK(!space->IsDiscontinuousSpace()); space::ContinuousSpace* continuous_space = space->AsContinuousSpace(); // Continuous spaces don't necessarily have bitmaps. - accounting::SpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap(); - accounting::SpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = continuous_space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = continuous_space->GetMarkBitmap(); if (live_bitmap != nullptr) { DCHECK(mark_bitmap != nullptr); live_bitmap_->RemoveContinuousSpaceBitmap(live_bitmap); @@ -626,6 +639,10 @@ void Heap::RemoveSpace(space::Space* space) { } if (continuous_space == main_space_) { main_space_ = nullptr; + } else if (continuous_space == bump_pointer_space_) { + bump_pointer_space_ = nullptr; + } else if (continuous_space == temp_space_) { + temp_space_ = nullptr; } } else { DCHECK(space->IsDiscontinuousSpace()); @@ -667,7 +684,8 @@ void Heap::DumpGcPerformanceInfo(std::ostream& os) { for (auto& collector : garbage_collectors_) { const CumulativeLogger& logger = collector->GetCumulativeTimings(); const size_t iterations = logger.GetIterations(); - if (iterations != 0) { + const Histogram<uint64_t>& pause_histogram = collector->GetPauseHistogram(); + if (iterations != 0 && pause_histogram.SampleSize() != 0) { os << ConstDumpable<CumulativeLogger>(logger); const uint64_t total_ns = logger.GetTotalNs(); const uint64_t total_pause_ns = collector->GetTotalPausedTimeNs(); @@ -675,8 +693,8 @@ void Heap::DumpGcPerformanceInfo(std::ostream& os) { const uint64_t freed_bytes = collector->GetTotalFreedBytes(); const uint64_t freed_objects = collector->GetTotalFreedObjects(); Histogram<uint64_t>::CumulativeData cumulative_data; - collector->GetPauseHistogram().CreateHistogram(&cumulative_data); - collector->GetPauseHistogram().PrintConfidenceIntervals(os, 0.99, cumulative_data); + pause_histogram.CreateHistogram(&cumulative_data); + pause_histogram.PrintConfidenceIntervals(os, 0.99, cumulative_data); os << collector->GetName() << " total time: " << PrettyDuration(total_ns) << " mean time: " << PrettyDuration(total_ns / iterations) << "\n" << collector->GetName() << " freed: " << freed_objects @@ -966,8 +984,10 @@ void Heap::Trim() { managed_reclaimed += alloc_space->Trim(); } } - total_alloc_space_allocated = GetBytesAllocated() - large_object_space_->GetBytesAllocated() - - bump_pointer_space_->Size(); + total_alloc_space_allocated = GetBytesAllocated() - large_object_space_->GetBytesAllocated(); + if (bump_pointer_space_ != nullptr) { + total_alloc_space_allocated -= bump_pointer_space_->Size(); + } const float managed_utilization = static_cast<float>(total_alloc_space_allocated) / static_cast<float>(total_alloc_space_size); uint64_t gc_heap_end_ns = NanoTime(); @@ -1083,8 +1103,8 @@ bool Heap::IsLiveObjectLocked(mirror::Object* obj, bool search_allocation_stack, void Heap::DumpSpaces(std::ostream& stream) { for (const auto& space : continuous_spaces_) { - accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); - accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); stream << space << " " << *space << "\n"; if (live_bitmap != nullptr) { stream << live_bitmap << " " << *live_bitmap << "\n"; @@ -1399,7 +1419,7 @@ void Heap::TransitionCollector(CollectorType collector_type) { ScopedThreadStateChange tsc(self, kWaitingPerformingGc); Locks::mutator_lock_->AssertNotHeld(self); const bool copying_transition = - IsCompactingGC(background_collector_type_) || IsCompactingGC(post_zygote_collector_type_); + IsMovingGc(background_collector_type_) || IsMovingGc(foreground_collector_type_); // Busy wait until we can GC (StartGC can fail if we have a non-zero // compacting_gc_disable_count_, this should rarely occurs). for (;;) { @@ -1430,42 +1450,20 @@ void Heap::TransitionCollector(CollectorType collector_type) { case kCollectorTypeSS: // Fall-through. case kCollectorTypeGSS: { - mprotect(temp_space_->Begin(), temp_space_->Capacity(), PROT_READ | PROT_WRITE); - CHECK(main_space_ != nullptr); - Compact(temp_space_, main_space_); - DCHECK(allocator_mem_map_.get() == nullptr); - allocator_mem_map_.reset(main_space_->ReleaseMemMap()); - madvise(main_space_->Begin(), main_space_->Size(), MADV_DONTNEED); - // RemoveSpace does not delete the removed space. - space::Space* old_space = main_space_; - RemoveSpace(old_space); - delete old_space; + if (!IsMovingGc(collector_type_)) { + // We are transitioning from non moving GC -> moving GC, since we copied from the bump + // pointer space last transition it will be protected. + bump_pointer_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); + Compact(bump_pointer_space_, main_space_); + } break; } case kCollectorTypeMS: // Fall through. case kCollectorTypeCMS: { - if (IsCompactingGC(collector_type_)) { - // TODO: Use mem-map from temp space? - MemMap* mem_map = allocator_mem_map_.release(); - CHECK(mem_map != nullptr); - size_t starting_size = kDefaultStartingSize; - size_t initial_size = kDefaultInitialSize; - mprotect(mem_map->Begin(), initial_size, PROT_READ | PROT_WRITE); - CHECK(main_space_ == nullptr); - if (kUseRosAlloc) { - main_space_ = - space::RosAllocSpace::CreateFromMemMap(mem_map, "alloc space", starting_size, - initial_size, mem_map->Size(), - mem_map->Size(), low_memory_mode_); - } else { - main_space_ = - space::DlMallocSpace::CreateFromMemMap(mem_map, "alloc space", starting_size, - initial_size, mem_map->Size(), - mem_map->Size()); - } - main_space_->SetFootprintLimit(main_space_->Capacity()); - AddSpace(main_space_); + if (IsMovingGc(collector_type_)) { + // Compact to the main space from the bump pointer space, don't need to swap semispaces. + main_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); Compact(main_space_, bump_pointer_space_); } break; @@ -1568,9 +1566,9 @@ class ZygoteCompactingCollector FINAL : public collector::SemiSpace { // Maps from bin sizes to locations. std::multimap<size_t, uintptr_t> bins_; // Live bitmap of the space which contains the bins. - accounting::SpaceBitmap* bin_live_bitmap_; + accounting::ContinuousSpaceBitmap* bin_live_bitmap_; // Mark bitmap of the space which contains the bins. - accounting::SpaceBitmap* bin_mark_bitmap_; + accounting::ContinuousSpaceBitmap* bin_mark_bitmap_; static void Callback(mirror::Object* obj, void* arg) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { @@ -1661,11 +1659,12 @@ void Heap::PreZygoteFork() { VLOG(heap) << "Starting PreZygoteFork"; // Trim the pages at the end of the non moving space. non_moving_space_->Trim(); + // The end of the non-moving space may be protected, unprotect it so that we can copy the zygote + // there. non_moving_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); // Change the collector to the post zygote one. - ChangeCollector(post_zygote_collector_type_); - // TODO: Delete bump_pointer_space_ and temp_pointer_space_? - if (semi_space_collector_ != nullptr) { + if (kCompactZygote) { + DCHECK(semi_space_collector_ != nullptr); // Temporarily disable rosalloc verification because the zygote // compaction will mess up the rosalloc internal metadata. ScopedDisableRosAllocVerification disable_rosalloc_verif(this); @@ -1675,18 +1674,47 @@ void Heap::PreZygoteFork() { space::BumpPointerSpace target_space("zygote bump space", non_moving_space_->End(), non_moving_space_->Limit()); // Compact the bump pointer space to a new zygote bump pointer space. - temp_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); - zygote_collector.SetFromSpace(bump_pointer_space_); + bool reset_main_space = false; + if (IsMovingGc(collector_type_)) { + zygote_collector.SetFromSpace(bump_pointer_space_); + } else { + CHECK(main_space_ != nullptr); + // Copy from the main space. + zygote_collector.SetFromSpace(main_space_); + reset_main_space = true; + } zygote_collector.SetToSpace(&target_space); + + Runtime::Current()->GetThreadList()->SuspendAll(); zygote_collector.Run(kGcCauseCollectorTransition, false); - CHECK(temp_space_->IsEmpty()); + if (IsMovingGc(collector_type_)) { + SwapSemiSpaces(); + } + Runtime::Current()->GetThreadList()->ResumeAll(); + + if (reset_main_space) { + main_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); + madvise(main_space_->Begin(), main_space_->Capacity(), MADV_DONTNEED); + MemMap* mem_map = main_space_->ReleaseMemMap(); + RemoveSpace(main_space_); + delete main_space_; + main_space_ = nullptr; + CreateMainMallocSpace(mem_map, kDefaultInitialSize, mem_map->Size(), mem_map->Size()); + AddSpace(main_space_); + } else { + bump_pointer_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); + } + if (temp_space_ != nullptr) { + CHECK(temp_space_->IsEmpty()); + } total_objects_freed_ever_ += semi_space_collector_->GetFreedObjects(); total_bytes_freed_ever_ += semi_space_collector_->GetFreedBytes(); // Update the end and write out image. non_moving_space_->SetEnd(target_space.End()); non_moving_space_->SetLimit(target_space.Limit()); - VLOG(heap) << "Zygote size " << non_moving_space_->Size() << " bytes"; + VLOG(heap) << "Zygote space size " << non_moving_space_->Size() << " bytes"; } + ChangeCollector(foreground_collector_type_); // Save the old space so that we can remove it after we complete creating the zygote space. space::MallocSpace* old_alloc_space = non_moving_space_; // Turn the current alloc space into a zygote space and obtain the new alloc space composed of @@ -1706,18 +1734,12 @@ void Heap::PreZygoteFork() { } space::ZygoteSpace* zygote_space = old_alloc_space->CreateZygoteSpace("alloc space", low_memory_mode_, - &main_space_); + &non_moving_space_); delete old_alloc_space; CHECK(zygote_space != nullptr) << "Failed creating zygote space"; AddSpace(zygote_space, false); - CHECK(main_space_ != nullptr); - if (main_space_->IsRosAllocSpace()) { - rosalloc_space_ = main_space_->AsRosAllocSpace(); - } else if (main_space_->IsDlMallocSpace()) { - dlmalloc_space_ = main_space_->AsDlMallocSpace(); - } - main_space_->SetFootprintLimit(main_space_->Capacity()); - AddSpace(main_space_); + non_moving_space_->SetFootprintLimit(non_moving_space_->Capacity()); + AddSpace(non_moving_space_); have_zygote_space_ = true; // Enable large object space allocations. large_object_threshold_ = kDefaultLargeObjectThreshold; @@ -1727,23 +1749,6 @@ void Heap::PreZygoteFork() { CHECK(mod_union_table != nullptr) << "Failed to create zygote space mod-union table"; AddModUnionTable(mod_union_table); if (collector::SemiSpace::kUseRememberedSet) { - // Add a new remembered set for the new main space. - accounting::RememberedSet* main_space_rem_set = - new accounting::RememberedSet("Main space remembered set", this, main_space_); - CHECK(main_space_rem_set != nullptr) << "Failed to create main space remembered set"; - AddRememberedSet(main_space_rem_set); - } - // Can't use RosAlloc for non moving space due to thread local buffers. - // TODO: Non limited space for non-movable objects? - MemMap* mem_map = post_zygote_non_moving_space_mem_map_.release(); - space::MallocSpace* new_non_moving_space = - space::DlMallocSpace::CreateFromMemMap(mem_map, "Non moving dlmalloc space", kPageSize, - 2 * MB, mem_map->Size(), mem_map->Size()); - AddSpace(new_non_moving_space, false); - CHECK(new_non_moving_space != nullptr) << "Failed to create new non-moving space"; - new_non_moving_space->SetFootprintLimit(new_non_moving_space->Capacity()); - non_moving_space_ = new_non_moving_space; - if (collector::SemiSpace::kUseRememberedSet) { // Add a new remembered set for the post-zygote non-moving space. accounting::RememberedSet* post_zygote_non_moving_space_rem_set = new accounting::RememberedSet("Post-zygote non-moving space remembered set", this, @@ -1759,8 +1764,8 @@ void Heap::FlushAllocStack() { allocation_stack_->Reset(); } -void Heap::MarkAllocStack(accounting::SpaceBitmap* bitmap1, - accounting::SpaceBitmap* bitmap2, +void Heap::MarkAllocStack(accounting::ContinuousSpaceBitmap* bitmap1, + accounting::ContinuousSpaceBitmap* bitmap2, accounting::ObjectSet* large_objects, accounting::ObjectStack* stack) { DCHECK(bitmap1 != nullptr); @@ -1781,9 +1786,9 @@ void Heap::MarkAllocStack(accounting::SpaceBitmap* bitmap1, } void Heap::SwapSemiSpaces() { - // Swap the spaces so we allocate into the space which we just evacuated. + CHECK(bump_pointer_space_ != nullptr); + CHECK(temp_space_ != nullptr); std::swap(bump_pointer_space_, temp_space_); - bump_pointer_space_->Clear(); } void Heap::Compact(space::ContinuousMemMapAllocSpace* target_space, @@ -1826,7 +1831,7 @@ collector::GcType Heap::CollectGarbageInternal(collector::GcType gc_type, GcCaus MutexLock mu(self, *gc_complete_lock_); // Ensure there is only one GC at a time. WaitForGcToCompleteLocked(self); - compacting_gc = IsCompactingGC(collector_type_); + compacting_gc = IsMovingGc(collector_type_); // GC can be disabled if someone has a used GetPrimitiveArrayCritical. if (compacting_gc && disable_moving_gc_count_ != 0) { LOG(WARNING) << "Skipping GC due to disable moving GC count " << disable_moving_gc_count_; @@ -1881,7 +1886,14 @@ collector::GcType Heap::CollectGarbageInternal(collector::GcType gc_type, GcCaus << "Could not find garbage collector with collector_type=" << static_cast<size_t>(collector_type_) << " and gc_type=" << gc_type; ATRACE_BEGIN(StringPrintf("%s %s GC", PrettyCause(gc_cause), collector->GetName()).c_str()); - collector->Run(gc_cause, clear_soft_references || runtime->IsZygote()); + if (compacting_gc) { + runtime->GetThreadList()->SuspendAll(); + collector->Run(gc_cause, clear_soft_references || runtime->IsZygote()); + SwapSemiSpaces(); + runtime->GetThreadList()->ResumeAll(); + } else { + collector->Run(gc_cause, clear_soft_references || runtime->IsZygote()); + } total_objects_freed_ever_ += collector->GetFreedObjects(); total_bytes_freed_ever_ += collector->GetFreedBytes(); RequestHeapTrim(); @@ -2026,7 +2038,8 @@ class VerifyReferenceVisitor { accounting::CardTable::kCardSize); LOG(ERROR) << "Card " << reinterpret_cast<void*>(card_addr) << " covers " << cover_begin << "-" << cover_end; - accounting::SpaceBitmap* bitmap = heap_->GetLiveBitmap()->GetContinuousSpaceBitmap(obj); + accounting::ContinuousSpaceBitmap* bitmap = + heap_->GetLiveBitmap()->GetContinuousSpaceBitmap(obj); if (bitmap == nullptr) { LOG(ERROR) << "Object " << obj << " has no bitmap"; @@ -2396,9 +2409,11 @@ void Heap::PreSweepingGcVerification(collector::GarbageCollector* gc) { WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); // Swapping bound bitmaps does nothing. gc->SwapBitmaps(); + SwapSemiSpaces(); if (!VerifyHeapReferences()) { LOG(FATAL) << "Pre sweeping " << gc->GetName() << " GC verification failed"; } + SwapSemiSpaces(); gc->SwapBitmaps(); } } @@ -2484,29 +2499,15 @@ void Heap::SetIdealFootprint(size_t max_allowed_footprint) { bool Heap::IsMovableObject(const mirror::Object* obj) const { if (kMovingCollector) { - DCHECK(!IsInTempSpace(obj)); - if (bump_pointer_space_->HasAddress(obj)) { - return true; - } - // TODO: Refactor this logic into the space itself? - // Objects in the main space are only copied during background -> foreground transitions or - // visa versa. - if (main_space_ != nullptr && main_space_->HasAddress(obj) && - (IsCompactingGC(background_collector_type_) || - IsCompactingGC(post_zygote_collector_type_))) { - return true; + space::Space* space = FindContinuousSpaceFromObject(obj, true); + if (space != nullptr) { + // TODO: Check large object? + return space->CanMoveObjects(); } } return false; } -bool Heap::IsInTempSpace(const mirror::Object* obj) const { - if (temp_space_->HasAddress(obj) && !temp_space_->Contains(obj)) { - return true; - } - return false; -} - void Heap::UpdateMaxNativeFootprint() { size_t native_size = native_bytes_allocated_; // TODO: Tune the native heap utilization to be a value other than the java heap utilization. @@ -2804,7 +2805,7 @@ void Heap::RegisterNativeAllocation(JNIEnv* env, int bytes) { if (IsGcConcurrent()) { RequestConcurrentGC(self); } else { - CollectGarbageInternal(gc_type, kGcCauseForAlloc, false); + CollectGarbageInternal(gc_type, kGcCauseForNativeAlloc, false); } } } @@ -2873,7 +2874,7 @@ void Heap::RemoveRememberedSet(space::Space* space) { void Heap::ClearMarkedObjects() { // Clear all of the spaces' mark bitmaps. for (const auto& space : GetContinuousSpaces()) { - accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); if (space->GetLiveBitmap() != mark_bitmap) { mark_bitmap->Clear(); } diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h index a8989ec..874357f 100644 --- a/runtime/gc/heap.h +++ b/runtime/gc/heap.h @@ -150,7 +150,7 @@ class Heap { explicit Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max_free, double target_utilization, size_t capacity, const std::string& original_image_file_name, - CollectorType post_zygote_collector_type, CollectorType background_collector_type, + CollectorType foreground_collector_type, CollectorType background_collector_type, size_t parallel_gc_threads, size_t conc_gc_threads, bool low_memory_mode, size_t long_pause_threshold, size_t long_gc_threshold, bool ignore_max_footprint, bool use_tlab, bool verify_pre_gc_heap, @@ -196,8 +196,6 @@ class Heap { void VisitObjects(ObjectCallback callback, void* arg) SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - void SwapSemiSpaces() EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); - void CheckPreconditionsForAllocObject(mirror::Class* c, size_t byte_count) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); void ThrowOutOfMemoryError(size_t byte_count, bool large_object_allocation); @@ -249,10 +247,6 @@ class Heap { // Returns true if there is any chance that the object (obj) will move. bool IsMovableObject(const mirror::Object* obj) const; - // Returns true if an object is in the temp space, if this happens its usually indicative of - // compaction related errors. - bool IsInTempSpace(const mirror::Object* obj) const; - // Enables us to compacting GC until objects are released. void IncrementDisableMovingGC(Thread* self); void DecrementDisableMovingGC(Thread* self); @@ -476,7 +470,9 @@ class Heap { LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_); // Mark all the objects in the allocation stack in the specified bitmap. - void MarkAllocStack(accounting::SpaceBitmap* bitmap1, accounting::SpaceBitmap* bitmap2, + // TODO: Refactor? + void MarkAllocStack(accounting::SpaceBitmap<kObjectAlignment>* bitmap1, + accounting::SpaceBitmap<kObjectAlignment>* bitmap2, accounting::ObjectSet* large_objects, accounting::ObjectStack* stack) EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_); @@ -568,7 +564,8 @@ class Heap { private: void Compact(space::ContinuousMemMapAllocSpace* target_space, - space::ContinuousMemMapAllocSpace* source_space); + space::ContinuousMemMapAllocSpace* source_space) + EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); void FinishGC(Thread* self, collector::GcType gc_type) LOCKS_EXCLUDED(gc_complete_lock_); @@ -580,7 +577,7 @@ class Heap { static ALWAYS_INLINE bool AllocatorMayHaveConcurrentGC(AllocatorType allocator_type) { return AllocatorHasAllocationStack(allocator_type); } - static bool IsCompactingGC(CollectorType collector_type) { + static bool IsMovingGc(CollectorType collector_type) { return collector_type == kCollectorTypeSS || collector_type == kCollectorTypeGSS || collector_type == kCollectorTypeCC; } @@ -609,6 +606,10 @@ class Heap { size_t bytes) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + // Need to do this with mutators paused so that somebody doesn't accidentally allocate into the + // wrong space. + void SwapSemiSpaces() EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); + // Try to allocate a number of bytes, this function never does any GCs. Needs to be inlined so // that the switch statement is constant optimized in the entrypoints. template <const bool kInstrumented, const bool kGrow> @@ -668,6 +669,10 @@ class Heap { // Find a collector based on GC type. collector::GarbageCollector* FindCollectorByGcType(collector::GcType gc_type); + // Create the main free list space, typically either a RosAlloc space or DlMalloc space. + void CreateMainMallocSpace(MemMap* mem_map, size_t initial_size, size_t growth_limit, + size_t capacity); + // Given the current contents of the alloc space, increase the allowed heap footprint to match // the target utilization ratio. This should only be called immediately after a full garbage // collection. @@ -737,17 +742,10 @@ class Heap { // A remembered set remembers all of the references from the it's space to the target space. SafeMap<space::Space*, accounting::RememberedSet*> remembered_sets_; - // Keep the free list allocator mem map lying around when we transition to background so that we - // don't have to worry about virtual address space fragmentation. - UniquePtr<MemMap> allocator_mem_map_; - - // The mem-map which we will use for the non-moving space after the zygote is done forking: - UniquePtr<MemMap> post_zygote_non_moving_space_mem_map_; - // The current collector type. CollectorType collector_type_; - // Which collector we will switch to after zygote fork. - CollectorType post_zygote_collector_type_; + // Which collector we use when the app is in the foreground. + CollectorType foreground_collector_type_; // Which collector we will use when the app is notified of a transition to background. CollectorType background_collector_type_; // Desired collector type, heap trimming daemon transitions the heap if it is != collector_type_. diff --git a/runtime/gc/heap_test.cc b/runtime/gc/heap_test.cc index 07e5088..a85ad4d 100644 --- a/runtime/gc/heap_test.cc +++ b/runtime/gc/heap_test.cc @@ -60,13 +60,11 @@ TEST_F(HeapTest, GarbageCollectClassLinkerInit) { TEST_F(HeapTest, HeapBitmapCapacityTest) { byte* heap_begin = reinterpret_cast<byte*>(0x1000); - const size_t heap_capacity = accounting::SpaceBitmap::kAlignment * (sizeof(intptr_t) * 8 + 1); - UniquePtr<accounting::SpaceBitmap> bitmap(accounting::SpaceBitmap::Create("test bitmap", - heap_begin, - heap_capacity)); + const size_t heap_capacity = kObjectAlignment * (sizeof(intptr_t) * 8 + 1); + UniquePtr<accounting::ContinuousSpaceBitmap> bitmap( + accounting::ContinuousSpaceBitmap::Create("test bitmap", heap_begin, heap_capacity)); mirror::Object* fake_end_of_heap_object = - reinterpret_cast<mirror::Object*>(&heap_begin[heap_capacity - - accounting::SpaceBitmap::kAlignment]); + reinterpret_cast<mirror::Object*>(&heap_begin[heap_capacity - kObjectAlignment]); bitmap->Set(fake_end_of_heap_object); } diff --git a/runtime/gc/space/bump_pointer_space.cc b/runtime/gc/space/bump_pointer_space.cc index a955cc8..90ffe59 100644 --- a/runtime/gc/space/bump_pointer_space.cc +++ b/runtime/gc/space/bump_pointer_space.cc @@ -38,6 +38,10 @@ BumpPointerSpace* BumpPointerSpace::Create(const std::string& name, size_t capac return new BumpPointerSpace(name, mem_map.release()); } +BumpPointerSpace* BumpPointerSpace::CreateFromMemMap(const std::string& name, MemMap* mem_map) { + return new BumpPointerSpace(name, mem_map); +} + BumpPointerSpace::BumpPointerSpace(const std::string& name, byte* begin, byte* limit) : ContinuousMemMapAllocSpace(name, nullptr, begin, begin, limit, kGcRetentionPolicyAlwaysCollect), @@ -61,9 +65,6 @@ BumpPointerSpace::BumpPointerSpace(const std::string& name, MemMap* mem_map) void BumpPointerSpace::Clear() { // Release the pages back to the operating system. CHECK_NE(madvise(Begin(), Limit() - Begin(), MADV_DONTNEED), -1) << "madvise failed"; -} - -void BumpPointerSpace::Reset() { // Reset the end of the space back to the beginning, we move the end forward as we allocate // objects. SetEnd(Begin()); @@ -196,7 +197,7 @@ void BumpPointerSpace::Walk(ObjectCallback* callback, void* arg) { } } -accounting::SpaceBitmap::SweepCallback* BumpPointerSpace::GetSweepCallback() { +accounting::ContinuousSpaceBitmap::SweepCallback* BumpPointerSpace::GetSweepCallback() { LOG(FATAL) << "Unimplemented"; return nullptr; } diff --git a/runtime/gc/space/bump_pointer_space.h b/runtime/gc/space/bump_pointer_space.h index 3ab5df4..e52a9a3 100644 --- a/runtime/gc/space/bump_pointer_space.h +++ b/runtime/gc/space/bump_pointer_space.h @@ -43,6 +43,7 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { // guaranteed to be granted, if it is required, the caller should call Begin on the returned // space to confirm the request was granted. static BumpPointerSpace* Create(const std::string& name, size_t capacity, byte* requested_begin); + static BumpPointerSpace* CreateFromMemMap(const std::string& name, MemMap* mem_map); // Allocate num_bytes, returns nullptr if the space is full. mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, @@ -84,19 +85,16 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { return GetMemMap()->Size(); } - accounting::SpaceBitmap* GetLiveBitmap() const OVERRIDE { + accounting::ContinuousSpaceBitmap* GetLiveBitmap() const OVERRIDE { return nullptr; } - accounting::SpaceBitmap* GetMarkBitmap() const OVERRIDE { + accounting::ContinuousSpaceBitmap* GetMarkBitmap() const OVERRIDE { return nullptr; } - // Madvise the memory back to the OS. - void Clear() OVERRIDE; - - // Reset the pointer to the start of the space. - void Reset() OVERRIDE LOCKS_EXCLUDED(block_lock_); + // Reset the space to empty. + void Clear() OVERRIDE LOCKS_EXCLUDED(block_lock_); void Dump(std::ostream& os) const; @@ -113,6 +111,9 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { return Begin() == End(); } + bool CanMoveObjects() const OVERRIDE { + return true; + } bool Contains(const mirror::Object* obj) const { const byte* byte_obj = reinterpret_cast<const byte*>(obj); @@ -137,7 +138,7 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { void Walk(ObjectCallback* callback, void* arg) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); - accounting::SpaceBitmap::SweepCallback* GetSweepCallback() OVERRIDE; + accounting::ContinuousSpaceBitmap::SweepCallback* GetSweepCallback() OVERRIDE; // Object alignment within the space. static constexpr size_t kAlignment = 8; diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h index 02d8b54..4c8a35e 100644 --- a/runtime/gc/space/dlmalloc_space-inl.h +++ b/runtime/gc/space/dlmalloc_space-inl.h @@ -52,7 +52,7 @@ inline size_t DlMallocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_ inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t num_bytes, size_t* bytes_allocated, size_t* usable_size) { - mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_for_alloc_, num_bytes)); + mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_, num_bytes)); if (LIKELY(result != NULL)) { if (kDebugSpaces) { CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result) diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc index 60f566c..41a0458 100644 --- a/runtime/gc/space/dlmalloc_space.cc +++ b/runtime/gc/space/dlmalloc_space.cc @@ -14,10 +14,10 @@ * limitations under the License. */ -#include "dlmalloc_space.h" - #include "dlmalloc_space-inl.h" + #include "gc/accounting/card_table.h" +#include "gc/accounting/space_bitmap-inl.h" #include "gc/heap.h" #include "mirror/class-inl.h" #include "mirror/object-inl.h" @@ -36,15 +36,19 @@ static constexpr bool kPrefetchDuringDlMallocFreeList = true; template class ValgrindMallocSpace<DlMallocSpace, void*>; DlMallocSpace::DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, byte* begin, - byte* end, byte* limit, size_t growth_limit) - : MallocSpace(name, mem_map, begin, end, limit, growth_limit), - mspace_(mspace), mspace_for_alloc_(mspace) { + byte* end, byte* limit, size_t growth_limit, + bool can_move_objects, size_t starting_size, + size_t initial_size) + : MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects, + starting_size, initial_size), + mspace_(mspace) { CHECK(mspace != NULL); } DlMallocSpace* DlMallocSpace::CreateFromMemMap(MemMap* mem_map, const std::string& name, size_t starting_size, size_t initial_size, - size_t growth_limit, size_t capacity) { + size_t growth_limit, size_t capacity, + bool can_move_objects) { DCHECK(mem_map != nullptr); void* mspace = CreateMspace(mem_map->Begin(), starting_size, initial_size); if (mspace == nullptr) { @@ -62,14 +66,17 @@ DlMallocSpace* DlMallocSpace::CreateFromMemMap(MemMap* mem_map, const std::strin byte* begin = mem_map->Begin(); if (Runtime::Current()->RunningOnValgrind()) { return new ValgrindMallocSpace<DlMallocSpace, void*>( - name, mem_map, mspace, begin, end, begin + capacity, growth_limit, initial_size); + name, mem_map, mspace, begin, end, begin + capacity, growth_limit, initial_size, + can_move_objects, starting_size); } else { - return new DlMallocSpace(name, mem_map, mspace, begin, end, begin + capacity, growth_limit); + return new DlMallocSpace(name, mem_map, mspace, begin, end, begin + capacity, growth_limit, + can_move_objects, starting_size, initial_size); } } -DlMallocSpace* DlMallocSpace::Create(const std::string& name, size_t initial_size, size_t growth_limit, - size_t capacity, byte* requested_begin) { +DlMallocSpace* DlMallocSpace::Create(const std::string& name, size_t initial_size, + size_t growth_limit, size_t capacity, byte* requested_begin, + bool can_move_objects) { uint64_t start_time = 0; if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { start_time = NanoTime(); @@ -93,7 +100,7 @@ DlMallocSpace* DlMallocSpace::Create(const std::string& name, size_t initial_siz return nullptr; } DlMallocSpace* space = CreateFromMemMap(mem_map, name, starting_size, initial_size, - growth_limit, capacity); + growth_limit, capacity, can_move_objects); // We start out with only the initial size possibly containing objects. if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { LOG(INFO) << "DlMallocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time) @@ -143,8 +150,10 @@ mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, MallocSpace* DlMallocSpace::CreateInstance(const std::string& name, MemMap* mem_map, void* allocator, byte* begin, byte* end, - byte* limit, size_t growth_limit) { - return new DlMallocSpace(name, mem_map, allocator, begin, end, limit, growth_limit); + byte* limit, size_t growth_limit, + bool can_move_objects) { + return new DlMallocSpace(name, mem_map, allocator, begin, end, limit, growth_limit, + can_move_objects, starting_size_, initial_size_); } size_t DlMallocSpace::Free(Thread* self, mirror::Object* ptr) { @@ -280,13 +289,13 @@ uint64_t DlMallocSpace::GetObjectsAllocated() { } void DlMallocSpace::Clear() { + size_t footprint_limit = GetFootprintLimit(); madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED); - GetLiveBitmap()->Clear(); - GetMarkBitmap()->Clear(); -} - -void DlMallocSpace::Reset() { - // TODO: Delete and create new mspace here. + live_bitmap_->Clear(); + mark_bitmap_->Clear(); + end_ = Begin() + starting_size_; + mspace_ = CreateMspace(mem_map_->Begin(), starting_size_, initial_size_); + SetFootprintLimit(footprint_limit); } #ifndef NDEBUG diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h index 76c4489..accd26b 100644 --- a/runtime/gc/space/dlmalloc_space.h +++ b/runtime/gc/space/dlmalloc_space.h @@ -36,14 +36,15 @@ class DlMallocSpace : public MallocSpace { // Create a DlMallocSpace from an existing mem_map. static DlMallocSpace* CreateFromMemMap(MemMap* mem_map, const std::string& name, size_t starting_size, size_t initial_size, - size_t growth_limit, size_t capacity); + size_t growth_limit, size_t capacity, + bool can_move_objects); // Create a DlMallocSpace with the requested sizes. The requested // base address is not guaranteed to be granted, if it is required, // the caller should call Begin on the returned space to confirm the // request was granted. static DlMallocSpace* Create(const std::string& name, size_t initial_size, size_t growth_limit, - size_t capacity, byte* requested_begin); + size_t capacity, byte* requested_begin, bool can_move_objects); // Virtual to allow ValgrindMallocSpace to intercept. virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, @@ -107,13 +108,13 @@ class DlMallocSpace : public MallocSpace { void SetFootprintLimit(size_t limit) OVERRIDE; MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator, - byte* begin, byte* end, byte* limit, size_t growth_limit); + byte* begin, byte* end, byte* limit, size_t growth_limit, + bool can_move_objects); uint64_t GetBytesAllocated() OVERRIDE; uint64_t GetObjectsAllocated() OVERRIDE; - void Clear() OVERRIDE; - void Reset() OVERRIDE; + virtual void Clear() OVERRIDE; bool IsDlMallocSpace() const OVERRIDE { return true; @@ -125,7 +126,8 @@ class DlMallocSpace : public MallocSpace { protected: DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, byte* begin, byte* end, - byte* limit, size_t growth_limit); + byte* limit, size_t growth_limit, bool can_move_objects, size_t starting_size, + size_t initial_size); private: mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated, @@ -142,11 +144,7 @@ class DlMallocSpace : public MallocSpace { static const size_t kChunkOverhead = kWordSize; // Underlying malloc space. - void* const mspace_; - - // An mspace pointer used for allocation. Equals mspace_ or nullptr after InvalidateAllocator() - // is called. - void* mspace_for_alloc_; + void* mspace_; friend class collector::MarkSweep; diff --git a/runtime/gc/space/dlmalloc_space_base_test.cc b/runtime/gc/space/dlmalloc_space_base_test.cc index 508d869..129eace 100644 --- a/runtime/gc/space/dlmalloc_space_base_test.cc +++ b/runtime/gc/space/dlmalloc_space_base_test.cc @@ -23,7 +23,7 @@ namespace space { MallocSpace* CreateDlMallocSpace(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin) { - return DlMallocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin); + return DlMallocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin, false); } TEST_SPACE_CREATE_FN_BASE(DlMallocSpace, CreateDlMallocSpace) diff --git a/runtime/gc/space/dlmalloc_space_random_test.cc b/runtime/gc/space/dlmalloc_space_random_test.cc index 43a1bf0..c4f8bae 100644 --- a/runtime/gc/space/dlmalloc_space_random_test.cc +++ b/runtime/gc/space/dlmalloc_space_random_test.cc @@ -23,7 +23,7 @@ namespace space { MallocSpace* CreateDlMallocSpace(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin) { - return DlMallocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin); + return DlMallocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin, false); } TEST_SPACE_CREATE_FN_RANDOM(DlMallocSpace, CreateDlMallocSpace) diff --git a/runtime/gc/space/dlmalloc_space_static_test.cc b/runtime/gc/space/dlmalloc_space_static_test.cc index 4fbc81e..edaa198 100644 --- a/runtime/gc/space/dlmalloc_space_static_test.cc +++ b/runtime/gc/space/dlmalloc_space_static_test.cc @@ -23,7 +23,7 @@ namespace space { MallocSpace* CreateDlMallocSpace(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin) { - return DlMallocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin); + return DlMallocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin, false); } TEST_SPACE_CREATE_FN_STATIC(DlMallocSpace, CreateDlMallocSpace) diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index faa539f..91d8820 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -35,7 +35,7 @@ namespace space { Atomic<uint32_t> ImageSpace::bitmap_index_(0); ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map, - accounting::SpaceBitmap* live_bitmap) + accounting::ContinuousSpaceBitmap* live_bitmap) : MemMapSpace(name, mem_map, mem_map->Begin(), mem_map->End(), mem_map->End(), kGcRetentionPolicyNeverCollect) { DCHECK(live_bitmap != nullptr); @@ -197,10 +197,10 @@ ImageSpace* ImageSpace::Init(const char* image_file_name, bool validate_oat_file uint32_t bitmap_index = bitmap_index_.FetchAndAdd(1); std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_file_name, bitmap_index)); - UniquePtr<accounting::SpaceBitmap> bitmap( - accounting::SpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(), - reinterpret_cast<byte*>(map->Begin()), - map->Size())); + UniquePtr<accounting::ContinuousSpaceBitmap> bitmap( + accounting::ContinuousSpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(), + reinterpret_cast<byte*>(map->Begin()), + map->Size())); if (bitmap.get() == nullptr) { *error_msg = StringPrintf("Could not create bitmap '%s'", bitmap_name.c_str()); return nullptr; diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h index 116c498..f6daf89 100644 --- a/runtime/gc/space/image_space.h +++ b/runtime/gc/space/image_space.h @@ -17,6 +17,7 @@ #ifndef ART_RUNTIME_GC_SPACE_IMAGE_SPACE_H_ #define ART_RUNTIME_GC_SPACE_IMAGE_SPACE_H_ +#include "gc/accounting/space_bitmap.h" #include "space.h" namespace art { @@ -59,11 +60,11 @@ class ImageSpace : public MemMapSpace { return GetName(); } - accounting::SpaceBitmap* GetLiveBitmap() const { + accounting::ContinuousSpaceBitmap* GetLiveBitmap() const OVERRIDE { return live_bitmap_.get(); } - accounting::SpaceBitmap* GetMarkBitmap() const { + accounting::ContinuousSpaceBitmap* GetMarkBitmap() const OVERRIDE { // ImageSpaces have the same bitmap for both live and marked. This helps reduce the number of // special cases to test against. return live_bitmap_.get(); @@ -75,6 +76,10 @@ class ImageSpace : public MemMapSpace { void Sweep(bool /* swap_bitmaps */, size_t* /* freed_objects */, size_t* /* freed_bytes */) { } + bool CanMoveObjects() const OVERRIDE { + return false; + } + private: // Tries to initialize an ImageSpace from the given image path, // returning NULL on error. @@ -96,9 +101,10 @@ class ImageSpace : public MemMapSpace { static Atomic<uint32_t> bitmap_index_; - UniquePtr<accounting::SpaceBitmap> live_bitmap_; + UniquePtr<accounting::ContinuousSpaceBitmap> live_bitmap_; - ImageSpace(const std::string& name, MemMap* mem_map, accounting::SpaceBitmap* live_bitmap); + ImageSpace(const std::string& name, MemMap* mem_map, + accounting::ContinuousSpaceBitmap* live_bitmap); // The OatFile associated with the image during early startup to // reserve space contiguous to the image. It is later released to diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h index eb01325..18e518f 100644 --- a/runtime/gc/space/large_object_space.h +++ b/runtime/gc/space/large_object_space.h @@ -75,6 +75,10 @@ class LargeObjectSpace : public DiscontinuousSpace, public AllocSpace { void Sweep(bool swap_bitmaps, size_t* freed_objects, size_t* freed_bytes); + virtual bool CanMoveObjects() const OVERRIDE { + return false; + } + protected: explicit LargeObjectSpace(const std::string& name); diff --git a/runtime/gc/space/malloc_space.cc b/runtime/gc/space/malloc_space.cc index dac043e..8f81446 100644 --- a/runtime/gc/space/malloc_space.cc +++ b/runtime/gc/space/malloc_space.cc @@ -37,24 +37,26 @@ size_t MallocSpace::bitmap_index_ = 0; MallocSpace::MallocSpace(const std::string& name, MemMap* mem_map, byte* begin, byte* end, byte* limit, size_t growth_limit, - bool create_bitmaps) + bool create_bitmaps, bool can_move_objects, size_t starting_size, + size_t initial_size) : ContinuousMemMapAllocSpace(name, mem_map, begin, end, limit, kGcRetentionPolicyAlwaysCollect), recent_free_pos_(0), lock_("allocation space lock", kAllocSpaceLock), - growth_limit_(growth_limit) { + growth_limit_(growth_limit), can_move_objects_(can_move_objects), + starting_size_(starting_size), initial_size_(initial_size) { if (create_bitmaps) { size_t bitmap_index = bitmap_index_++; static const uintptr_t kGcCardSize = static_cast<uintptr_t>(accounting::CardTable::kCardSize); CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->Begin()))); CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->End()))); - live_bitmap_.reset(accounting::SpaceBitmap::Create( + live_bitmap_.reset(accounting::ContinuousSpaceBitmap::Create( StringPrintf("allocspace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)), Begin(), Capacity())); - DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace live bitmap #" + DCHECK(live_bitmap_.get() != nullptr) << "could not create allocspace live bitmap #" << bitmap_index; - mark_bitmap_.reset(accounting::SpaceBitmap::Create( + mark_bitmap_.reset(accounting::ContinuousSpaceBitmap::Create( StringPrintf("allocspace %s mark-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)), Begin(), Capacity())); - DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace mark bitmap #" + DCHECK(live_bitmap_.get() != nullptr) << "could not create allocspace mark bitmap #" << bitmap_index; } for (auto& freed : recent_freed_objects_) { @@ -201,7 +203,7 @@ ZygoteSpace* MallocSpace::CreateZygoteSpace(const char* alloc_space_name, bool l CHECK_MEMORY_CALL(mprotect, (end, capacity - initial_size, PROT_NONE), alloc_space_name); } *out_malloc_space = CreateInstance(alloc_space_name, mem_map.release(), allocator, end_, end, - limit_, growth_limit); + limit_, growth_limit, CanMoveObjects()); SetLimit(End()); live_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End())); CHECK_EQ(live_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End())); @@ -236,7 +238,7 @@ void MallocSpace::SweepCallback(size_t num_ptrs, mirror::Object** ptrs, void* ar // If the bitmaps aren't swapped we need to clear the bits since the GC isn't going to re-swap // the bitmaps as an optimization. if (!context->swap_bitmaps) { - accounting::SpaceBitmap* bitmap = space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* bitmap = space->GetLiveBitmap(); for (size_t i = 0; i < num_ptrs; ++i) { bitmap->Clear(ptrs[i]); } diff --git a/runtime/gc/space/malloc_space.h b/runtime/gc/space/malloc_space.h index fbcee5f..d24016c 100644 --- a/runtime/gc/space/malloc_space.h +++ b/runtime/gc/space/malloc_space.h @@ -114,7 +114,8 @@ class MallocSpace : public ContinuousMemMapAllocSpace { void SetGrowthLimit(size_t growth_limit); virtual MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator, - byte* begin, byte* end, byte* limit, size_t growth_limit) = 0; + byte* begin, byte* end, byte* limit, size_t growth_limit, + bool can_move_objects) = 0; // Splits ourself into a zygote space and new malloc space which has our unused memory. When true, // the low memory mode argument specifies that the heap wishes the created space to be more @@ -127,9 +128,14 @@ class MallocSpace : public ContinuousMemMapAllocSpace { // Returns the class of a recently freed object. mirror::Class* FindRecentFreedObject(const mirror::Object* obj); + bool CanMoveObjects() const OVERRIDE { + return can_move_objects_; + } + protected: MallocSpace(const std::string& name, MemMap* mem_map, byte* begin, byte* end, - byte* limit, size_t growth_limit, bool create_bitmaps = true); + byte* limit, size_t growth_limit, bool create_bitmaps, bool can_move_objects, + size_t starting_size, size_t initial_size); static MemMap* CreateMemMap(const std::string& name, size_t starting_size, size_t* initial_size, size_t* growth_limit, size_t* capacity, byte* requested_begin); @@ -143,7 +149,7 @@ class MallocSpace : public ContinuousMemMapAllocSpace { SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) EXCLUSIVE_LOCKS_REQUIRED(lock_); - virtual accounting::SpaceBitmap::SweepCallback* GetSweepCallback() { + virtual accounting::ContinuousSpaceBitmap::SweepCallback* GetSweepCallback() { return &SweepCallback; } @@ -167,6 +173,13 @@ class MallocSpace : public ContinuousMemMapAllocSpace { // one time by a call to ClearGrowthLimit. size_t growth_limit_; + // True if objects in the space are movable. + const bool can_move_objects_; + + // Starting and initial sized, used when you reset the space. + const size_t starting_size_; + const size_t initial_size_; + private: static void SweepCallback(size_t num_ptrs, mirror::Object** ptrs, void* arg) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc index 5c5e7f8..5a7d941 100644 --- a/runtime/gc/space/rosalloc_space.cc +++ b/runtime/gc/space/rosalloc_space.cc @@ -15,10 +15,10 @@ * limitations under the License. */ -#include "rosalloc_space.h" - #include "rosalloc_space-inl.h" + #include "gc/accounting/card_table.h" +#include "gc/accounting/space_bitmap-inl.h" #include "gc/heap.h" #include "mirror/class-inl.h" #include "mirror/object-inl.h" @@ -34,19 +34,23 @@ namespace space { static constexpr bool kPrefetchDuringRosAllocFreeList = true; -template class ValgrindMallocSpace<RosAllocSpace, allocator::RosAlloc*>; +// TODO: Fix +// template class ValgrindMallocSpace<RosAllocSpace, allocator::RosAlloc*>; RosAllocSpace::RosAllocSpace(const std::string& name, MemMap* mem_map, art::gc::allocator::RosAlloc* rosalloc, byte* begin, byte* end, - byte* limit, size_t growth_limit) - : MallocSpace(name, mem_map, begin, end, limit, growth_limit), rosalloc_(rosalloc) { - CHECK(rosalloc != NULL); + byte* limit, size_t growth_limit, bool can_move_objects, + size_t starting_size, size_t initial_size, bool low_memory_mode) + : MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects, + starting_size, initial_size), + rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) { + CHECK(rosalloc != nullptr); } RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap* mem_map, const std::string& name, size_t starting_size, size_t initial_size, size_t growth_limit, size_t capacity, - bool low_memory_mode) { + bool low_memory_mode, bool can_move_objects) { DCHECK(mem_map != nullptr); allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map->Begin(), starting_size, initial_size, capacity, low_memory_mode); @@ -66,10 +70,10 @@ RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap* mem_map, const std::strin // TODO: Fix RosAllocSpace to support valgrind. There is currently some issues with // AllocationSize caused by redzones. b/12944686 if (false && Runtime::Current()->GetHeap()->RunningOnValgrind()) { - return new ValgrindMallocSpace<RosAllocSpace, allocator::RosAlloc*>( - name, mem_map, rosalloc, begin, end, begin + capacity, growth_limit, initial_size); + LOG(FATAL) << "Unimplemented"; } else { - return new RosAllocSpace(name, mem_map, rosalloc, begin, end, begin + capacity, growth_limit); + return new RosAllocSpace(name, mem_map, rosalloc, begin, end, begin + capacity, growth_limit, + can_move_objects, starting_size, initial_size, low_memory_mode); } } @@ -79,7 +83,7 @@ RosAllocSpace::~RosAllocSpace() { RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin, - bool low_memory_mode) { + bool low_memory_mode, bool can_move_objects) { uint64_t start_time = 0; if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { start_time = NanoTime(); @@ -104,7 +108,8 @@ RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_siz } RosAllocSpace* space = CreateFromMemMap(mem_map, name, starting_size, initial_size, - growth_limit, capacity, low_memory_mode); + growth_limit, capacity, low_memory_mode, + can_move_objects); // We start out with only the initial size possibly containing objects. if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time) @@ -113,7 +118,8 @@ RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_siz return space; } -allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start, size_t initial_size, +allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start, + size_t initial_size, size_t maximum_size, bool low_memory_mode) { // clear errno to allow PLOG on error errno = 0; @@ -154,9 +160,11 @@ mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, } MallocSpace* RosAllocSpace::CreateInstance(const std::string& name, MemMap* mem_map, void* allocator, - byte* begin, byte* end, byte* limit, size_t growth_limit) { + byte* begin, byte* end, byte* limit, size_t growth_limit, + bool can_move_objects) { return new RosAllocSpace(name, mem_map, reinterpret_cast<allocator::RosAlloc*>(allocator), - begin, end, limit, growth_limit); + begin, end, limit, growth_limit, can_move_objects, starting_size_, + initial_size_, low_memory_mode_); } size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) { @@ -333,13 +341,15 @@ void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() { } void RosAllocSpace::Clear() { + size_t footprint_limit = GetFootprintLimit(); madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED); - GetLiveBitmap()->Clear(); - GetMarkBitmap()->Clear(); -} - -void RosAllocSpace::Reset() { - // TODO: Delete and create new mspace here. + live_bitmap_->Clear(); + mark_bitmap_->Clear(); + end_ = begin_ + starting_size_; + delete rosalloc_; + rosalloc_ = CreateRosAlloc(mem_map_->Begin(), starting_size_, initial_size_, Capacity(), + low_memory_mode_); + SetFootprintLimit(footprint_limit); } } // namespace space diff --git a/runtime/gc/space/rosalloc_space.h b/runtime/gc/space/rosalloc_space.h index 900e7a9..a156738 100644 --- a/runtime/gc/space/rosalloc_space.h +++ b/runtime/gc/space/rosalloc_space.h @@ -39,11 +39,12 @@ class RosAllocSpace : public MallocSpace { // the caller should call Begin on the returned space to confirm the // request was granted. static RosAllocSpace* Create(const std::string& name, size_t initial_size, size_t growth_limit, - size_t capacity, byte* requested_begin, bool low_memory_mode); + size_t capacity, byte* requested_begin, bool low_memory_mode, + bool can_move_objects); static RosAllocSpace* CreateFromMemMap(MemMap* mem_map, const std::string& name, size_t starting_size, size_t initial_size, size_t growth_limit, size_t capacity, - bool low_memory_mode); + bool low_memory_mode, bool can_move_objects); mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_); @@ -80,9 +81,10 @@ class RosAllocSpace : public MallocSpace { void SetFootprintLimit(size_t limit) OVERRIDE; void Clear() OVERRIDE; - void Reset() OVERRIDE; + MallocSpace* CreateInstance(const std::string& name, MemMap* mem_map, void* allocator, - byte* begin, byte* end, byte* limit, size_t growth_limit); + byte* begin, byte* end, byte* limit, size_t growth_limit, + bool can_move_objects) OVERRIDE; uint64_t GetBytesAllocated() OVERRIDE; uint64_t GetObjectsAllocated() OVERRIDE; @@ -110,7 +112,8 @@ class RosAllocSpace : public MallocSpace { protected: RosAllocSpace(const std::string& name, MemMap* mem_map, allocator::RosAlloc* rosalloc, - byte* begin, byte* end, byte* limit, size_t growth_limit); + byte* begin, byte* end, byte* limit, size_t growth_limit, bool can_move_objects, + size_t starting_size, size_t initial_size, bool low_memory_mode); private: mirror::Object* AllocCommon(Thread* self, size_t num_bytes, size_t* bytes_allocated, @@ -132,7 +135,9 @@ class RosAllocSpace : public MallocSpace { LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_); // Underlying rosalloc. - allocator::RosAlloc* const rosalloc_; + allocator::RosAlloc* rosalloc_; + + const bool low_memory_mode_; friend class collector::MarkSweep; diff --git a/runtime/gc/space/rosalloc_space_base_test.cc b/runtime/gc/space/rosalloc_space_base_test.cc index df42076..c3157fa 100644 --- a/runtime/gc/space/rosalloc_space_base_test.cc +++ b/runtime/gc/space/rosalloc_space_base_test.cc @@ -23,7 +23,7 @@ namespace space { MallocSpace* CreateRosAllocSpace(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin) { return RosAllocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin, - Runtime::Current()->GetHeap()->IsLowMemoryMode()); + Runtime::Current()->GetHeap()->IsLowMemoryMode(), false); } TEST_SPACE_CREATE_FN_BASE(RosAllocSpace, CreateRosAllocSpace) diff --git a/runtime/gc/space/rosalloc_space_random_test.cc b/runtime/gc/space/rosalloc_space_random_test.cc index 4d37c9e..864bbc9 100644 --- a/runtime/gc/space/rosalloc_space_random_test.cc +++ b/runtime/gc/space/rosalloc_space_random_test.cc @@ -23,7 +23,7 @@ namespace space { MallocSpace* CreateRosAllocSpace(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin) { return RosAllocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin, - Runtime::Current()->GetHeap()->IsLowMemoryMode()); + Runtime::Current()->GetHeap()->IsLowMemoryMode(), false); } TEST_SPACE_CREATE_FN_RANDOM(RosAllocSpace, CreateRosAllocSpace) diff --git a/runtime/gc/space/rosalloc_space_static_test.cc b/runtime/gc/space/rosalloc_space_static_test.cc index 9f11fd0..c0e2ac8 100644 --- a/runtime/gc/space/rosalloc_space_static_test.cc +++ b/runtime/gc/space/rosalloc_space_static_test.cc @@ -23,7 +23,7 @@ namespace space { MallocSpace* CreateRosAllocSpace(const std::string& name, size_t initial_size, size_t growth_limit, size_t capacity, byte* requested_begin) { return RosAllocSpace::Create(name, initial_size, growth_limit, capacity, requested_begin, - Runtime::Current()->GetHeap()->IsLowMemoryMode()); + Runtime::Current()->GetHeap()->IsLowMemoryMode(), false); } TEST_SPACE_CREATE_FN_STATIC(RosAllocSpace, CreateRosAllocSpace) diff --git a/runtime/gc/space/space.cc b/runtime/gc/space/space.cc index 4af65a9..01e8b04 100644 --- a/runtime/gc/space/space.cc +++ b/runtime/gc/space/space.cc @@ -18,6 +18,7 @@ #include "base/logging.h" #include "gc/accounting/heap_bitmap.h" +#include "gc/accounting/space_bitmap-inl.h" #include "runtime.h" #include "thread-inl.h" @@ -77,8 +78,8 @@ DiscontinuousSpace::DiscontinuousSpace(const std::string& name, void ContinuousMemMapAllocSpace::Sweep(bool swap_bitmaps, size_t* freed_objects, size_t* freed_bytes) { DCHECK(freed_objects != nullptr); DCHECK(freed_bytes != nullptr); - accounting::SpaceBitmap* live_bitmap = GetLiveBitmap(); - accounting::SpaceBitmap* mark_bitmap = GetMarkBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* mark_bitmap = GetMarkBitmap(); // If the bitmaps are bound then sweeping this space clearly won't do anything. if (live_bitmap == mark_bitmap) { return; @@ -94,11 +95,9 @@ void ContinuousMemMapAllocSpace::Sweep(bool swap_bitmaps, size_t* freed_objects, std::swap(live_bitmap, mark_bitmap); } // Bitmaps are pre-swapped for optimization which enables sweeping with the heap unlocked. - accounting::SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, - reinterpret_cast<uintptr_t>(Begin()), - reinterpret_cast<uintptr_t>(End()), - GetSweepCallback(), - reinterpret_cast<void*>(&scc)); + accounting::ContinuousSpaceBitmap::SweepWalk( + *live_bitmap, *mark_bitmap, reinterpret_cast<uintptr_t>(Begin()), + reinterpret_cast<uintptr_t>(End()), GetSweepCallback(), reinterpret_cast<void*>(&scc)); *freed_objects += scc.freed_objects; *freed_bytes += scc.freed_bytes; } @@ -106,9 +105,9 @@ void ContinuousMemMapAllocSpace::Sweep(bool swap_bitmaps, size_t* freed_objects, // Returns the old mark bitmap. void ContinuousMemMapAllocSpace::BindLiveToMarkBitmap() { CHECK(!HasBoundBitmaps()); - accounting::SpaceBitmap* live_bitmap = GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* live_bitmap = GetLiveBitmap(); if (live_bitmap != mark_bitmap_.get()) { - accounting::SpaceBitmap* mark_bitmap = mark_bitmap_.release(); + accounting::ContinuousSpaceBitmap* mark_bitmap = mark_bitmap_.release(); Runtime::Current()->GetHeap()->GetMarkBitmap()->ReplaceBitmap(mark_bitmap, live_bitmap); temp_bitmap_.reset(mark_bitmap); mark_bitmap_.reset(live_bitmap); @@ -122,7 +121,7 @@ bool ContinuousMemMapAllocSpace::HasBoundBitmaps() const { void ContinuousMemMapAllocSpace::UnBindBitmaps() { CHECK(HasBoundBitmaps()); // At this point, the temp_bitmap holds our old mark bitmap. - accounting::SpaceBitmap* new_bitmap = temp_bitmap_.release(); + accounting::ContinuousSpaceBitmap* new_bitmap = temp_bitmap_.release(); Runtime::Current()->GetHeap()->GetMarkBitmap()->ReplaceBitmap(mark_bitmap_.get(), new_bitmap); CHECK_EQ(mark_bitmap_.release(), live_bitmap_.get()); mark_bitmap_.reset(new_bitmap); diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h index 37d7c80..2b27f87 100644 --- a/runtime/gc/space/space.h +++ b/runtime/gc/space/space.h @@ -34,10 +34,6 @@ namespace mirror { namespace gc { -namespace accounting { - class SpaceBitmap; -} // namespace accounting - class Heap; namespace space { @@ -160,6 +156,9 @@ class Space { } virtual ContinuousMemMapAllocSpace* AsContinuousMemMapAllocSpace(); + // Returns true if objects in the space are movable. + virtual bool CanMoveObjects() const = 0; + virtual ~Space() {} protected: @@ -265,8 +264,8 @@ class ContinuousSpace : public Space { return End() - Begin(); } - virtual accounting::SpaceBitmap* GetLiveBitmap() const = 0; - virtual accounting::SpaceBitmap* GetMarkBitmap() const = 0; + virtual accounting::ContinuousSpaceBitmap* GetLiveBitmap() const = 0; + virtual accounting::ContinuousSpaceBitmap* GetMarkBitmap() const = 0; // Maximum which the mapped space can grow to. virtual size_t Capacity() const { @@ -396,27 +395,24 @@ class ContinuousMemMapAllocSpace : public MemMapSpace, public AllocSpace { // Swap the live and mark bitmaps of this space. This is used by the GC for concurrent sweeping. void SwapBitmaps(); - // Free all memory associated with this space. + // Clear the space back to an empty space. virtual void Clear() = 0; - // Reset the space back to an empty space. - virtual void Reset() = 0; - - accounting::SpaceBitmap* GetLiveBitmap() const { + accounting::ContinuousSpaceBitmap* GetLiveBitmap() const { return live_bitmap_.get(); } - accounting::SpaceBitmap* GetMarkBitmap() const { + accounting::ContinuousSpaceBitmap* GetMarkBitmap() const { return mark_bitmap_.get(); } void Sweep(bool swap_bitmaps, size_t* freed_objects, size_t* freed_bytes); - virtual accounting::SpaceBitmap::SweepCallback* GetSweepCallback() = 0; + virtual accounting::ContinuousSpaceBitmap::SweepCallback* GetSweepCallback() = 0; protected: - UniquePtr<accounting::SpaceBitmap> live_bitmap_; - UniquePtr<accounting::SpaceBitmap> mark_bitmap_; - UniquePtr<accounting::SpaceBitmap> temp_bitmap_; + UniquePtr<accounting::ContinuousSpaceBitmap> live_bitmap_; + UniquePtr<accounting::ContinuousSpaceBitmap> mark_bitmap_; + UniquePtr<accounting::ContinuousSpaceBitmap> temp_bitmap_; ContinuousMemMapAllocSpace(const std::string& name, MemMap* mem_map, byte* begin, byte* end, byte* limit, GcRetentionPolicy gc_retention_policy) diff --git a/runtime/gc/space/valgrind_malloc_space-inl.h b/runtime/gc/space/valgrind_malloc_space-inl.h index ed97e60..966c276 100644 --- a/runtime/gc/space/valgrind_malloc_space-inl.h +++ b/runtime/gc/space/valgrind_malloc_space-inl.h @@ -95,8 +95,10 @@ template <typename S, typename A> ValgrindMallocSpace<S, A>::ValgrindMallocSpace(const std::string& name, MemMap* mem_map, A allocator, byte* begin, byte* end, byte* limit, size_t growth_limit, - size_t initial_size) : - S(name, mem_map, allocator, begin, end, limit, growth_limit) { + size_t initial_size, + bool can_move_objects, size_t starting_size) : + S(name, mem_map, allocator, begin, end, limit, growth_limit, can_move_objects, starting_size, + initial_size) { VALGRIND_MAKE_MEM_UNDEFINED(mem_map->Begin() + initial_size, mem_map->Size() - initial_size); } diff --git a/runtime/gc/space/valgrind_malloc_space.h b/runtime/gc/space/valgrind_malloc_space.h index 6b755c4..200ad83 100644 --- a/runtime/gc/space/valgrind_malloc_space.h +++ b/runtime/gc/space/valgrind_malloc_space.h @@ -48,7 +48,7 @@ class ValgrindMallocSpace FINAL : public BaseMallocSpaceType { ValgrindMallocSpace(const std::string& name, MemMap* mem_map, AllocatorType allocator, byte* begin, byte* end, byte* limit, size_t growth_limit, - size_t initial_size); + size_t initial_size, bool can_move_objects, size_t starting_size); virtual ~ValgrindMallocSpace() {} private: diff --git a/runtime/gc/space/zygote_space.cc b/runtime/gc/space/zygote_space.cc index d1c3d03..1b06b63 100644 --- a/runtime/gc/space/zygote_space.cc +++ b/runtime/gc/space/zygote_space.cc @@ -40,8 +40,8 @@ class CountObjectsAllocated { }; ZygoteSpace* ZygoteSpace::Create(const std::string& name, MemMap* mem_map, - accounting::SpaceBitmap* live_bitmap, - accounting::SpaceBitmap* mark_bitmap) { + accounting::ContinuousSpaceBitmap* live_bitmap, + accounting::ContinuousSpaceBitmap* mark_bitmap) { DCHECK(live_bitmap != nullptr); DCHECK(mark_bitmap != nullptr); size_t objects_allocated = 0; @@ -61,10 +61,6 @@ void ZygoteSpace::Clear() { LOG(FATAL) << "Unimplemented"; } -void ZygoteSpace::Reset() { - LOG(FATAL) << "Unimplemented"; -} - ZygoteSpace::ZygoteSpace(const std::string& name, MemMap* mem_map, size_t objects_allocated) : ContinuousMemMapAllocSpace(name, mem_map, mem_map->Begin(), mem_map->End(), mem_map->End(), kGcRetentionPolicyFullCollect), @@ -109,7 +105,7 @@ void ZygoteSpace::SweepCallback(size_t num_ptrs, mirror::Object** ptrs, void* ar // If the bitmaps aren't swapped we need to clear the bits since the GC isn't going to re-swap // the bitmaps as an optimization. if (!context->swap_bitmaps) { - accounting::SpaceBitmap* bitmap = zygote_space->GetLiveBitmap(); + accounting::ContinuousSpaceBitmap* bitmap = zygote_space->GetLiveBitmap(); for (size_t i = 0; i < num_ptrs; ++i) { bitmap->Clear(ptrs[i]); } diff --git a/runtime/gc/space/zygote_space.h b/runtime/gc/space/zygote_space.h index 8880548..50fc62b 100644 --- a/runtime/gc/space/zygote_space.h +++ b/runtime/gc/space/zygote_space.h @@ -17,16 +17,13 @@ #ifndef ART_RUNTIME_GC_SPACE_ZYGOTE_SPACE_H_ #define ART_RUNTIME_GC_SPACE_ZYGOTE_SPACE_H_ +#include "gc/accounting/space_bitmap.h" #include "malloc_space.h" #include "mem_map.h" namespace art { namespace gc { -namespace accounting { -class SpaceBitmap; -} - namespace space { // An zygote space is a space which you cannot allocate into or free from. @@ -34,8 +31,8 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { public: // Returns the remaining storage in the out_map field. static ZygoteSpace* Create(const std::string& name, MemMap* mem_map, - accounting::SpaceBitmap* live_bitmap, - accounting::SpaceBitmap* mark_bitmap) + accounting::ContinuousSpaceBitmap* live_bitmap, + accounting::ContinuousSpaceBitmap* mark_bitmap) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); void Dump(std::ostream& os) const; @@ -72,10 +69,13 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { } void Clear() OVERRIDE; - void Reset() OVERRIDE; + + bool CanMoveObjects() const OVERRIDE { + return false; + } protected: - virtual accounting::SpaceBitmap::SweepCallback* GetSweepCallback() { + virtual accounting::ContinuousSpaceBitmap::SweepCallback* GetSweepCallback() { return &SweepCallback; } diff --git a/runtime/globals.h b/runtime/globals.h index 7e85231..e3c54b8 100644 --- a/runtime/globals.h +++ b/runtime/globals.h @@ -50,9 +50,6 @@ static constexpr size_t kWordHighBitMask = static_cast<size_t>(1) << (kBitsPerWo // Required stack alignment static constexpr size_t kStackAlignment = 16; -// Required object alignment -static constexpr size_t kObjectAlignment = 8; - // ARM instruction alignment. ARM processors require code to be 4-byte aligned, // but ARM ELF requires 8.. static constexpr size_t kArmAlignment = 8; @@ -72,6 +69,10 @@ static constexpr size_t kX86Alignment = 16; // compile-time constant so the compiler can generate better code. static constexpr int kPageSize = 4096; +// Required object alignment +static constexpr size_t kObjectAlignment = 8; +static constexpr size_t kLargeObjectAlignment = kPageSize; + // Whether or not this is a debug build. Useful in conditionals where NDEBUG isn't. #if defined(NDEBUG) static constexpr bool kIsDebugBuild = false; diff --git a/runtime/jdwp/jdwp_main.cc b/runtime/jdwp/jdwp_main.cc index 5fc0228..8e22c1d 100644 --- a/runtime/jdwp/jdwp_main.cc +++ b/runtime/jdwp/jdwp_main.cc @@ -237,55 +237,41 @@ JdwpState* JdwpState::Create(const JdwpOptions* options) { Locks::mutator_lock_->AssertNotHeld(self); UniquePtr<JdwpState> state(new JdwpState(options)); switch (options->transport) { - case kJdwpTransportSocket: - InitSocketTransport(state.get(), options); - break; + case kJdwpTransportSocket: + InitSocketTransport(state.get(), options); + break; #ifdef HAVE_ANDROID_OS - case kJdwpTransportAndroidAdb: - InitAdbTransport(state.get(), options); - break; + case kJdwpTransportAndroidAdb: + InitAdbTransport(state.get(), options); + break; #endif - default: - LOG(FATAL) << "Unknown transport: " << options->transport; + default: + LOG(FATAL) << "Unknown transport: " << options->transport; } - if (!options->suspend) { + { /* * Grab a mutex before starting the thread. This ensures they * won't signal the cond var before we're waiting. */ MutexLock thread_start_locker(self, state->thread_start_lock_); + /* * We have bound to a port, or are trying to connect outbound to a * debugger. Create the JDWP thread and let it continue the mission. */ - CHECK_PTHREAD_CALL(pthread_create, (&state->pthread_, NULL, StartJdwpThread, state.get()), "JDWP thread"); + CHECK_PTHREAD_CALL(pthread_create, (&state->pthread_, nullptr, StartJdwpThread, state.get()), + "JDWP thread"); /* * Wait until the thread finishes basic initialization. - * TODO: cond vars should be waited upon in a loop */ - state->thread_start_cond_.Wait(self); - } else { - { - /* - * Grab a mutex before starting the thread. This ensures they - * won't signal the cond var before we're waiting. - */ - MutexLock thread_start_locker(self, state->thread_start_lock_); - /* - * We have bound to a port, or are trying to connect outbound to a - * debugger. Create the JDWP thread and let it continue the mission. - */ - CHECK_PTHREAD_CALL(pthread_create, (&state->pthread_, NULL, StartJdwpThread, state.get()), "JDWP thread"); - - /* - * Wait until the thread finishes basic initialization. - * TODO: cond vars should be waited upon in a loop - */ + while (!state->debug_thread_started_) { state->thread_start_cond_.Wait(self); } + } + if (options->suspend) { /* * For suspend=y, wait for the debugger to connect to us or for us to * connect to the debugger. @@ -481,11 +467,8 @@ void JdwpState::Run() { /* process requests until the debugger drops */ bool first = true; while (!Dbg::IsDisposed()) { - { - // sanity check -- shouldn't happen? - MutexLock mu(thread_, *Locks::thread_suspend_count_lock_); - CHECK_EQ(thread_->GetState(), kWaitingInMainDebuggerLoop); - } + // sanity check -- shouldn't happen? + CHECK_EQ(thread_->GetState(), kWaitingInMainDebuggerLoop); if (!netState->ProcessIncoming()) { /* blocking read */ diff --git a/runtime/parsed_options.cc b/runtime/parsed_options.cc index bc8f51f..29fe536 100644 --- a/runtime/parsed_options.cc +++ b/runtime/parsed_options.cc @@ -584,10 +584,10 @@ bool ParsedOptions::Parse(const Runtime::Options& options, bool ignore_unrecogni Usage("Unknown -Xverify option %s", verify_mode.c_str()); return false; } - } else if (StartsWith(option, "-ea:") || - StartsWith(option, "-da:") || - StartsWith(option, "-enableassertions:") || - StartsWith(option, "-disableassertions:") || + } else if (StartsWith(option, "-ea") || + StartsWith(option, "-da") || + StartsWith(option, "-enableassertions") || + StartsWith(option, "-disableassertions") || (option == "--runtime-arg") || (option == "-esa") || (option == "-dsa") || diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc index 8dad419..270deb0 100644 --- a/runtime/thread_list.cc +++ b/runtime/thread_list.cc @@ -617,7 +617,7 @@ void ThreadList::SuspendSelfForDebugger() { DCHECK(pReq != NULL); if (pReq->invoke_needed) { // Clear this before signaling. - pReq->invoke_needed = false; + pReq->Clear(); VLOG(jdwp) << "invoke complete, signaling"; MutexLock mu(self, pReq->lock); |