diff options
Diffstat (limited to 'runtime/gc/space')
22 files changed, 348 insertions, 157 deletions
diff --git a/runtime/gc/space/bump_pointer_space-inl.h b/runtime/gc/space/bump_pointer_space-inl.h index 9f1f953..14a93d1 100644 --- a/runtime/gc/space/bump_pointer_space-inl.h +++ b/runtime/gc/space/bump_pointer_space-inl.h @@ -24,7 +24,8 @@ namespace gc { namespace space { inline mirror::Object* BumpPointerSpace::Alloc(Thread*, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { num_bytes = RoundUp(num_bytes, kAlignment); mirror::Object* ret = AllocNonvirtual(num_bytes); if (LIKELY(ret != nullptr)) { @@ -32,13 +33,15 @@ inline mirror::Object* BumpPointerSpace::Alloc(Thread*, size_t num_bytes, size_t if (usable_size != nullptr) { *usable_size = num_bytes; } + *bytes_tl_bulk_allocated = num_bytes; } return ret; } inline mirror::Object* BumpPointerSpace::AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { Locks::mutator_lock_->AssertExclusiveHeld(self); num_bytes = RoundUp(num_bytes, kAlignment); uint8_t* end = end_.LoadRelaxed(); @@ -54,6 +57,7 @@ inline mirror::Object* BumpPointerSpace::AllocThreadUnsafe(Thread* self, size_t if (UNLIKELY(usable_size != nullptr)) { *usable_size = num_bytes; } + *bytes_tl_bulk_allocated = num_bytes; return obj; } diff --git a/runtime/gc/space/bump_pointer_space.cc b/runtime/gc/space/bump_pointer_space.cc index fbfc449..1303d77 100644 --- a/runtime/gc/space/bump_pointer_space.cc +++ b/runtime/gc/space/bump_pointer_space.cc @@ -93,12 +93,13 @@ mirror::Object* BumpPointerSpace::GetNextObject(mirror::Object* obj) { return reinterpret_cast<mirror::Object*>(RoundUp(position, kAlignment)); } -void BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) { +size_t BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) { MutexLock mu(Thread::Current(), block_lock_); RevokeThreadLocalBuffersLocked(thread); + return 0U; } -void BumpPointerSpace::RevokeAllThreadLocalBuffers() { +size_t BumpPointerSpace::RevokeAllThreadLocalBuffers() { Thread* self = Thread::Current(); MutexLock mu(self, *Locks::runtime_shutdown_lock_); MutexLock mu2(self, *Locks::thread_list_lock_); @@ -107,6 +108,7 @@ void BumpPointerSpace::RevokeAllThreadLocalBuffers() { for (Thread* thread : thread_list) { RevokeThreadLocalBuffers(thread); } + return 0U; } void BumpPointerSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { diff --git a/runtime/gc/space/bump_pointer_space.h b/runtime/gc/space/bump_pointer_space.h index 089ede4..c496a42 100644 --- a/runtime/gc/space/bump_pointer_space.h +++ b/runtime/gc/space/bump_pointer_space.h @@ -47,10 +47,10 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { // Allocate num_bytes, returns nullptr if the space is full. mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; // Thread-unsafe allocation for when mutators are suspended, used by the semispace collector. mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); mirror::Object* AllocNonvirtual(size_t num_bytes); @@ -103,9 +103,9 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { void Dump(std::ostream& os) const; - void RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(block_lock_); - void RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, - Locks::thread_list_lock_); + size_t RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(block_lock_); + size_t RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, + Locks::thread_list_lock_); void AssertThreadLocalBuffersAreRevoked(Thread* thread) LOCKS_EXCLUDED(block_lock_); void AssertAllThreadLocalBuffersAreRevoked() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_); diff --git a/runtime/gc/space/dlmalloc_space-inl.h b/runtime/gc/space/dlmalloc_space-inl.h index 4c8a35e..9eace89 100644 --- a/runtime/gc/space/dlmalloc_space-inl.h +++ b/runtime/gc/space/dlmalloc_space-inl.h @@ -27,11 +27,13 @@ namespace space { inline mirror::Object* DlMallocSpace::AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* obj; { MutexLock mu(self, lock_); - obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size); + obj = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } if (LIKELY(obj != NULL)) { // Zero freshly allocated memory, done while not holding the space's lock. @@ -49,9 +51,11 @@ inline size_t DlMallocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_ return size + kChunkOverhead; } -inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t num_bytes, - size_t* bytes_allocated, - size_t* usable_size) { +inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked( + Thread* /*self*/, size_t num_bytes, + size_t* bytes_allocated, + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* result = reinterpret_cast<mirror::Object*>(mspace_malloc(mspace_, num_bytes)); if (LIKELY(result != NULL)) { if (kDebugSpaces) { @@ -61,6 +65,7 @@ inline mirror::Object* DlMallocSpace::AllocWithoutGrowthLocked(Thread* /*self*/, size_t allocation_size = AllocationSizeNonvirtual(result, usable_size); DCHECK(bytes_allocated != NULL); *bytes_allocated = allocation_size; + *bytes_tl_bulk_allocated = allocation_size; } return result; } diff --git a/runtime/gc/space/dlmalloc_space.cc b/runtime/gc/space/dlmalloc_space.cc index b8a9dd6..225861d 100644 --- a/runtime/gc/space/dlmalloc_space.cc +++ b/runtime/gc/space/dlmalloc_space.cc @@ -123,7 +123,8 @@ void* DlMallocSpace::CreateMspace(void* begin, size_t morecore_start, size_t ini } mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* result; { MutexLock mu(self, lock_); @@ -131,7 +132,8 @@ mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, size_t max_allowed = Capacity(); mspace_set_footprint_limit(mspace_, max_allowed); // Try the allocation. - result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size); + result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); // Shrink back down as small as possible. size_t footprint = mspace_footprint(mspace_); mspace_set_footprint_limit(mspace_, footprint); diff --git a/runtime/gc/space/dlmalloc_space.h b/runtime/gc/space/dlmalloc_space.h index 6ce138c..1f80f1f 100644 --- a/runtime/gc/space/dlmalloc_space.h +++ b/runtime/gc/space/dlmalloc_space.h @@ -48,11 +48,15 @@ class DlMallocSpace : public MallocSpace { // Virtual to allow ValgrindMallocSpace to intercept. virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_); + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) + OVERRIDE LOCKS_EXCLUDED(lock_); // Virtual to allow ValgrindMallocSpace to intercept. virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_) { - return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE LOCKS_EXCLUDED(lock_) { + return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } // Virtual to allow ValgrindMallocSpace to intercept. virtual size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE { @@ -67,15 +71,22 @@ class DlMallocSpace : public MallocSpace { LOCKS_EXCLUDED(lock_) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE { + return num_bytes; + } + // DlMallocSpaces don't have thread local state. - void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + return 0U; } - void RevokeAllThreadLocalBuffers() OVERRIDE { + size_t RevokeAllThreadLocalBuffers() OVERRIDE { + return 0U; } // Faster non-virtual allocation path. mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) LOCKS_EXCLUDED(lock_); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + LOCKS_EXCLUDED(lock_); // Faster non-virtual allocation size path. size_t AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size); @@ -134,7 +145,8 @@ class DlMallocSpace : public MallocSpace { private: mirror::Object* AllocWithoutGrowthLocked(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) EXCLUSIVE_LOCKS_REQUIRED(lock_); void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size, diff --git a/runtime/gc/space/large_object_space.cc b/runtime/gc/space/large_object_space.cc index 7523de5..5c8e4b9 100644 --- a/runtime/gc/space/large_object_space.cc +++ b/runtime/gc/space/large_object_space.cc @@ -38,10 +38,11 @@ class ValgrindLargeObjectMapSpace FINAL : public LargeObjectMapSpace { } virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE { + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE { mirror::Object* obj = LargeObjectMapSpace::Alloc(self, num_bytes + kValgrindRedZoneBytes * 2, bytes_allocated, - usable_size); + usable_size, bytes_tl_bulk_allocated); mirror::Object* object_without_rdz = reinterpret_cast<mirror::Object*>( reinterpret_cast<uintptr_t>(obj) + kValgrindRedZoneBytes); VALGRIND_MAKE_MEM_NOACCESS(reinterpret_cast<void*>(obj), kValgrindRedZoneBytes); @@ -108,7 +109,8 @@ LargeObjectMapSpace* LargeObjectMapSpace::Create(const std::string& name) { } mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { std::string error_msg; MemMap* mem_map = MemMap::MapAnonymous("large object space allocation", nullptr, num_bytes, PROT_READ | PROT_WRITE, true, false, &error_msg); @@ -131,6 +133,8 @@ mirror::Object* LargeObjectMapSpace::Alloc(Thread* self, size_t num_bytes, if (usable_size != nullptr) { *usable_size = allocation_size; } + DCHECK(bytes_tl_bulk_allocated != nullptr); + *bytes_tl_bulk_allocated = allocation_size; num_bytes_allocated_ += allocation_size; total_bytes_allocated_ += allocation_size; ++num_objects_allocated_; @@ -413,7 +417,7 @@ size_t FreeListSpace::AllocationSize(mirror::Object* obj, size_t* usable_size) { } mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, size_t* bytes_tl_bulk_allocated) { MutexLock mu(self, lock_); const size_t allocation_size = RoundUp(num_bytes, kAlignment); AllocationInfo temp_info; @@ -451,6 +455,8 @@ mirror::Object* FreeListSpace::Alloc(Thread* self, size_t num_bytes, size_t* byt if (usable_size != nullptr) { *usable_size = allocation_size; } + DCHECK(bytes_tl_bulk_allocated != nullptr); + *bytes_tl_bulk_allocated = allocation_size; // Need to do these inside of the lock. ++num_objects_allocated_; ++total_objects_allocated_; diff --git a/runtime/gc/space/large_object_space.h b/runtime/gc/space/large_object_space.h index 847f575..d1f9386 100644 --- a/runtime/gc/space/large_object_space.h +++ b/runtime/gc/space/large_object_space.h @@ -62,9 +62,11 @@ class LargeObjectSpace : public DiscontinuousSpace, public AllocSpace { } size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) OVERRIDE; // LargeObjectSpaces don't have thread local state. - void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + return 0U; } - void RevokeAllThreadLocalBuffers() OVERRIDE { + size_t RevokeAllThreadLocalBuffers() OVERRIDE { + return 0U; } bool IsAllocSpace() const OVERRIDE { return true; @@ -124,7 +126,7 @@ class LargeObjectMapSpace : public LargeObjectSpace { // Return the storage space required by obj. size_t AllocationSize(mirror::Object* obj, size_t* usable_size); mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated); size_t Free(Thread* self, mirror::Object* ptr); void Walk(DlMallocSpace::WalkCallback, void* arg) OVERRIDE LOCKS_EXCLUDED(lock_); // TODO: disabling thread safety analysis as this may be called when we already hold lock_. @@ -153,7 +155,7 @@ class FreeListSpace FINAL : public LargeObjectSpace { size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(lock_); mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; size_t Free(Thread* self, mirror::Object* obj) OVERRIDE; void Walk(DlMallocSpace::WalkCallback callback, void* arg) OVERRIDE LOCKS_EXCLUDED(lock_); void Dump(std::ostream& os) const; diff --git a/runtime/gc/space/large_object_space_test.cc b/runtime/gc/space/large_object_space_test.cc index e17bad8..a261663 100644 --- a/runtime/gc/space/large_object_space_test.cc +++ b/runtime/gc/space/large_object_space_test.cc @@ -49,11 +49,13 @@ void LargeObjectSpaceTest::LargeObjectTest() { while (requests.size() < num_allocations) { size_t request_size = test_rand(&rand_seed) % max_allocation_size; size_t allocation_size = 0; + size_t bytes_tl_bulk_allocated; mirror::Object* obj = los->Alloc(Thread::Current(), request_size, &allocation_size, - nullptr); + nullptr, &bytes_tl_bulk_allocated); ASSERT_TRUE(obj != nullptr); ASSERT_EQ(allocation_size, los->AllocationSize(obj, nullptr)); ASSERT_GE(allocation_size, request_size); + ASSERT_EQ(allocation_size, bytes_tl_bulk_allocated); // Fill in our magic value. uint8_t magic = (request_size & 0xFF) | 1; memset(obj, magic, request_size); @@ -83,9 +85,10 @@ void LargeObjectSpaceTest::LargeObjectTest() { // Test that dump doesn't crash. los->Dump(LOG(INFO)); - size_t bytes_allocated = 0; + size_t bytes_allocated = 0, bytes_tl_bulk_allocated; // Checks that the coalescing works. - mirror::Object* obj = los->Alloc(Thread::Current(), 100 * MB, &bytes_allocated, nullptr); + mirror::Object* obj = los->Alloc(Thread::Current(), 100 * MB, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated); EXPECT_TRUE(obj != nullptr); los->Free(Thread::Current(), obj); @@ -102,8 +105,9 @@ class AllocRaceTask : public Task { void Run(Thread* self) { for (size_t i = 0; i < iterations_ ; ++i) { - size_t alloc_size; - mirror::Object* ptr = los_->Alloc(self, size_, &alloc_size, nullptr); + size_t alloc_size, bytes_tl_bulk_allocated; + mirror::Object* ptr = los_->Alloc(self, size_, &alloc_size, nullptr, + &bytes_tl_bulk_allocated); NanoSleep((id_ + 3) * 1000); // (3+id) mu s diff --git a/runtime/gc/space/malloc_space.h b/runtime/gc/space/malloc_space.h index 06239e5..bbf1bbb 100644 --- a/runtime/gc/space/malloc_space.h +++ b/runtime/gc/space/malloc_space.h @@ -55,10 +55,11 @@ class MallocSpace : public ContinuousMemMapAllocSpace { // Allocate num_bytes allowing the underlying space to grow. virtual mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) = 0; + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) = 0; // Allocate num_bytes without allowing the underlying space to grow. virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) = 0; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) = 0; // Return the storage space required by obj. If usable_size isn't nullptr then it is set to the // amount of the storage space that may be used by obj. virtual size_t AllocationSize(mirror::Object* obj, size_t* usable_size) = 0; @@ -67,6 +68,11 @@ class MallocSpace : public ContinuousMemMapAllocSpace { virtual size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) = 0; + // Returns the maximum bytes that could be allocated for the given + // size in bulk, that is the maximum value for the + // bytes_allocated_bulk out param returned by MallocSpace::Alloc(). + virtual size_t MaxBytesBulkAllocatedFor(size_t num_bytes) = 0; + #ifndef NDEBUG virtual void CheckMoreCoreForPrecondition() {} // to be overridden in the debug build. #else diff --git a/runtime/gc/space/region_space-inl.h b/runtime/gc/space/region_space-inl.h index a4ed718..1cdf69d 100644 --- a/runtime/gc/space/region_space-inl.h +++ b/runtime/gc/space/region_space-inl.h @@ -24,30 +24,36 @@ namespace gc { namespace space { inline mirror::Object* RegionSpace::Alloc(Thread*, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { num_bytes = RoundUp(num_bytes, kAlignment); - return AllocNonvirtual<false>(num_bytes, bytes_allocated, usable_size); + return AllocNonvirtual<false>(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } inline mirror::Object* RegionSpace::AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { Locks::mutator_lock_->AssertExclusiveHeld(self); - return Alloc(self, num_bytes, bytes_allocated, usable_size); + return Alloc(self, num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); } template<bool kForEvac> inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { DCHECK(IsAligned<kAlignment>(num_bytes)); mirror::Object* obj; if (LIKELY(num_bytes <= kRegionSize)) { // Non-large object. if (!kForEvac) { - obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } else { DCHECK(evac_region_ != nullptr); - obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } if (LIKELY(obj != nullptr)) { return obj; @@ -55,9 +61,11 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by MutexLock mu(Thread::Current(), region_lock_); // Retry with current region since another thread may have updated it. if (!kForEvac) { - obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = current_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } else { - obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size); + obj = evac_region_->Alloc(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } if (LIKELY(obj != nullptr)) { return obj; @@ -73,7 +81,7 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by r->Unfree(time_); r->SetNewlyAllocated(); ++num_non_free_regions_; - obj = r->Alloc(num_bytes, bytes_allocated, usable_size); + obj = r->Alloc(num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); CHECK(obj != nullptr); current_region_ = r; return obj; @@ -85,7 +93,7 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by if (r->IsFree()) { r->Unfree(time_); ++num_non_free_regions_; - obj = r->Alloc(num_bytes, bytes_allocated, usable_size); + obj = r->Alloc(num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); CHECK(obj != nullptr); evac_region_ = r; return obj; @@ -94,7 +102,8 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by } } else { // Large object. - obj = AllocLarge<kForEvac>(num_bytes, bytes_allocated, usable_size); + obj = AllocLarge<kForEvac>(num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (LIKELY(obj != nullptr)) { return obj; } @@ -103,7 +112,8 @@ inline mirror::Object* RegionSpace::AllocNonvirtual(size_t num_bytes, size_t* by } inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { DCHECK(IsAllocated() && IsInToSpace()); DCHECK(IsAligned<kAlignment>(num_bytes)); Atomic<uint8_t*>* atomic_top = reinterpret_cast<Atomic<uint8_t*>*>(&top_); @@ -124,6 +134,7 @@ inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, size_t* byte if (usable_size != nullptr) { *usable_size = num_bytes; } + *bytes_tl_bulk_allocated = num_bytes; return reinterpret_cast<mirror::Object*>(old_top); } @@ -253,7 +264,8 @@ inline mirror::Object* RegionSpace::GetNextObject(mirror::Object* obj) { template<bool kForEvac> mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { DCHECK(IsAligned<kAlignment>(num_bytes)); DCHECK_GT(num_bytes, kRegionSize); size_t num_regs = RoundUp(num_bytes, kRegionSize) / kRegionSize; @@ -300,6 +312,7 @@ mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, size_t* bytes_allocate if (usable_size != nullptr) { *usable_size = num_regs * kRegionSize; } + *bytes_tl_bulk_allocated = num_bytes; return reinterpret_cast<mirror::Object*>(first_reg->Begin()); } else { // right points to the non-free region. Start with the one after it. diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc index 8bb73d6..814ab6c 100644 --- a/runtime/gc/space/region_space.cc +++ b/runtime/gc/space/region_space.cc @@ -76,7 +76,7 @@ RegionSpace::RegionSpace(const std::string& name, MemMap* mem_map) current_region_ = &full_region_; evac_region_ = nullptr; size_t ignored; - DCHECK(full_region_.Alloc(kAlignment, &ignored, nullptr) == nullptr); + DCHECK(full_region_.Alloc(kAlignment, &ignored, nullptr, &ignored) == nullptr); } size_t RegionSpace::FromSpaceSize() { @@ -356,9 +356,10 @@ bool RegionSpace::AllocNewTlab(Thread* self) { return false; } -void RegionSpace::RevokeThreadLocalBuffers(Thread* thread) { +size_t RegionSpace::RevokeThreadLocalBuffers(Thread* thread) { MutexLock mu(Thread::Current(), region_lock_); RevokeThreadLocalBuffersLocked(thread); + return 0U; } void RegionSpace::RevokeThreadLocalBuffersLocked(Thread* thread) { @@ -377,7 +378,7 @@ void RegionSpace::RevokeThreadLocalBuffersLocked(Thread* thread) { thread->SetTlab(nullptr, nullptr); } -void RegionSpace::RevokeAllThreadLocalBuffers() { +size_t RegionSpace::RevokeAllThreadLocalBuffers() { Thread* self = Thread::Current(); MutexLock mu(self, *Locks::runtime_shutdown_lock_); MutexLock mu2(self, *Locks::thread_list_lock_); @@ -385,6 +386,7 @@ void RegionSpace::RevokeAllThreadLocalBuffers() { for (Thread* thread : thread_list) { RevokeThreadLocalBuffers(thread); } + return 0U; } void RegionSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h index 4160547..b88ce24 100644 --- a/runtime/gc/space/region_space.h +++ b/runtime/gc/space/region_space.h @@ -42,18 +42,20 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { // Allocate num_bytes, returns nullptr if the space is full. mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; // Thread-unsafe allocation for when mutators are suspended, used by the semispace collector. mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); // The main allocation routine. template<bool kForEvac> ALWAYS_INLINE mirror::Object* AllocNonvirtual(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, + size_t* bytes_tl_bulk_allocated); // Allocate/free large objects (objects that are larger than the region size.) template<bool kForEvac> - mirror::Object* AllocLarge(size_t num_bytes, size_t* bytes_allocated, size_t* usable_size); + mirror::Object* AllocLarge(size_t num_bytes, size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated); void FreeLarge(mirror::Object* large_obj, size_t bytes_allocated); // Return the storage space required by obj. @@ -87,10 +89,10 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { void DumpRegions(std::ostream& os); void DumpNonFreeRegions(std::ostream& os); - void RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(region_lock_); + size_t RevokeThreadLocalBuffers(Thread* thread) LOCKS_EXCLUDED(region_lock_); void RevokeThreadLocalBuffersLocked(Thread* thread) EXCLUSIVE_LOCKS_REQUIRED(region_lock_); - void RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, - Locks::thread_list_lock_); + size_t RevokeAllThreadLocalBuffers() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, + Locks::thread_list_lock_); void AssertThreadLocalBuffersAreRevoked(Thread* thread) LOCKS_EXCLUDED(region_lock_); void AssertAllThreadLocalBuffersAreRevoked() LOCKS_EXCLUDED(Locks::runtime_shutdown_lock_, Locks::thread_list_lock_); @@ -269,7 +271,8 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { } ALWAYS_INLINE mirror::Object* Alloc(size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, + size_t* bytes_tl_bulk_allocated); bool IsFree() const { bool is_free = state_ == RegionState::kRegionStateFree; diff --git a/runtime/gc/space/rosalloc_space-inl.h b/runtime/gc/space/rosalloc_space-inl.h index 5d6642d..9d582a3 100644 --- a/runtime/gc/space/rosalloc_space-inl.h +++ b/runtime/gc/space/rosalloc_space-inl.h @@ -26,13 +26,19 @@ namespace art { namespace gc { namespace space { +template<bool kMaybeRunningOnValgrind> inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { // obj is a valid object. Use its class in the header to get the size. // Don't use verification since the object may be dead if we are sweeping. size_t size = obj->SizeOf<kVerifyNone>(); - bool running_on_valgrind = RUNNING_ON_VALGRIND != 0; - if (running_on_valgrind) { - size += 2 * kDefaultValgrindRedZoneBytes; + bool running_on_valgrind = false; + if (kMaybeRunningOnValgrind) { + running_on_valgrind = RUNNING_ON_VALGRIND != 0; + if (running_on_valgrind) { + size += 2 * kDefaultValgrindRedZoneBytes; + } + } else { + DCHECK_EQ(RUNNING_ON_VALGRIND, 0U); } size_t size_by_size = rosalloc_->UsableSize(size); if (kIsDebugBuild) { @@ -55,28 +61,50 @@ inline size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_ template<bool kThreadSafe> inline mirror::Object* RosAllocSpace::AllocCommon(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { - size_t rosalloc_size = 0; + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { + size_t rosalloc_bytes_allocated = 0; + size_t rosalloc_usable_size = 0; + size_t rosalloc_bytes_tl_bulk_allocated = 0; if (!kThreadSafe) { Locks::mutator_lock_->AssertExclusiveHeld(self); } mirror::Object* result = reinterpret_cast<mirror::Object*>( - rosalloc_->Alloc<kThreadSafe>(self, num_bytes, &rosalloc_size)); + rosalloc_->Alloc<kThreadSafe>(self, num_bytes, &rosalloc_bytes_allocated, + &rosalloc_usable_size, + &rosalloc_bytes_tl_bulk_allocated)); if (LIKELY(result != NULL)) { if (kDebugSpaces) { CHECK(Contains(result)) << "Allocation (" << reinterpret_cast<void*>(result) << ") not in bounds of allocation space " << *this; } DCHECK(bytes_allocated != NULL); - *bytes_allocated = rosalloc_size; - DCHECK_EQ(rosalloc_size, rosalloc_->UsableSize(result)); + *bytes_allocated = rosalloc_bytes_allocated; + DCHECK_EQ(rosalloc_usable_size, rosalloc_->UsableSize(result)); if (usable_size != nullptr) { - *usable_size = rosalloc_size; + *usable_size = rosalloc_usable_size; } + DCHECK(bytes_tl_bulk_allocated != NULL); + *bytes_tl_bulk_allocated = rosalloc_bytes_tl_bulk_allocated; } return result; } +inline bool RosAllocSpace::CanAllocThreadLocal(Thread* self, size_t num_bytes) { + return rosalloc_->CanAllocFromThreadLocalRun(self, num_bytes); +} + +inline mirror::Object* RosAllocSpace::AllocThreadLocal(Thread* self, size_t num_bytes, + size_t* bytes_allocated) { + DCHECK(bytes_allocated != nullptr); + return reinterpret_cast<mirror::Object*>( + rosalloc_->AllocFromThreadLocalRun(self, num_bytes, bytes_allocated)); +} + +inline size_t RosAllocSpace::MaxBytesBulkAllocatedForNonvirtual(size_t num_bytes) { + return rosalloc_->MaxBytesBulkAllocatedFor(num_bytes); +} + } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/rosalloc_space.cc b/runtime/gc/space/rosalloc_space.cc index ced25a4..f140021 100644 --- a/runtime/gc/space/rosalloc_space.cc +++ b/runtime/gc/space/rosalloc_space.cc @@ -154,7 +154,8 @@ allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_ } mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { mirror::Object* result; { MutexLock mu(self, lock_); @@ -162,7 +163,8 @@ mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, size_t max_allowed = Capacity(); rosalloc_->SetFootprintLimit(max_allowed); // Try the allocation. - result = AllocCommon(self, num_bytes, bytes_allocated, usable_size); + result = AllocCommon(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); // Shrink back down as small as possible. size_t footprint = rosalloc_->Footprint(); rosalloc_->SetFootprintLimit(footprint); @@ -209,7 +211,7 @@ size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** p __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead])); } if (kVerifyFreedBytes) { - verify_bytes += AllocationSizeNonvirtual(ptrs[i], nullptr); + verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr); } } @@ -338,12 +340,12 @@ void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, } } -void RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) { - rosalloc_->RevokeThreadLocalRuns(thread); +size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) { + return rosalloc_->RevokeThreadLocalRuns(thread); } -void RosAllocSpace::RevokeAllThreadLocalBuffers() { - rosalloc_->RevokeAllThreadLocalRuns(); +size_t RosAllocSpace::RevokeAllThreadLocalBuffers() { + return rosalloc_->RevokeAllThreadLocalRuns(); } void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { diff --git a/runtime/gc/space/rosalloc_space.h b/runtime/gc/space/rosalloc_space.h index c856e95..36268f7 100644 --- a/runtime/gc/space/rosalloc_space.h +++ b/runtime/gc/space/rosalloc_space.h @@ -47,18 +47,21 @@ class RosAllocSpace : public MallocSpace { bool low_memory_mode, bool can_move_objects); mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE LOCKS_EXCLUDED(lock_); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE LOCKS_EXCLUDED(lock_); mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE { - return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE { + return AllocNonvirtual(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) { - return AllocNonvirtualThreadUnsafe(self, num_bytes, bytes_allocated, usable_size); + return AllocNonvirtualThreadUnsafe(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE { - return AllocationSizeNonvirtual(obj, usable_size); + return AllocationSizeNonvirtual<true>(obj, usable_size); } size_t Free(Thread* self, mirror::Object* ptr) OVERRIDE SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); @@ -66,17 +69,33 @@ class RosAllocSpace : public MallocSpace { SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); mirror::Object* AllocNonvirtual(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) { + size_t* usable_size, size_t* bytes_tl_bulk_allocated) { // RosAlloc zeroes memory internally. - return AllocCommon(self, num_bytes, bytes_allocated, usable_size); + return AllocCommon(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } mirror::Object* AllocNonvirtualThreadUnsafe(Thread* self, size_t num_bytes, - size_t* bytes_allocated, size_t* usable_size) { + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) { // RosAlloc zeroes memory internally. Pass in false for thread unsafe. - return AllocCommon<false>(self, num_bytes, bytes_allocated, usable_size); + return AllocCommon<false>(self, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); } + // Returns true if the given allocation request can be allocated in + // an existing thread local run without allocating a new run. + ALWAYS_INLINE bool CanAllocThreadLocal(Thread* self, size_t num_bytes); + // Allocate the given allocation request in an existing thread local + // run without allocating a new run. + ALWAYS_INLINE mirror::Object* AllocThreadLocal(Thread* self, size_t num_bytes, + size_t* bytes_allocated); + size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE { + return MaxBytesBulkAllocatedForNonvirtual(num_bytes); + } + ALWAYS_INLINE size_t MaxBytesBulkAllocatedForNonvirtual(size_t num_bytes); + // TODO: NO_THREAD_SAFETY_ANALYSIS because SizeOf() requires that mutator_lock is held. + template<bool kMaybeRunningOnValgrind> size_t AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) NO_THREAD_SAFETY_ANALYSIS; @@ -99,8 +118,8 @@ class RosAllocSpace : public MallocSpace { uint64_t GetBytesAllocated() OVERRIDE; uint64_t GetObjectsAllocated() OVERRIDE; - void RevokeThreadLocalBuffers(Thread* thread); - void RevokeAllThreadLocalBuffers(); + size_t RevokeThreadLocalBuffers(Thread* thread); + size_t RevokeAllThreadLocalBuffers(); void AssertThreadLocalBuffersAreRevoked(Thread* thread); void AssertAllThreadLocalBuffersAreRevoked(); @@ -134,7 +153,7 @@ class RosAllocSpace : public MallocSpace { private: template<bool kThreadSafe = true> mirror::Object* AllocCommon(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size); + size_t* usable_size, size_t* bytes_tl_bulk_allocated); void* CreateAllocator(void* base, size_t morecore_start, size_t initial_size, size_t maximum_size, bool low_memory_mode) OVERRIDE { diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h index d24650b..f2378d9 100644 --- a/runtime/gc/space/space.h +++ b/runtime/gc/space/space.h @@ -203,14 +203,24 @@ class AllocSpace { // succeeds, the output parameter bytes_allocated will be set to the // actually allocated bytes which is >= num_bytes. // Alloc can be called from multiple threads at the same time and must be thread-safe. + // + // bytes_tl_bulk_allocated - bytes allocated in bulk ahead of time for a thread local allocation, + // if applicable. It can be + // 1) equal to bytes_allocated if it's not a thread local allocation, + // 2) greater than bytes_allocated if it's a thread local + // allocation that required a new buffer, or + // 3) zero if it's a thread local allocation in an existing + // buffer. + // This is what is to be added to Heap::num_bytes_allocated_. virtual mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) = 0; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) = 0; // Thread-unsafe allocation for when mutators are suspended, used by the semispace collector. virtual mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) + size_t* usable_size, + size_t* bytes_tl_bulk_allocated) EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) { - return Alloc(self, num_bytes, bytes_allocated, usable_size); + return Alloc(self, num_bytes, bytes_allocated, usable_size, bytes_tl_bulk_allocated); } // Return the storage space required by obj. @@ -224,11 +234,15 @@ class AllocSpace { // Revoke any sort of thread-local buffers that are used to speed up allocations for the given // thread, if the alloc space implementation uses any. - virtual void RevokeThreadLocalBuffers(Thread* thread) = 0; + // Returns the total free bytes in the revoked thread local runs that's to be subtracted + // from Heap::num_bytes_allocated_ or zero if unnecessary. + virtual size_t RevokeThreadLocalBuffers(Thread* thread) = 0; // Revoke any sort of thread-local buffers that are used to speed up allocations for all the // threads, if the alloc space implementation uses any. - virtual void RevokeAllThreadLocalBuffers() = 0; + // Returns the total free bytes in the revoked thread local runs that's to be subtracted + // from Heap::num_bytes_allocated_ or zero if unnecessary. + virtual size_t RevokeAllThreadLocalBuffers() = 0; virtual void LogFragmentationAllocFailure(std::ostream& os, size_t failed_alloc_bytes) = 0; diff --git a/runtime/gc/space/space_test.h b/runtime/gc/space/space_test.h index 09d10dd..3e9e9f7 100644 --- a/runtime/gc/space/space_test.h +++ b/runtime/gc/space/space_test.h @@ -61,11 +61,13 @@ class SpaceTest : public CommonRuntimeTest { } mirror::Object* Alloc(space::MallocSpace* alloc_space, Thread* self, size_t bytes, - size_t* bytes_allocated, size_t* usable_size) + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { StackHandleScope<1> hs(self); Handle<mirror::Class> byte_array_class(hs.NewHandle(GetByteArrayClass(self))); - mirror::Object* obj = alloc_space->Alloc(self, bytes, bytes_allocated, usable_size); + mirror::Object* obj = alloc_space->Alloc(self, bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (obj != nullptr) { InstallClass(obj, byte_array_class.Get(), bytes); } @@ -73,11 +75,13 @@ class SpaceTest : public CommonRuntimeTest { } mirror::Object* AllocWithGrowth(space::MallocSpace* alloc_space, Thread* self, size_t bytes, - size_t* bytes_allocated, size_t* usable_size) + size_t* bytes_allocated, size_t* usable_size, + size_t* bytes_tl_bulk_allocated) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { StackHandleScope<1> hs(self); Handle<mirror::Class> byte_array_class(hs.NewHandle(GetByteArrayClass(self))); - mirror::Object* obj = alloc_space->AllocWithGrowth(self, bytes, bytes_allocated, usable_size); + mirror::Object* obj = alloc_space->AllocWithGrowth(self, bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated); if (obj != nullptr) { InstallClass(obj, byte_array_class.Get(), bytes); } @@ -182,34 +186,38 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { ScopedObjectAccess soa(self); // Succeeds, fits without adjusting the footprint limit. - size_t ptr1_bytes_allocated, ptr1_usable_size; + size_t ptr1_bytes_allocated, ptr1_usable_size, ptr1_bytes_tl_bulk_allocated; StackHandleScope<3> hs(soa.Self()); MutableHandle<mirror::Object> ptr1( - hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size))); + hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size, + &ptr1_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr1.Get() != nullptr); EXPECT_LE(1U * MB, ptr1_bytes_allocated); EXPECT_LE(1U * MB, ptr1_usable_size); EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated); + EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr2 == nullptr); // Succeeds, adjusts the footprint. - size_t ptr3_bytes_allocated, ptr3_usable_size; + size_t ptr3_bytes_allocated, ptr3_usable_size, ptr3_bytes_tl_bulk_allocated; MutableHandle<mirror::Object> ptr3( - hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size, + &ptr3_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr3.Get() != nullptr); EXPECT_LE(8U * MB, ptr3_bytes_allocated); EXPECT_LE(8U * MB, ptr3_usable_size); EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated); + EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr4 = space->Alloc(self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr4 == nullptr); // Also fails, requires a higher allowed footprint. - mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr5 = space->AllocWithGrowth(self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr5 == nullptr); // Release some memory. @@ -219,13 +227,15 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { EXPECT_LE(8U * MB, free3); // Succeeds, now that memory has been freed. - size_t ptr6_bytes_allocated, ptr6_usable_size; + size_t ptr6_bytes_allocated, ptr6_usable_size, ptr6_bytes_tl_bulk_allocated; Handle<mirror::Object> ptr6( - hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size, + &ptr6_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr6.Get() != nullptr); EXPECT_LE(9U * MB, ptr6_bytes_allocated); EXPECT_LE(9U * MB, ptr6_usable_size); EXPECT_LE(ptr6_usable_size, ptr6_bytes_allocated); + EXPECT_EQ(ptr6_bytes_tl_bulk_allocated, ptr6_bytes_allocated); // Final clean up. size_t free1 = space->AllocationSize(ptr1.Get(), nullptr); @@ -233,7 +243,7 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { EXPECT_LE(1U * MB, free1); // Make sure that the zygote space isn't directly at the start of the space. - EXPECT_TRUE(space->Alloc(self, 1U * MB, &dummy, nullptr) != nullptr); + EXPECT_TRUE(space->Alloc(self, 1U * MB, &dummy, nullptr, &dummy) != nullptr); gc::Heap* heap = Runtime::Current()->GetHeap(); space::Space* old_space = space; @@ -250,22 +260,26 @@ void SpaceTest::ZygoteSpaceTestBody(CreateSpaceFn create_space) { AddSpace(space, false); // Succeeds, fits without adjusting the footprint limit. - ptr1.Assign(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size)); + ptr1.Assign(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size, + &ptr1_bytes_tl_bulk_allocated)); EXPECT_TRUE(ptr1.Get() != nullptr); EXPECT_LE(1U * MB, ptr1_bytes_allocated); EXPECT_LE(1U * MB, ptr1_usable_size); EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated); + EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated); // Fails, requires a higher footprint limit. - ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr); + ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr2 == nullptr); // Succeeds, adjusts the footprint. - ptr3.Assign(AllocWithGrowth(space, self, 2 * MB, &ptr3_bytes_allocated, &ptr3_usable_size)); + ptr3.Assign(AllocWithGrowth(space, self, 2 * MB, &ptr3_bytes_allocated, &ptr3_usable_size, + &ptr3_bytes_tl_bulk_allocated)); EXPECT_TRUE(ptr3.Get() != nullptr); EXPECT_LE(2U * MB, ptr3_bytes_allocated); EXPECT_LE(2U * MB, ptr3_usable_size); EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated); + EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated); space->Free(self, ptr3.Assign(nullptr)); // Final clean up. @@ -285,34 +299,38 @@ void SpaceTest::AllocAndFreeTestBody(CreateSpaceFn create_space) { AddSpace(space); // Succeeds, fits without adjusting the footprint limit. - size_t ptr1_bytes_allocated, ptr1_usable_size; + size_t ptr1_bytes_allocated, ptr1_usable_size, ptr1_bytes_tl_bulk_allocated; StackHandleScope<3> hs(soa.Self()); MutableHandle<mirror::Object> ptr1( - hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size))); + hs.NewHandle(Alloc(space, self, 1 * MB, &ptr1_bytes_allocated, &ptr1_usable_size, + &ptr1_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr1.Get() != nullptr); EXPECT_LE(1U * MB, ptr1_bytes_allocated); EXPECT_LE(1U * MB, ptr1_usable_size); EXPECT_LE(ptr1_usable_size, ptr1_bytes_allocated); + EXPECT_EQ(ptr1_bytes_tl_bulk_allocated, ptr1_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr2 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr2 == nullptr); // Succeeds, adjusts the footprint. - size_t ptr3_bytes_allocated, ptr3_usable_size; + size_t ptr3_bytes_allocated, ptr3_usable_size, ptr3_bytes_tl_bulk_allocated; MutableHandle<mirror::Object> ptr3( - hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 8 * MB, &ptr3_bytes_allocated, &ptr3_usable_size, + &ptr3_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr3.Get() != nullptr); EXPECT_LE(8U * MB, ptr3_bytes_allocated); EXPECT_LE(8U * MB, ptr3_usable_size); EXPECT_LE(ptr3_usable_size, ptr3_bytes_allocated); + EXPECT_EQ(ptr3_bytes_tl_bulk_allocated, ptr3_bytes_allocated); // Fails, requires a higher footprint limit. - mirror::Object* ptr4 = Alloc(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr4 = Alloc(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr4 == nullptr); // Also fails, requires a higher allowed footprint. - mirror::Object* ptr5 = AllocWithGrowth(space, self, 8 * MB, &dummy, nullptr); + mirror::Object* ptr5 = AllocWithGrowth(space, self, 8 * MB, &dummy, nullptr, &dummy); EXPECT_TRUE(ptr5 == nullptr); // Release some memory. @@ -322,13 +340,15 @@ void SpaceTest::AllocAndFreeTestBody(CreateSpaceFn create_space) { EXPECT_LE(8U * MB, free3); // Succeeds, now that memory has been freed. - size_t ptr6_bytes_allocated, ptr6_usable_size; + size_t ptr6_bytes_allocated, ptr6_usable_size, ptr6_bytes_tl_bulk_allocated; Handle<mirror::Object> ptr6( - hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size))); + hs.NewHandle(AllocWithGrowth(space, self, 9 * MB, &ptr6_bytes_allocated, &ptr6_usable_size, + &ptr6_bytes_tl_bulk_allocated))); EXPECT_TRUE(ptr6.Get() != nullptr); EXPECT_LE(9U * MB, ptr6_bytes_allocated); EXPECT_LE(9U * MB, ptr6_usable_size); EXPECT_LE(ptr6_usable_size, ptr6_bytes_allocated); + EXPECT_EQ(ptr6_bytes_tl_bulk_allocated, ptr6_bytes_allocated); // Final clean up. size_t free1 = space->AllocationSize(ptr1.Get(), nullptr); @@ -348,14 +368,16 @@ void SpaceTest::AllocAndFreeListTestBody(CreateSpaceFn create_space) { // Succeeds, fits without adjusting the max allowed footprint. mirror::Object* lots_of_objects[1024]; for (size_t i = 0; i < arraysize(lots_of_objects); i++) { - size_t allocation_size, usable_size; + size_t allocation_size, usable_size, bytes_tl_bulk_allocated; size_t size_of_zero_length_byte_array = SizeOfZeroLengthByteArray(); lots_of_objects[i] = Alloc(space, self, size_of_zero_length_byte_array, &allocation_size, - &usable_size); + &usable_size, &bytes_tl_bulk_allocated); EXPECT_TRUE(lots_of_objects[i] != nullptr); size_t computed_usable_size; EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i], &computed_usable_size)); EXPECT_EQ(usable_size, computed_usable_size); + EXPECT_TRUE(bytes_tl_bulk_allocated == 0 || + bytes_tl_bulk_allocated >= allocation_size); } // Release memory. @@ -363,12 +385,15 @@ void SpaceTest::AllocAndFreeListTestBody(CreateSpaceFn create_space) { // Succeeds, fits by adjusting the max allowed footprint. for (size_t i = 0; i < arraysize(lots_of_objects); i++) { - size_t allocation_size, usable_size; - lots_of_objects[i] = AllocWithGrowth(space, self, 1024, &allocation_size, &usable_size); + size_t allocation_size, usable_size, bytes_tl_bulk_allocated; + lots_of_objects[i] = AllocWithGrowth(space, self, 1024, &allocation_size, &usable_size, + &bytes_tl_bulk_allocated); EXPECT_TRUE(lots_of_objects[i] != nullptr); size_t computed_usable_size; EXPECT_EQ(allocation_size, space->AllocationSize(lots_of_objects[i], &computed_usable_size)); EXPECT_EQ(usable_size, computed_usable_size); + EXPECT_TRUE(bytes_tl_bulk_allocated == 0 || + bytes_tl_bulk_allocated >= allocation_size); } // Release memory. @@ -425,10 +450,13 @@ void SpaceTest::SizeFootPrintGrowthLimitAndTrimBody(MallocSpace* space, intptr_t StackHandleScope<1> hs(soa.Self()); auto object(hs.NewHandle<mirror::Object>(nullptr)); size_t bytes_allocated = 0; + size_t bytes_tl_bulk_allocated; if (round <= 1) { - object.Assign(Alloc(space, self, alloc_size, &bytes_allocated, nullptr)); + object.Assign(Alloc(space, self, alloc_size, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated)); } else { - object.Assign(AllocWithGrowth(space, self, alloc_size, &bytes_allocated, nullptr)); + object.Assign(AllocWithGrowth(space, self, alloc_size, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated)); } footprint = space->GetFootprint(); EXPECT_GE(space->Size(), footprint); // invariant @@ -441,6 +469,8 @@ void SpaceTest::SizeFootPrintGrowthLimitAndTrimBody(MallocSpace* space, intptr_t } else { EXPECT_GE(allocation_size, 8u); } + EXPECT_TRUE(bytes_tl_bulk_allocated == 0 || + bytes_tl_bulk_allocated >= allocation_size); amount_allocated += allocation_size; break; } @@ -518,11 +548,13 @@ void SpaceTest::SizeFootPrintGrowthLimitAndTrimBody(MallocSpace* space, intptr_t auto large_object(hs.NewHandle<mirror::Object>(nullptr)); size_t three_quarters_space = (growth_limit / 2) + (growth_limit / 4); size_t bytes_allocated = 0; + size_t bytes_tl_bulk_allocated; if (round <= 1) { - large_object.Assign(Alloc(space, self, three_quarters_space, &bytes_allocated, nullptr)); + large_object.Assign(Alloc(space, self, three_quarters_space, &bytes_allocated, nullptr, + &bytes_tl_bulk_allocated)); } else { large_object.Assign(AllocWithGrowth(space, self, three_quarters_space, &bytes_allocated, - nullptr)); + nullptr, &bytes_tl_bulk_allocated)); } EXPECT_TRUE(large_object.Get() != nullptr); diff --git a/runtime/gc/space/valgrind_malloc_space-inl.h b/runtime/gc/space/valgrind_malloc_space-inl.h index ae8e892..bc329e1 100644 --- a/runtime/gc/space/valgrind_malloc_space-inl.h +++ b/runtime/gc/space/valgrind_malloc_space-inl.h @@ -32,10 +32,15 @@ namespace valgrind_details { template <size_t kValgrindRedZoneBytes, bool kUseObjSizeForUsable> inline mirror::Object* AdjustForValgrind(void* obj_with_rdz, size_t num_bytes, size_t bytes_allocated, size_t usable_size, - size_t* bytes_allocated_out, size_t* usable_size_out) { + size_t bytes_tl_bulk_allocated, + size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { if (bytes_allocated_out != nullptr) { *bytes_allocated_out = bytes_allocated; } + if (bytes_tl_bulk_allocated_out != nullptr) { + *bytes_tl_bulk_allocated_out = bytes_tl_bulk_allocated; + } // This cuts over-provision and is a trade-off between testing the over-provisioning code paths // vs checking overflows in the regular paths. @@ -82,20 +87,25 @@ ValgrindMallocSpace<S, kValgrindRedZoneBytes, kAdjustForRedzoneInAllocSize, kUseObjSizeForUsable>::AllocWithGrowth( - Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) { + Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { size_t bytes_allocated; size_t usable_size; + size_t bytes_tl_bulk_allocated; void* obj_with_rdz = S::AllocWithGrowth(self, num_bytes + 2 * kValgrindRedZoneBytes, - &bytes_allocated, &usable_size); + &bytes_allocated, &usable_size, + &bytes_tl_bulk_allocated); if (obj_with_rdz == nullptr) { return nullptr; } - return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, - kUseObjSizeForUsable>(obj_with_rdz, num_bytes, - bytes_allocated, usable_size, - bytes_allocated_out, - usable_size_out); + return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>( + obj_with_rdz, num_bytes, + bytes_allocated, usable_size, + bytes_tl_bulk_allocated, + bytes_allocated_out, + usable_size_out, + bytes_tl_bulk_allocated_out); } template <typename S, @@ -106,11 +116,13 @@ mirror::Object* ValgrindMallocSpace<S, kValgrindRedZoneBytes, kAdjustForRedzoneInAllocSize, kUseObjSizeForUsable>::Alloc( - Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) { + Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { size_t bytes_allocated; size_t usable_size; + size_t bytes_tl_bulk_allocated; void* obj_with_rdz = S::Alloc(self, num_bytes + 2 * kValgrindRedZoneBytes, - &bytes_allocated, &usable_size); + &bytes_allocated, &usable_size, &bytes_tl_bulk_allocated); if (obj_with_rdz == nullptr) { return nullptr; } @@ -118,8 +130,10 @@ mirror::Object* ValgrindMallocSpace<S, return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>(obj_with_rdz, num_bytes, bytes_allocated, usable_size, + bytes_tl_bulk_allocated, bytes_allocated_out, - usable_size_out); + usable_size_out, + bytes_tl_bulk_allocated_out); } template <typename S, @@ -130,20 +144,25 @@ mirror::Object* ValgrindMallocSpace<S, kValgrindRedZoneBytes, kAdjustForRedzoneInAllocSize, kUseObjSizeForUsable>::AllocThreadUnsafe( - Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out) { + Thread* self, size_t num_bytes, size_t* bytes_allocated_out, size_t* usable_size_out, + size_t* bytes_tl_bulk_allocated_out) { size_t bytes_allocated; size_t usable_size; + size_t bytes_tl_bulk_allocated; void* obj_with_rdz = S::AllocThreadUnsafe(self, num_bytes + 2 * kValgrindRedZoneBytes, - &bytes_allocated, &usable_size); + &bytes_allocated, &usable_size, + &bytes_tl_bulk_allocated); if (obj_with_rdz == nullptr) { return nullptr; } - return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, - kUseObjSizeForUsable>(obj_with_rdz, num_bytes, - bytes_allocated, usable_size, - bytes_allocated_out, - usable_size_out); + return valgrind_details::AdjustForValgrind<kValgrindRedZoneBytes, kUseObjSizeForUsable>( + obj_with_rdz, num_bytes, + bytes_allocated, usable_size, + bytes_tl_bulk_allocated, + bytes_allocated_out, + usable_size_out, + bytes_tl_bulk_allocated_out); } template <typename S, @@ -226,6 +245,17 @@ ValgrindMallocSpace<S, mem_map->Size() - initial_size); } +template <typename S, + size_t kValgrindRedZoneBytes, + bool kAdjustForRedzoneInAllocSize, + bool kUseObjSizeForUsable> +size_t ValgrindMallocSpace<S, + kValgrindRedZoneBytes, + kAdjustForRedzoneInAllocSize, + kUseObjSizeForUsable>::MaxBytesBulkAllocatedFor(size_t num_bytes) { + return S::MaxBytesBulkAllocatedFor(num_bytes + 2 * kValgrindRedZoneBytes); +} + } // namespace space } // namespace gc } // namespace art diff --git a/runtime/gc/space/valgrind_malloc_space.h b/runtime/gc/space/valgrind_malloc_space.h index 707ea69..a6b010a 100644 --- a/runtime/gc/space/valgrind_malloc_space.h +++ b/runtime/gc/space/valgrind_malloc_space.h @@ -34,12 +34,13 @@ template <typename BaseMallocSpaceType, class ValgrindMallocSpace FINAL : public BaseMallocSpaceType { public: mirror::Object* AllocWithGrowth(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE; mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; mirror::Object* AllocThreadUnsafe(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE - EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); + size_t* usable_size, size_t* bytes_tl_bulk_allocated) + OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_); size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE; @@ -53,6 +54,8 @@ class ValgrindMallocSpace FINAL : public BaseMallocSpaceType { UNUSED(ptr); } + size_t MaxBytesBulkAllocatedFor(size_t num_bytes) OVERRIDE; + template <typename... Params> explicit ValgrindMallocSpace(MemMap* mem_map, size_t initial_size, Params... params); virtual ~ValgrindMallocSpace() {} diff --git a/runtime/gc/space/zygote_space.cc b/runtime/gc/space/zygote_space.cc index a868e68..9e882a8 100644 --- a/runtime/gc/space/zygote_space.cc +++ b/runtime/gc/space/zygote_space.cc @@ -77,7 +77,7 @@ void ZygoteSpace::Dump(std::ostream& os) const { << ",name=\"" << GetName() << "\"]"; } -mirror::Object* ZygoteSpace::Alloc(Thread*, size_t, size_t*, size_t*) { +mirror::Object* ZygoteSpace::Alloc(Thread*, size_t, size_t*, size_t*, size_t*) { UNIMPLEMENTED(FATAL); UNREACHABLE(); } diff --git a/runtime/gc/space/zygote_space.h b/runtime/gc/space/zygote_space.h index 0cf4bb1..934a234 100644 --- a/runtime/gc/space/zygote_space.h +++ b/runtime/gc/space/zygote_space.h @@ -46,7 +46,7 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { } mirror::Object* Alloc(Thread* self, size_t num_bytes, size_t* bytes_allocated, - size_t* usable_size) OVERRIDE; + size_t* usable_size, size_t* bytes_tl_bulk_allocated) OVERRIDE; size_t AllocationSize(mirror::Object* obj, size_t* usable_size) OVERRIDE; @@ -55,9 +55,11 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { size_t FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) OVERRIDE; // ZygoteSpaces don't have thread local state. - void RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + size_t RevokeThreadLocalBuffers(art::Thread*) OVERRIDE { + return 0U; } - void RevokeAllThreadLocalBuffers() OVERRIDE { + size_t RevokeAllThreadLocalBuffers() OVERRIDE { + return 0U; } uint64_t GetBytesAllocated() { |