Reserve bits in the lock word for read barriers.

This prepares for the CC collector to use the standard object header
model by storing the read barrier state in the lock word.

Bug: 19355854
Bug: 12687968
Change-Id: Ia7585662dd2cebf0479a3e74f734afe5059fb70f

1 files changed, 102 insertions, 27 deletions

diff --git a/runtime/lock_word.h b/runtime/lock_word.h
index 2d5c71b..46c3bd4 100644
--- a/runtime/lock_word.h
+++ b/runtime/lock_word.h
@@ -21,6 +21,7 @@
 #include <stdint.h>
 
 #include "base/logging.h"
+#include "read_barrier.h"
 #include "utils.h"
 
 namespace art {
@@ -31,34 +32,43 @@ namespace mirror {
 class Monitor;
 
 /* The lock value itself as stored in mirror::Object::monitor_.  The two most significant bits of
- * the state. The three possible states are fat locked, thin/unlocked, and hash code.
- * When the lock word is in the "thin" state and its bits are formatted as follows:
+ * the state. The four possible states are fat locked, thin/unlocked, hash code, and forwarding
+ * address. When the lock word is in the "thin" state and its bits are formatted as follows:
  *
- *  |33|22222222221111|1111110000000000|
- *  |10|98765432109876|5432109876543210|
- *  |00| lock count   |thread id owner |
+ *  |33|22|222222221111|1111110000000000|
+ *  |10|98|765432109876|5432109876543210|
+ *  |00|rb| lock count |thread id owner |
  *
  * When the lock word is in the "fat" state and its bits are formatted as follows:
  *
- *  |33|222222222211111111110000000000|
- *  |10|987654321098765432109876543210|
- *  |01| MonitorId                    |
+ *  |33|22|2222222211111111110000000000|
+ *  |10|98|7654321098765432109876543210|
+ *  |01|rb| MonitorId                  |
  *
  * When the lock word is in hash state and its bits are formatted as follows:
  *
- *  |33|222222222211111111110000000000|
- *  |10|987654321098765432109876543210|
- *  |10| HashCode                     |
+ *  |33|22|2222222211111111110000000000|
+ *  |10|98|7654321098765432109876543210|
+ *  |10|rb| HashCode                   |
+ *
+ * When the lock word is in fowarding address state and its bits are formatted as follows:
+ *
+ *  |33|22|2222222211111111110000000000|
+ *  |10|98|7654321098765432109876543210|
+ *  |11| ForwardingAddress             |
+ *
+ * The rb bits store the read barrier state.
  */
 class LockWord {
  public:
   enum SizeShiftsAndMasks {  // private marker to avoid generate-operator-out.py from processing.
     // Number of bits to encode the state, currently just fat or thin/unlocked or hash code.
     kStateSize = 2,
+    kReadBarrierStateSize = 2,
     // Number of bits to encode the thin lock owner.
     kThinLockOwnerSize = 16,
     // Remaining bits are the recursive lock count.
-    kThinLockCountSize = 32 - kThinLockOwnerSize - kStateSize,
+    kThinLockCountSize = 32 - kThinLockOwnerSize - kStateSize - kReadBarrierStateSize,
     // Thin lock bits. Owner in lowest bits.
 
     kThinLockOwnerShift = 0,
@@ -68,28 +78,41 @@ class LockWord {
     kThinLockCountShift = kThinLockOwnerSize + kThinLockOwnerShift,
     kThinLockCountMask = (1 << kThinLockCountSize) - 1,
     kThinLockMaxCount = kThinLockCountMask,
+    kThinLockCountOne = 1 << kThinLockCountShift,  // == 65536 (0x10000)
 
     // State in the highest bits.
-    kStateShift = kThinLockCountSize + kThinLockCountShift,
+    kStateShift = kReadBarrierStateSize + kThinLockCountSize + kThinLockCountShift,
     kStateMask = (1 << kStateSize) - 1,
+    kStateMaskShifted = kStateMask << kStateShift,
     kStateThinOrUnlocked = 0,
     kStateFat = 1,
     kStateHash = 2,
     kStateForwardingAddress = 3,
+    kReadBarrierStateShift = kThinLockCountSize + kThinLockCountShift,
+    kReadBarrierStateMask = (1 << kReadBarrierStateSize) - 1,
+    kReadBarrierStateMaskShifted = kReadBarrierStateMask << kReadBarrierStateShift,
+    kReadBarrierStateMaskShiftedToggled = ~kReadBarrierStateMaskShifted,
 
     // When the state is kHashCode, the non-state bits hold the hashcode.
     kHashShift = 0,
-    kHashSize = 32 - kStateSize,
+    kHashSize = 32 - kStateSize - kReadBarrierStateSize,
     kHashMask = (1 << kHashSize) - 1,
     kMaxHash = kHashMask,
+
+    kMonitorIdShift = kHashShift,
+    kMonitorIdSize = kHashSize,
+    kMonitorIdMask = kHashMask,
+    kMonitorIdAlignmentShift = 32 - kMonitorIdSize,
+    kMonitorIdAlignment = 1 << kMonitorIdAlignmentShift,
     kMaxMonitorId = kMaxHash
   };
 
-  static LockWord FromThinLockId(uint32_t thread_id, uint32_t count) {
+  static LockWord FromThinLockId(uint32_t thread_id, uint32_t count, uint32_t rb_state) {
     CHECK_LE(thread_id, static_cast<uint32_t>(kThinLockMaxOwner));
     CHECK_LE(count, static_cast<uint32_t>(kThinLockMaxCount));
     return LockWord((thread_id << kThinLockOwnerShift) | (count << kThinLockCountShift) |
-                     (kStateThinOrUnlocked << kStateShift));
+                    (rb_state << kReadBarrierStateShift) |
+                    (kStateThinOrUnlocked << kStateShift));
   }
 
   static LockWord FromForwardingAddress(size_t target) {
@@ -97,9 +120,23 @@ class LockWord {
     return LockWord((target >> kStateSize) | (kStateForwardingAddress << kStateShift));
   }
 
-  static LockWord FromHashCode(uint32_t hash_code) {
+  static LockWord FromHashCode(uint32_t hash_code, uint32_t rb_state) {
     CHECK_LE(hash_code, static_cast<uint32_t>(kMaxHash));
-    return LockWord((hash_code << kHashShift) | (kStateHash << kStateShift));
+    return LockWord((hash_code << kHashShift) |
+                    (rb_state << kReadBarrierStateShift) |
+                    (kStateHash << kStateShift));
+  }
+
+  static LockWord FromDefault(uint32_t rb_state) {
+    return LockWord(rb_state << kReadBarrierStateShift);
+  }
+
+  static bool IsDefault(LockWord lw) {
+    return LockWord().GetValue() == lw.GetValue();
+  }
+
+  static LockWord Default() {
+    return LockWord();
   }
 
   enum LockState {
@@ -111,6 +148,7 @@ class LockWord {
   };
 
   LockState GetState() const {
+    CheckReadBarrierState();
     if (UNLIKELY(value_ == 0)) {
       return kUnlocked;
     } else {
@@ -129,6 +167,10 @@ class LockWord {
     }
   }
 
+  uint32_t ReadBarrierState() const {
+    return (value_ >> kReadBarrierStateShift) & kReadBarrierStateMask;
+  }
+
   // Return the owner thin lock thread id.
   uint32_t ThinLockOwner() const;
 
@@ -141,25 +183,58 @@ class LockWord {
   // Return the forwarding address stored in the monitor.
   size_t ForwardingAddress() const;
 
+  // Constructor a lock word for inflation to use a Monitor.
+  explicit LockWord(Monitor* mon, uint32_t rb_state);
+
+  // Return the hash code stored in the lock word, must be kHashCode state.
+  int32_t GetHashCode() const;
+
+  template <bool kIncludeReadBarrierState>
+  static bool Equal(LockWord lw1, LockWord lw2) {
+    if (kIncludeReadBarrierState) {
+      return lw1.GetValue() == lw2.GetValue();
+    }
+    return lw1.GetValueWithoutReadBarrierState() == lw2.GetValueWithoutReadBarrierState();
+  }
+
+ private:
   // Default constructor with no lock ownership.
   LockWord();
 
-  // Constructor a lock word for inflation to use a Monitor.
-  explicit LockWord(Monitor* mon);
-
-  bool operator==(const LockWord& rhs) const {
-    return GetValue() == rhs.GetValue();
+  explicit LockWord(uint32_t val) : value_(val) {
+    CheckReadBarrierState();
   }
 
-  // Return the hash code stored in the lock word, must be kHashCode state.
-  int32_t GetHashCode() const;
+  // Disallow this in favor of explicit Equal() with the
+  // kIncludeReadBarrierState param to make clients be aware of the
+  // read barrier state.
+  bool operator==(const LockWord& rhs) = delete;
+
+  void CheckReadBarrierState() const {
+    if (kIsDebugBuild && ((value_ >> kStateShift) & kStateMask) != kStateForwardingAddress) {
+      uint32_t rb_state = ReadBarrierState();
+      if (!kUseReadBarrier) {
+        DCHECK_EQ(rb_state, 0U);
+      } else {
+        DCHECK(rb_state == ReadBarrier::white_ptr_ ||
+               rb_state == ReadBarrier::gray_ptr_ ||
+               rb_state == ReadBarrier::black_ptr_) << rb_state;
+      }
+    }
+  }
 
+  // Note GetValue() includes the read barrier bits and comparing (==)
+  // GetValue() between two lock words to compare the lock states may
+  // not work. Prefer Equal() or GetValueWithoutReadBarrierState().
   uint32_t GetValue() const {
+    CheckReadBarrierState();
     return value_;
   }
 
- private:
-  explicit LockWord(uint32_t val) : value_(val) {}
+  uint32_t GetValueWithoutReadBarrierState() const {
+    CheckReadBarrierState();
+    return value_ & ~(kReadBarrierStateMask << kReadBarrierStateShift);
+  }
 
   // Only Object should be converting LockWords to/from uints.
   friend class mirror::Object;