Re-land: [mojo-edk] Serialise windows handles into an "extra header" section.

This change introduces an "extra header" section of a channel message
header. On windows, platform handles are place into this section instead
of being appended to the end of a message.

At the same time, make a couple of smaller changes:
1. Allow the number of serialised handles to be less than the number
stated in the constructor.
2. Add a has_mach_ports() helper function.
3. Limit the number of attached handles.

These changes are all prep work for Mach ports support.

This re-land perserves the old wire format and message semantics on ChromeOS
and changes it on all other platforms.

Original change: https://codereview.chromium.org/1740953002

BUG=582468

Review URL: https://codereview.chromium.org/1759033002

Cr-Commit-Position: refs/heads/master@{#379114}

4 files changed, 168 insertions, 54 deletions

diff --git a/mojo/edk/system/channel.cc b/mojo/edk/system/channel.cc
index 2d6142e..375d65b 100644
--- a/mojo/edk/system/channel.cc
+++ b/mojo/edk/system/channel.cc
@@ -21,21 +21,40 @@ namespace {
 static_assert(sizeof(Channel::Message::Header) % kChannelMessageAlignment == 0,
     "Invalid Header size.");
 
+#if defined(OS_CHROMEOS)
+static_assert(sizeof(Channel::Message::Header) == 8,
+              "Header must be 8 bytes on ChromeOS");
+#endif
+
 }  // namespace
 
 const size_t kReadBufferSize = 4096;
 const size_t kMaxUnusedReadBufferCapacity = 256 * 1024;
 const size_t kMaxChannelMessageSize = 256 * 1024 * 1024;
+const size_t kMaxAttachedHandles = 128;
 
 Channel::Message::Message(size_t payload_size,
-                          size_t num_handles,
-                          Header::MessageType message_type) {
-  size_ = payload_size + sizeof(Header);
+                          size_t max_handles,
+                          Header::MessageType message_type)
+    : max_handles_(max_handles) {
+  DCHECK_LE(max_handles_, kMaxAttachedHandles);
+
+  size_t extra_header_size = 0;
 #if defined(OS_WIN)
-  // On Windows we serialize platform handles directly into the message buffer.
-  size_ += num_handles * sizeof(PlatformHandle);
+  // On Windows we serialize platform handles into the extra header space.
+  extra_header_size = max_handles_ * sizeof(PlatformHandle);
+#endif
+  // Pad extra header data to be aliged to |kChannelMessageAlignment| bytes.
+  if (extra_header_size % kChannelMessageAlignment) {
+    extra_header_size += kChannelMessageAlignment -
+                         (extra_header_size % kChannelMessageAlignment);
+  }
+  DCHECK_EQ(0u, extra_header_size % kChannelMessageAlignment);
+#if defined(OS_CHROMEOS)
+  DCHECK_EQ(0u, extra_header_size);
 #endif
 
+  size_ = sizeof(Header) + extra_header_size + payload_size;
   data_ = static_cast<char*>(base::AlignedAlloc(size_,
                                                 kChannelMessageAlignment));
   memset(data_, 0, size_);
@@ -44,17 +63,21 @@ Channel::Message::Message(size_t payload_size,
   DCHECK_LE(size_, std::numeric_limits<uint32_t>::max());
   header_->num_bytes = static_cast<uint32_t>(size_);
 
-  DCHECK_LE(num_handles, std::numeric_limits<uint16_t>::max());
-  header_->num_handles = static_cast<uint16_t>(num_handles);
-
+  DCHECK_LE(sizeof(Header) + extra_header_size,
+            std::numeric_limits<uint16_t>::max());
   header_->message_type = message_type;
+#if defined(OS_CHROMEOS)
+  header_->num_handles = static_cast<uint16_t>(max_handles);
+#else
+  header_->num_header_bytes =
+      static_cast<uint16_t>(sizeof(Header) + extra_header_size);
+#endif
 
 #if defined(OS_WIN)
-  if (num_handles > 0) {
-    handles_ = reinterpret_cast<PlatformHandle*>(
-        data_ + sizeof(Header) + payload_size);
+  if (max_handles_ > 0) {
+    handles_ = reinterpret_cast<PlatformHandle*>(mutable_extra_header());
     // Initialize all handles to invalid values.
-    for (size_t i = 0; i < header_->num_handles; ++i)
+    for (size_t i = 0; i < max_handles_; ++i)
       handles()[i] = PlatformHandle();
   }
 #endif
@@ -76,7 +99,8 @@ Channel::MessagePtr Channel::Message::Deserialize(const void* data,
   // We only serialize messages into other messages when performing message
   // relay on Windows.
   NOTREACHED();
-#endif
+  return nullptr;
+#else
   if (data_num_bytes < sizeof(Header))
     return nullptr;
 
@@ -87,63 +111,107 @@ Channel::MessagePtr Channel::Message::Deserialize(const void* data,
     return nullptr;
   }
 
-  uint32_t handles_size = header->num_handles * sizeof(PlatformHandle);
-  if (data_num_bytes < sizeof(Header) + handles_size) {
-    DLOG(ERROR) << "Decoding invalid message:" << data_num_bytes
-                << " < " << (sizeof(Header) + handles_size);
+  if (header->num_bytes < header->num_header_bytes) {
+    DLOG(ERROR) << "Decoding invalid message: " << header->num_bytes << " < "
+                << header->num_header_bytes;
     return nullptr;
   }
 
-  DCHECK_LE(handles_size, data_num_bytes - sizeof(Header));
+  uint32_t extra_header_size = header->num_header_bytes - sizeof(Header);
+  uint32_t max_handles = extra_header_size / sizeof(PlatformHandle);
+  if (header->num_handles > max_handles) {
+    DLOG(ERROR) << "Decoding invalid message:" << header->num_handles << " > "
+                << max_handles;
+    return nullptr;
+  }
 
-  MessagePtr message(new Message(data_num_bytes - sizeof(Header) - handles_size,
-                                 header->num_handles));
+  MessagePtr message(
+      new Message(data_num_bytes - header->num_header_bytes, max_handles));
 
   DCHECK_EQ(message->data_num_bytes(), data_num_bytes);
+  DCHECK_EQ(message->extra_header_size(), extra_header_size);
+  DCHECK_EQ(message->header_->num_header_bytes, header->num_header_bytes);
 
-  // Copy all bytes, including the serialized handles.
+  // Copy all payload bytes.
   memcpy(message->mutable_payload(),
+         static_cast<const char*>(data) + header->num_header_bytes,
+         data_num_bytes - header->num_header_bytes);
+  // Copy extra header bytes.
+  memcpy(message->mutable_extra_header(),
          static_cast<const char*>(data) + sizeof(Header),
-         data_num_bytes - sizeof(Header));
+         message->extra_header_size());
+  message->header_->num_handles = header->num_handles;
 
   return message;
+#endif
 }
 
 size_t Channel::Message::payload_size() const {
-#if defined(OS_WIN)
-  return size_ - sizeof(Header) -
-      sizeof(PlatformHandle) * header_->num_handles;
-#else
+#if defined(OS_CHROMEOS)
   return header_->num_bytes - sizeof(Header);
+#else
+  return size_ - header_->num_header_bytes;
 #endif
 }
 
 PlatformHandle* Channel::Message::handles() {
+#if defined(OS_CHROMEOS)
+  // Old semantics for ChromeOS.
   if (header_->num_handles == 0)
     return nullptr;
+  CHECK(handle_vector_);
+  return handle_vector_->data();
+#else
+  if (max_handles_ == 0)
+    return nullptr;
 #if defined(OS_WIN)
-  return reinterpret_cast<PlatformHandle*>(static_cast<char*>(data_) +
-                                           sizeof(Header) + payload_size());
+  return handles_;
 #else
   CHECK(handle_vector_);
   return handle_vector_->data();
-#endif
+#endif  // defined(OS_WIN)
+#endif  // defined(OS_CHROMEOS)
+}
+
+#if defined(OS_MACOSX) && !defined(OS_IOS)
+bool Channel::Message::has_mach_ports() const {
+  if (!has_handles())
+    return false;
+
+  for (const auto& handle : (*handle_vector_)) {
+    if (handle.type == PlatformHandle::Type::MACH)
+      return true;
+  }
+  return false;
 }
+#endif
 
 void Channel::Message::SetHandles(ScopedPlatformHandleVectorPtr new_handles) {
+#if defined(OS_CHROMEOS)
+  // Old semantics for ChromeOS.
   if (header_->num_handles == 0) {
     CHECK(!new_handles || new_handles->size() == 0);
     return;
   }
-
   CHECK(new_handles && new_handles->size() == header_->num_handles);
+  std::swap(handle_vector_, new_handles);
+
+#else
+  if (max_handles_ == 0) {
+    CHECK(!new_handles || new_handles->size() == 0);
+    return;
+  }
+
+  CHECK(new_handles && new_handles->size() <= max_handles_);
+  header_->num_handles = static_cast<uint16_t>(new_handles->size());
 #if defined(OS_WIN)
   memcpy(handles(), new_handles->data(),
-         sizeof(PlatformHandle) * header_->num_handles);
+         sizeof(PlatformHandle) * new_handles->size());
   new_handles->clear();
 #else
   std::swap(handle_vector_, new_handles);
-#endif
+#endif  // defined(OS_WIN)
+#endif  // defined(OS_CHROMEOS)
 }
 
 ScopedPlatformHandleVectorPtr Channel::Message::TakeHandles() {
@@ -329,14 +397,28 @@ bool Channel::OnReadComplete(size_t bytes_read, size_t *next_read_size_hint) {
       return true;
     }
 
+#if defined(OS_CHROMEOS)
+    size_t extra_header_size = 0;
+    const void* extra_header = nullptr;
     size_t payload_size = header->num_bytes - sizeof(Message::Header);
     void* payload = payload_size ? const_cast<Message::Header*>(&header[1])
                                  : nullptr;
+#else
+    size_t extra_header_size =
+        header->num_header_bytes - sizeof(Message::Header);
+    const void* extra_header = header + 1;
+    size_t payload_size = header->num_bytes - header->num_header_bytes;
+    void* payload =
+        payload_size ? reinterpret_cast<Message::Header*>(
+                           const_cast<char*>(read_buffer_->occupied_bytes()) +
+                           header->num_header_bytes)
+                     : nullptr;
+#endif
 
     ScopedPlatformHandleVectorPtr handles;
     if (header->num_handles > 0) {
-      handles = GetReadPlatformHandles(header->num_handles,
-                                       &payload, &payload_size);
+      handles = GetReadPlatformHandles(header->num_handles, extra_header,
+                                       extra_header_size);
       if (!handles) {
         // Not enough handles available for this message.
         break;
diff --git a/mojo/edk/system/channel.h b/mojo/edk/system/channel.h
index bfc1232..0d7ccdb 100644
--- a/mojo/edk/system/channel.h
+++ b/mojo/edk/system/channel.h
@@ -29,6 +29,7 @@ class Channel : public base::RefCountedThreadSafe<Channel> {
 
   // A message to be written to a channel.
   struct Message {
+#pragma pack(push, 1)
     struct Header {
       enum class MessageType : uint16_t {
         // A normal message.
@@ -44,17 +45,33 @@ class Channel : public base::RefCountedThreadSafe<Channel> {
       // Message size in bytes, including the header.
       uint32_t num_bytes;
 
+#if defined(OS_CHROMEOS)
+      // Old message wire format for ChromeOS.
       // Number of attached handles.
       uint16_t num_handles;
 
       MessageType message_type;
+#else
+      // Total size of header, including extra header data (i.e. HANDLEs on
+      // windows).
+      uint16_t num_header_bytes;
+
+      // Number of attached handles. May be less than the reserved handle
+      // storage size in this message on platforms that serialise handles as
+      // data (i.e. HANDLEs on Windows, Mach ports on OSX).
+      uint16_t num_handles;
+
+      MessageType message_type;
+
+      char padding[6];
+#endif  // defined(OS_CHROMEOS)
     };
+#pragma pack(pop)
 
     // Allocates and owns a buffer for message data with enough capacity for
-    // |payload_size| bytes plus a header. Takes ownership of |handles|, which
-    // may be null.
+    // |payload_size| bytes plus a header, plus |max_handles| platform handles.
     Message(size_t payload_size,
-            size_t num_handles,
+            size_t max_handles,
             Header::MessageType message_type = Header::MessageType::NORMAL);
 
     ~Message();
@@ -65,15 +82,30 @@ class Channel : public base::RefCountedThreadSafe<Channel> {
     const void* data() const { return data_; }
     size_t data_num_bytes() const { return size_; }
 
+#if defined(OS_CHROMEOS)
     void* mutable_payload() { return static_cast<void*>(header_ + 1); }
     const void* payload() const {
       return static_cast<const void*>(header_ + 1);
     }
     size_t payload_size() const;
+#else
+    const void* extra_header() const { return data_ + sizeof(Header); }
+    void* mutable_extra_header() { return data_ + sizeof(Header); }
+    size_t extra_header_size() const {
+      return header_->num_header_bytes - sizeof(Header);
+    }
+
+    void* mutable_payload() { return data_ + header_->num_header_bytes; }
+    const void* payload() const { return data_ + header_->num_header_bytes; }
+    size_t payload_size() const;
+#endif  // defined(OS_CHROMEOS)
 
     size_t num_handles() const { return header_->num_handles; }
     bool has_handles() const { return header_->num_handles > 0; }
     PlatformHandle* handles();
+#if defined(OS_MACOSX) && !defined(OS_IOS)
+    bool has_mach_ports() const;
+#endif
 
     // Note: SetHandles() and TakeHandles() invalidate any previous value of
     // handles().
@@ -94,6 +126,7 @@ class Channel : public base::RefCountedThreadSafe<Channel> {
 
    private:
     size_t size_;
+    size_t max_handles_;
     char* data_;
     Header* header_;
 
@@ -175,17 +208,16 @@ class Channel : public base::RefCountedThreadSafe<Channel> {
   // OK to call this synchronously from any public interface methods.
   void OnError();
 
-  // Retrieves the set of platform handles read for a given message. |payload|
-  // and |payload_size| correspond to the full message body. Depending on
-  // the Channel implementation, this body may encode platform handles, or
-  // handles may be stored and managed elsewhere by the implementation.
-  // If |num_handles| handles cannot be returned, this must return null.
-  // The implementation may also adjust the values of |*payload| and/or
-  // |*payload_size| to hide handle data from the user.
+  // Retrieves the set of platform handles read for a given message.
+  // |extra_header| and |extra_header_size| correspond to the extra header data.
+  // Depending on the Channel implementation, this body may encode platform
+  // handles, or handles may be stored and managed elsewhere by the
+  // implementation.  If |num_handles| handles cannot be returned, this must
+  // return null.
   virtual ScopedPlatformHandleVectorPtr GetReadPlatformHandles(
       size_t num_handles,
-      void** payload,
-      size_t* payload_size) = 0;
+      const void* extra_header,
+      size_t extra_header_size) = 0;
 
   virtual void OnControlMessage(Message::Header::MessageType message_type,
                                 const void* payload,
diff --git a/mojo/edk/system/channel_posix.cc b/mojo/edk/system/channel_posix.cc
index 8edafd94..2576eac 100644
--- a/mojo/edk/system/channel_posix.cc
+++ b/mojo/edk/system/channel_posix.cc
@@ -134,16 +134,18 @@ class ChannelPosix : public Channel,
 
   ScopedPlatformHandleVectorPtr GetReadPlatformHandles(
       size_t num_handles,
-      void** payload,
-      size_t* payload_size) override {
+      const void* extra_header,
+      size_t extra_header_size) override {
     if (incoming_platform_handles_.size() < num_handles)
       return nullptr;
+
     ScopedPlatformHandleVectorPtr handles(
         new PlatformHandleVector(num_handles));
     for (size_t i = 0; i < num_handles; ++i) {
       (*handles)[i] = incoming_platform_handles_.front();
       incoming_platform_handles_.pop_front();
     }
+
     return handles;
   }
 
diff --git a/mojo/edk/system/channel_win.cc b/mojo/edk/system/channel_win.cc
index 3678835..1eeb2f6 100644
--- a/mojo/edk/system/channel_win.cc
+++ b/mojo/edk/system/channel_win.cc
@@ -121,17 +121,15 @@ class ChannelWin : public Channel,
 
   ScopedPlatformHandleVectorPtr GetReadPlatformHandles(
       size_t num_handles,
-      void** payload,
-      size_t* payload_size) override {
+      const void* extra_header,
+      size_t extra_header_size) override {
     size_t handles_size = sizeof(PlatformHandle) * num_handles;
-    if (handles_size > *payload_size)
+    if (handles_size > extra_header_size)
       return nullptr;
 
-    *payload_size -= handles_size;
     ScopedPlatformHandleVectorPtr handles(
         new PlatformHandleVector(num_handles));
-    memcpy(handles->data(),
-           static_cast<const char*>(*payload) + *payload_size, handles_size);
+    memcpy(handles->data(), extra_header, handles_size);
     return handles;
   }