gpu: Reuse transfer buffers more aggresively

By keeping track of transfer buffer usage (both sync and async), it is
possible to reuse an existing transfer buffer earlier than it is today.

For synchronous uploads, this is done by inserting tokens marking a
buffer's last usage. If such a token has passed, the buffer can be
reused.

For asynchronous uploads, this is done by adding an internal async
upload token to the GLES2Implementation and GLES2CmdDecoderImpl that
enumerates all asynchronous uploads. When an upload is completed, the
token is synchronized with the client which then can free the memory
associated with the async token.

The fenced allocator and mapped memory manager gets a callback that is
used to allow fenced allocator to make the user, in this case
GLES2Implementation, poll the current async upload token and free any
associated memory.

BUG=328808

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=260177

Review URL: https://codereview.chromium.org/116863003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@260507 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 198 insertions, 20 deletions

diff --git a/gpu/command_buffer/common/gles2_cmd_format.h b/gpu/command_buffer/common/gles2_cmd_format.h
index 736c15b..f11b1e9 100644
--- a/gpu/command_buffer/common/gles2_cmd_format.h
+++ b/gpu/command_buffer/common/gles2_cmd_format.h
@@ -152,6 +152,25 @@ struct QuerySync {
   uint64 result;
 };
 
+struct AsyncUploadSync {
+  void Reset() {
+    base::subtle::Release_Store(&async_upload_token, 0);
+  }
+
+  void SetAsyncUploadToken(uint32 token) {
+    DCHECK_NE(token, 0u);
+    base::subtle::Release_Store(&async_upload_token, token);
+  }
+
+  bool HasAsyncUploadTokenPassed(uint32 token) {
+    DCHECK_NE(token, 0u);
+    uint32_t current_token = base::subtle::Acquire_Load(&async_upload_token);
+    return (current_token - token < 0x80000000);
+  }
+
+  base::subtle::Atomic32 async_upload_token;
+};
+
 COMPILE_ASSERT(sizeof(ProgramInput) == 20, ProgramInput_size_not_20);
 COMPILE_ASSERT(offsetof(ProgramInput, type) == 0,
                OffsetOf_ProgramInput_type_not_0);
diff --git a/gpu/command_buffer/common/gles2_cmd_format_autogen.h b/gpu/command_buffer/common/gles2_cmd_format_autogen.h
index c678d24..49888b3 100644
--- a/gpu/command_buffer/common/gles2_cmd_format_autogen.h
+++ b/gpu/command_buffer/common/gles2_cmd_format_autogen.h
@@ -10102,7 +10102,10 @@ struct AsyncTexSubImage2DCHROMIUM {
             GLenum _format,
             GLenum _type,
             uint32 _data_shm_id,
-            uint32 _data_shm_offset) {
+            uint32 _data_shm_offset,
+            uint32 _async_upload_token,
+            uint32 _sync_data_shm_id,
+            uint32 _sync_data_shm_offset) {
     SetHeader();
     target = _target;
     level = _level;
@@ -10114,6 +10117,9 @@ struct AsyncTexSubImage2DCHROMIUM {
     type = _type;
     data_shm_id = _data_shm_id;
     data_shm_offset = _data_shm_offset;
+    async_upload_token = _async_upload_token;
+    sync_data_shm_id = _sync_data_shm_id;
+    sync_data_shm_offset = _sync_data_shm_offset;
   }
 
   void* Set(void* cmd,
@@ -10126,7 +10132,10 @@ struct AsyncTexSubImage2DCHROMIUM {
             GLenum _format,
             GLenum _type,
             uint32 _data_shm_id,
-            uint32 _data_shm_offset) {
+            uint32 _data_shm_offset,
+            uint32 _async_upload_token,
+            uint32 _sync_data_shm_id,
+            uint32 _sync_data_shm_offset) {
     static_cast<ValueType*>(cmd)->Init(_target,
                                        _level,
                                        _xoffset,
@@ -10136,7 +10145,10 @@ struct AsyncTexSubImage2DCHROMIUM {
                                        _format,
                                        _type,
                                        _data_shm_id,
-                                       _data_shm_offset);
+                                       _data_shm_offset,
+                                       _async_upload_token,
+                                       _sync_data_shm_id,
+                                       _sync_data_shm_offset);
     return NextCmdAddress<ValueType>(cmd);
   }
 
@@ -10151,10 +10163,13 @@ struct AsyncTexSubImage2DCHROMIUM {
   uint32 type;
   uint32 data_shm_id;
   uint32 data_shm_offset;
+  uint32 async_upload_token;
+  uint32 sync_data_shm_id;
+  uint32 sync_data_shm_offset;
 };
 
-COMPILE_ASSERT(sizeof(AsyncTexSubImage2DCHROMIUM) == 44,
-               Sizeof_AsyncTexSubImage2DCHROMIUM_is_not_44);
+COMPILE_ASSERT(sizeof(AsyncTexSubImage2DCHROMIUM) == 56,
+               Sizeof_AsyncTexSubImage2DCHROMIUM_is_not_56);
 COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, header) == 0,
                OffsetOf_AsyncTexSubImage2DCHROMIUM_header_not_0);
 COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, target) == 4,
@@ -10177,6 +10192,12 @@ COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, data_shm_id) == 36,
                OffsetOf_AsyncTexSubImage2DCHROMIUM_data_shm_id_not_36);
 COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, data_shm_offset) == 40,
                OffsetOf_AsyncTexSubImage2DCHROMIUM_data_shm_offset_not_40);
+COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, async_upload_token) == 44,
+               OffsetOf_AsyncTexSubImage2DCHROMIUM_async_upload_token_not_44);
+COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, sync_data_shm_id) == 48,
+               OffsetOf_AsyncTexSubImage2DCHROMIUM_sync_data_shm_id_not_48);
+COMPILE_ASSERT(offsetof(AsyncTexSubImage2DCHROMIUM, sync_data_shm_offset) == 52,
+               OffsetOf_AsyncTexSubImage2DCHROMIUM_sync_data_shm_offset_not_52);
 
 struct AsyncTexImage2DCHROMIUM {
   typedef AsyncTexImage2DCHROMIUM ValueType;
@@ -10199,7 +10220,10 @@ struct AsyncTexImage2DCHROMIUM {
             GLenum _format,
             GLenum _type,
             uint32 _pixels_shm_id,
-            uint32 _pixels_shm_offset) {
+            uint32 _pixels_shm_offset,
+            uint32 _async_upload_token,
+            uint32 _sync_data_shm_id,
+            uint32 _sync_data_shm_offset) {
     SetHeader();
     target = _target;
     level = _level;
@@ -10211,6 +10235,9 @@ struct AsyncTexImage2DCHROMIUM {
     type = _type;
     pixels_shm_id = _pixels_shm_id;
     pixels_shm_offset = _pixels_shm_offset;
+    async_upload_token = _async_upload_token;
+    sync_data_shm_id = _sync_data_shm_id;
+    sync_data_shm_offset = _sync_data_shm_offset;
   }
 
   void* Set(void* cmd,
@@ -10223,7 +10250,10 @@ struct AsyncTexImage2DCHROMIUM {
             GLenum _format,
             GLenum _type,
             uint32 _pixels_shm_id,
-            uint32 _pixels_shm_offset) {
+            uint32 _pixels_shm_offset,
+            uint32 _async_upload_token,
+            uint32 _sync_data_shm_id,
+            uint32 _sync_data_shm_offset) {
     static_cast<ValueType*>(cmd)->Init(_target,
                                        _level,
                                        _internalformat,
@@ -10233,7 +10263,10 @@ struct AsyncTexImage2DCHROMIUM {
                                        _format,
                                        _type,
                                        _pixels_shm_id,
-                                       _pixels_shm_offset);
+                                       _pixels_shm_offset,
+                                       _async_upload_token,
+                                       _sync_data_shm_id,
+                                       _sync_data_shm_offset);
     return NextCmdAddress<ValueType>(cmd);
   }
 
@@ -10248,10 +10281,13 @@ struct AsyncTexImage2DCHROMIUM {
   uint32 type;
   uint32 pixels_shm_id;
   uint32 pixels_shm_offset;
+  uint32 async_upload_token;
+  uint32 sync_data_shm_id;
+  uint32 sync_data_shm_offset;
 };
 
-COMPILE_ASSERT(sizeof(AsyncTexImage2DCHROMIUM) == 44,
-               Sizeof_AsyncTexImage2DCHROMIUM_is_not_44);
+COMPILE_ASSERT(sizeof(AsyncTexImage2DCHROMIUM) == 56,
+               Sizeof_AsyncTexImage2DCHROMIUM_is_not_56);
 COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, header) == 0,
                OffsetOf_AsyncTexImage2DCHROMIUM_header_not_0);
 COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, target) == 4,
@@ -10274,6 +10310,12 @@ COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, pixels_shm_id) == 36,
                OffsetOf_AsyncTexImage2DCHROMIUM_pixels_shm_id_not_36);
 COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, pixels_shm_offset) == 40,
                OffsetOf_AsyncTexImage2DCHROMIUM_pixels_shm_offset_not_40);
+COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, async_upload_token) == 44,
+               OffsetOf_AsyncTexImage2DCHROMIUM_async_upload_token_not_44);
+COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, sync_data_shm_id) == 48,
+               OffsetOf_AsyncTexImage2DCHROMIUM_sync_data_shm_id_not_48);
+COMPILE_ASSERT(offsetof(AsyncTexImage2DCHROMIUM, sync_data_shm_offset) == 52,
+               OffsetOf_AsyncTexImage2DCHROMIUM_sync_data_shm_offset_not_52);
 
 struct WaitAsyncTexImage2DCHROMIUM {
   typedef WaitAsyncTexImage2DCHROMIUM ValueType;
@@ -10308,6 +10350,33 @@ COMPILE_ASSERT(offsetof(WaitAsyncTexImage2DCHROMIUM, header) == 0,
 COMPILE_ASSERT(offsetof(WaitAsyncTexImage2DCHROMIUM, target) == 4,
                OffsetOf_WaitAsyncTexImage2DCHROMIUM_target_not_4);
 
+struct WaitAllAsyncTexImage2DCHROMIUM {
+  typedef WaitAllAsyncTexImage2DCHROMIUM ValueType;
+  static const CommandId kCmdId = kWaitAllAsyncTexImage2DCHROMIUM;
+  static const cmd::ArgFlags kArgFlags = cmd::kFixed;
+  static const uint8 cmd_flags = CMD_FLAG_SET_TRACE_LEVEL(3);
+
+  static uint32 ComputeSize() {
+    return static_cast<uint32>(sizeof(ValueType));  // NOLINT
+  }
+
+  void SetHeader() { header.SetCmd<ValueType>(); }
+
+  void Init() { SetHeader(); }
+
+  void* Set(void* cmd) {
+    static_cast<ValueType*>(cmd)->Init();
+    return NextCmdAddress<ValueType>(cmd);
+  }
+
+  gpu::CommandHeader header;
+};
+
+COMPILE_ASSERT(sizeof(WaitAllAsyncTexImage2DCHROMIUM) == 4,
+               Sizeof_WaitAllAsyncTexImage2DCHROMIUM_is_not_4);
+COMPILE_ASSERT(offsetof(WaitAllAsyncTexImage2DCHROMIUM, header) == 0,
+               OffsetOf_WaitAllAsyncTexImage2DCHROMIUM_header_not_0);
+
 struct DiscardFramebufferEXT {
   typedef DiscardFramebufferEXT ValueType;
   static const CommandId kCmdId = kDiscardFramebufferEXT;
diff --git a/gpu/command_buffer/common/gles2_cmd_format_test.cc b/gpu/command_buffer/common/gles2_cmd_format_test.cc
index 396ccb3..717e6fb 100644
--- a/gpu/command_buffer/common/gles2_cmd_format_test.cc
+++ b/gpu/command_buffer/common/gles2_cmd_format_test.cc
@@ -4,6 +4,11 @@
 
 // This file contains unit tests for gles2 commmands
 
+#include <limits>
+
+#include "base/bind.h"
+#include "base/synchronization/waitable_event.h"
+#include "base/threading/thread.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "gpu/command_buffer/common/gles2_cmd_format.h"
 
@@ -46,6 +51,68 @@ class GLES2FormatTest : public testing::Test {
   unsigned char buffer_[1024];
 };
 
+void SignalCompletion(uint32* assigned_async_token_ptr,
+                      uint32 async_token,
+                      AsyncUploadSync* sync) {
+  EXPECT_EQ(async_token, *assigned_async_token_ptr);
+  sync->SetAsyncUploadToken(async_token);
+}
+
+TEST(GLES2FormatAsyncUploadSyncTest, AsyncUploadSync) {
+  const size_t kSize = 10;
+  const size_t kCount = 1000;
+
+  base::Thread thread("GLES2FormatUploadSyncTest - Fake Upload Thread");
+  thread.Start();
+
+  // Run the same test 50 times so we retest the wrap as well.
+  for (size_t test_run = 0; test_run < 50; ++test_run) {
+    AsyncUploadSync sync;
+    sync.Reset();
+
+    uint32 buffer_tokens[kSize];
+    memset(buffer_tokens, 0, sizeof(buffer_tokens));
+
+    // Start with a token large enough so that we'll wrap.
+    uint32 async_token = std::numeric_limits<uint32>::max() - kCount / 2;
+
+    // Set initial async token.
+    sync.SetAsyncUploadToken(async_token);
+
+    for (size_t i = 0; i < kCount; ++i) {
+      size_t buffer = i % kSize;
+
+      // Loop until previous async token has passed if any was set.
+      while (buffer_tokens[buffer] &&
+             !sync.HasAsyncUploadTokenPassed(buffer_tokens[buffer]))
+        base::PlatformThread::YieldCurrentThread();
+
+      // Next token, skip 0.
+      async_token++;
+      if (async_token == 0)
+        async_token++;
+
+      // Set the buffer's associated token.
+      buffer_tokens[buffer] = async_token;
+
+      // Set the async upload token on the fake upload thread and assert that
+      // the associated buffer still has the given token.
+      thread.message_loop()->PostTask(FROM_HERE,
+                                      base::Bind(&SignalCompletion,
+                                                 &buffer_tokens[buffer],
+                                                 async_token,
+                                                 &sync));
+    }
+
+    // Flush the thread message loop before starting again.
+    base::WaitableEvent waitable(false, false);
+    thread.message_loop()->PostTask(FROM_HERE,
+                                    base::Bind(&base::WaitableEvent::Signal,
+                                               base::Unretained(&waitable)));
+    waitable.Wait();
+  }
+}
+
 // GCC requires these declarations, but MSVC requires they not be present
 #ifndef _MSC_VER
 const unsigned char GLES2FormatTest::kInitialValue;
diff --git a/gpu/command_buffer/common/gles2_cmd_format_test_autogen.h b/gpu/command_buffer/common/gles2_cmd_format_test_autogen.h
index 6e5594b..fe48327 100644
--- a/gpu/command_buffer/common/gles2_cmd_format_test_autogen.h
+++ b/gpu/command_buffer/common/gles2_cmd_format_test_autogen.h
@@ -3638,7 +3638,10 @@ TEST_F(GLES2FormatTest, AsyncTexSubImage2DCHROMIUM) {
                            static_cast<GLenum>(17),
                            static_cast<GLenum>(18),
                            static_cast<uint32>(19),
-                           static_cast<uint32>(20));
+                           static_cast<uint32>(20),
+                           static_cast<uint32>(21),
+                           static_cast<uint32>(22),
+                           static_cast<uint32>(23));
   EXPECT_EQ(static_cast<uint32>(cmds::AsyncTexSubImage2DCHROMIUM::kCmdId),
             cmd.header.command);
   EXPECT_EQ(sizeof(cmd), cmd.header.size * 4u);
@@ -3652,6 +3655,9 @@ TEST_F(GLES2FormatTest, AsyncTexSubImage2DCHROMIUM) {
   EXPECT_EQ(static_cast<GLenum>(18), cmd.type);
   EXPECT_EQ(static_cast<uint32>(19), cmd.data_shm_id);
   EXPECT_EQ(static_cast<uint32>(20), cmd.data_shm_offset);
+  EXPECT_EQ(static_cast<uint32>(21), cmd.async_upload_token);
+  EXPECT_EQ(static_cast<uint32>(22), cmd.sync_data_shm_id);
+  EXPECT_EQ(static_cast<uint32>(23), cmd.sync_data_shm_offset);
   CheckBytesWrittenMatchesExpectedSize(next_cmd, sizeof(cmd));
 }
 
@@ -3668,7 +3674,10 @@ TEST_F(GLES2FormatTest, AsyncTexImage2DCHROMIUM) {
                            static_cast<GLenum>(17),
                            static_cast<GLenum>(18),
                            static_cast<uint32>(19),
-                           static_cast<uint32>(20));
+                           static_cast<uint32>(20),
+                           static_cast<uint32>(21),
+                           static_cast<uint32>(22),
+                           static_cast<uint32>(23));
   EXPECT_EQ(static_cast<uint32>(cmds::AsyncTexImage2DCHROMIUM::kCmdId),
             cmd.header.command);
   EXPECT_EQ(sizeof(cmd), cmd.header.size * 4u);
@@ -3682,6 +3691,9 @@ TEST_F(GLES2FormatTest, AsyncTexImage2DCHROMIUM) {
   EXPECT_EQ(static_cast<GLenum>(18), cmd.type);
   EXPECT_EQ(static_cast<uint32>(19), cmd.pixels_shm_id);
   EXPECT_EQ(static_cast<uint32>(20), cmd.pixels_shm_offset);
+  EXPECT_EQ(static_cast<uint32>(21), cmd.async_upload_token);
+  EXPECT_EQ(static_cast<uint32>(22), cmd.sync_data_shm_id);
+  EXPECT_EQ(static_cast<uint32>(23), cmd.sync_data_shm_offset);
   CheckBytesWrittenMatchesExpectedSize(next_cmd, sizeof(cmd));
 }
 
@@ -3696,6 +3708,16 @@ TEST_F(GLES2FormatTest, WaitAsyncTexImage2DCHROMIUM) {
   CheckBytesWrittenMatchesExpectedSize(next_cmd, sizeof(cmd));
 }
 
+TEST_F(GLES2FormatTest, WaitAllAsyncTexImage2DCHROMIUM) {
+  cmds::WaitAllAsyncTexImage2DCHROMIUM& cmd =
+      *GetBufferAs<cmds::WaitAllAsyncTexImage2DCHROMIUM>();
+  void* next_cmd = cmd.Set(&cmd);
+  EXPECT_EQ(static_cast<uint32>(cmds::WaitAllAsyncTexImage2DCHROMIUM::kCmdId),
+            cmd.header.command);
+  EXPECT_EQ(sizeof(cmd), cmd.header.size * 4u);
+  CheckBytesWrittenMatchesExpectedSize(next_cmd, sizeof(cmd));
+}
+
 TEST_F(GLES2FormatTest, DiscardFramebufferEXT) {
   cmds::DiscardFramebufferEXT& cmd =
       *GetBufferAs<cmds::DiscardFramebufferEXT>();
diff --git a/gpu/command_buffer/common/gles2_cmd_ids_autogen.h b/gpu/command_buffer/common/gles2_cmd_ids_autogen.h
index ed69305..f63df0e 100644
--- a/gpu/command_buffer/common/gles2_cmd_ids_autogen.h
+++ b/gpu/command_buffer/common/gles2_cmd_ids_autogen.h
@@ -237,14 +237,15 @@
   OP(AsyncTexSubImage2DCHROMIUM)             /* 478 */             \
   OP(AsyncTexImage2DCHROMIUM)                /* 479 */                \
   OP(WaitAsyncTexImage2DCHROMIUM)            /* 480 */            \
-  OP(DiscardFramebufferEXT)                  /* 481 */                  \
-  OP(DiscardFramebufferEXTImmediate)         /* 482 */         \
-  OP(LoseContextCHROMIUM)                    /* 483 */                    \
-  OP(InsertSyncPointCHROMIUM)                /* 484 */                \
-  OP(WaitSyncPointCHROMIUM)                  /* 485 */                  \
-  OP(DrawBuffersEXT)                         /* 486 */                         \
-  OP(DrawBuffersEXTImmediate)                /* 487 */                \
-  OP(DiscardBackbufferCHROMIUM)              /* 488 */
+  OP(WaitAllAsyncTexImage2DCHROMIUM)         /* 481 */         \
+  OP(DiscardFramebufferEXT)                  /* 482 */                  \
+  OP(DiscardFramebufferEXTImmediate)         /* 483 */         \
+  OP(LoseContextCHROMIUM)                    /* 484 */                    \
+  OP(InsertSyncPointCHROMIUM)                /* 485 */                \
+  OP(WaitSyncPointCHROMIUM)                  /* 486 */                  \
+  OP(DrawBuffersEXT)                         /* 487 */                         \
+  OP(DrawBuffersEXTImmediate)                /* 488 */                \
+  OP(DiscardBackbufferCHROMIUM)              /* 489 */
 
 enum CommandId {
   kStartPoint = cmd::kLastCommonId,  // All GLES2 commands start after this.