Add support for drawing video onto a Java ByteBuffer

We now place frames into a Java-accessible memory region as we decode them.

Review URL: https://chromiumcodereview.appspot.com/19297003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@212063 0039d316-1c4b-4281-b951-d872f2087c98

9 files changed, 305 insertions, 41 deletions

diff --git a/remoting/client/frame_consumer.h b/remoting/client/frame_consumer.h
index 81a6013..b5f937c 100644
--- a/remoting/client/frame_consumer.h
+++ b/remoting/client/frame_consumer.h
@@ -5,6 +5,7 @@
 #ifndef REMOTING_CLIENT_FRAME_CONSUMER_H_
 #define REMOTING_CLIENT_FRAME_CONSUMER_H_
 
+#include "base/basictypes.h"
 #include "third_party/skia/include/core/SkRect.h"
 #include "third_party/skia/include/core/SkRegion.h"
 #include "third_party/skia/include/core/SkSize.h"
diff --git a/remoting/client/jni/chromoting_jni.cc b/remoting/client/jni/chromoting_jni.cc
index 4d1d0cf4..647ac5a 100644
--- a/remoting/client/jni/chromoting_jni.cc
+++ b/remoting/client/jni/chromoting_jni.cc
@@ -7,14 +7,12 @@
 #include "base/android/base_jni_registrar.h"
 #include "base/android/jni_android.h"
 #include "base/memory/singleton.h"
+#include "media/base/yuv_convert.h"
 #include "net/android/net_jni_registrar.h"
 #include "remoting/base/url_request_context.h"
-#include "remoting/client/jni/chromoting_jni_instance.h"
 
-namespace {
 // Class and package name of the Java class supporting the methods we call.
-const char* const JAVA_CLASS = "org/chromium/chromoting/jni/JNIInterface";
-}  // namespace
+const char* const JAVA_CLASS = "org/chromium/chromoting/jni/JniInterface";
 
 namespace remoting {
 
@@ -54,6 +52,9 @@ ChromotingJni::ChromotingJni() {
   url_requester_ = new URLRequestContextGetter(ui_task_runner_,
                                                network_task_runner_);
 
+  // Allows later decoding of video frames.
+  media::InitializeCPUSpecificYUVConversions();
+
   class_ = static_cast<jclass>(env->NewGlobalRef(env->FindClass(JAVA_CLASS)));
 }
 
@@ -114,4 +115,31 @@ void ChromotingJni::DisplayAuthenticationPrompt() {
       env->GetStaticMethodID(class_, "displayAuthenticationPrompt", "()V"));
 }
 
+void ChromotingJni::UpdateImageBuffer(int width, int height, jobject buffer) {
+  DCHECK(display_task_runner_->BelongsToCurrentThread());
+
+  JNIEnv* env = base::android::AttachCurrentThread();
+  env->SetStaticIntField(
+      class_,
+      env->GetStaticFieldID(class_, "sWidth", "I"),
+      width);
+  env->SetStaticIntField(
+      class_,
+      env->GetStaticFieldID(class_, "sHeight", "I"),
+      height);
+  env->SetStaticObjectField(
+      class_,
+      env->GetStaticFieldID(class_, "sBuffer", "Ljava/nio/ByteBuffer;"),
+      buffer);
+}
+
+void ChromotingJni::RedrawCanvas() {
+  DCHECK(display_task_runner_->BelongsToCurrentThread());
+
+  JNIEnv* env = base::android::AttachCurrentThread();
+  env->CallStaticVoidMethod(
+      class_,
+      env->GetStaticMethodID(class_, "redrawGraphicsInternal", "()V"));
+}
+
 }  // namespace remoting
diff --git a/remoting/client/jni/chromoting_jni.h b/remoting/client/jni/chromoting_jni.h
index 23df7b0..197cf39 100644
--- a/remoting/client/jni/chromoting_jni.h
+++ b/remoting/client/jni/chromoting_jni.h
@@ -10,17 +10,17 @@
 #include "base/at_exit.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "remoting/base/auto_thread.h"
+#include "remoting/client/jni/chromoting_jni_instance.h"
 #include "remoting/protocol/connection_to_host.h"
 
 template<typename T> struct DefaultSingletonTraits;
 
 namespace remoting {
-class ChromotingJniInstance;
 
 // Houses the global resources on which the Chromoting components run
 // (e.g. message loops and task runners). Proxies outgoing JNI calls from its
 // ChromotingJniInstance member to Java. All its methods should be invoked
-// exclusively from the UI thread.
+// exclusively from the UI thread unless otherwise noted.
 class ChromotingJni {
  public:
   // This class is instantiated at process initialization and persists until
@@ -43,9 +43,9 @@ class ChromotingJni {
     return url_requester_;
   }
 
-  // Initiates a connection with the specified host. Must only be called when
-  // |session_| is null (i.e. before any other call to Connect() or following
-  // a call to Disconnect()).
+  // Initiates a connection with the specified host. Only call when a host
+  // connection is active (i.e. between a call to Connect() and the
+  // corresponding call to Disconnect()).
   void ConnectToHost(const char* username,
                      const char* auth_token,
                      const char* host_jid,
@@ -70,6 +70,12 @@ class ChromotingJni {
   // Pops up a dialog box asking the user to enter a PIN.
   void DisplayAuthenticationPrompt();
 
+  // Updates image dimensions and canvas memory space. Call on display thread.
+  void UpdateImageBuffer(int width, int height, jobject buffer);
+
+  // Draws the latest image buffer onto the canvas. Call on the display thread.
+  void RedrawCanvas();
+
  private:
   ChromotingJni();
 
diff --git a/remoting/client/jni/chromoting_jni_instance.cc b/remoting/client/jni/chromoting_jni_instance.cc
index df6583d..6c6f075 100644
--- a/remoting/client/jni/chromoting_jni_instance.cc
+++ b/remoting/client/jni/chromoting_jni_instance.cc
@@ -10,21 +10,18 @@
 #include "remoting/client/jni/chromoting_jni.h"
 #include "remoting/protocol/libjingle_transport_factory.h"
 
-namespace {
 // TODO(solb) Move into location shared with client plugin.
 const char* const CHAT_SERVER = "talk.google.com";
 const int CHAT_PORT = 5222;
 const bool CHAT_USE_TLS = true;
-const char* const CHAT_AUTH_METHOD = "oauth2";
-}  // namespace
 
 namespace remoting {
 
 ChromotingJniInstance::ChromotingJniInstance(const char* username,
-                                                  const char* auth_token,
-                                                  const char* host_jid,
-                                                  const char* host_id,
-                                                  const char* host_pubkey) {
+                                             const char* auth_token,
+                                             const char* host_jid,
+                                             const char* host_id,
+                                             const char* host_pubkey) {
   DCHECK(ChromotingJni::GetInstance()->
       ui_task_runner()->BelongsToCurrentThread());
 
@@ -44,28 +41,22 @@ ChromotingJniInstance::~ChromotingJniInstance() {}
 
 void ChromotingJniInstance::Cleanup() {
   if (!ChromotingJni::GetInstance()->
-      network_task_runner()->BelongsToCurrentThread()) {
-    ChromotingJni::GetInstance()->network_task_runner()->PostTask(
+      display_task_runner()->BelongsToCurrentThread()) {
+    ChromotingJni::GetInstance()->display_task_runner()->PostTask(
         FROM_HERE,
         base::Bind(&ChromotingJniInstance::Cleanup, this));
     return;
   }
 
-  username_ = "";
-  auth_token_ = "";
-  host_jid_ = "";
-  host_id_ = "";
-  host_pubkey_ = "";
+  // This must be destroyed on the display thread before the producer is gone.
+  view_.reset();
 
-  // |client_| must be torn down before |signaling_|.
-  pin_callback_.Reset();
-  client_.reset();
-  connection_.reset();
-  client_context_.reset();
-  client_config_.reset();
-  signaling_.reset();
-  signaling_config_.reset();
-  network_settings_.reset();
+  // The weak pointers must be invalidated on the same thread they were used.
+  view_weak_factory_->InvalidateWeakPtrs();
+
+  ChromotingJni::GetInstance()->network_task_runner()->PostTask(FROM_HERE,
+      base::Bind(&ChromotingJniInstance::DisconnectFromHostOnNetworkThread,
+                 this));
 }
 
 void ChromotingJniInstance::ProvideSecret(const char* pin) {
@@ -79,6 +70,19 @@ void ChromotingJniInstance::ProvideSecret(const char* pin) {
                                  base::Bind(pin_callback_, pin));
 }
 
+void ChromotingJniInstance::RedrawDesktop() {
+  if (!ChromotingJni::GetInstance()->
+      display_task_runner()->BelongsToCurrentThread()) {
+    ChromotingJni::GetInstance()->display_task_runner()->PostTask(
+        FROM_HERE,
+        base::Bind(&ChromotingJniInstance::RedrawDesktop,
+                   this));
+    return;
+  }
+
+  ChromotingJni::GetInstance()->RedrawCanvas();
+}
+
 void ChromotingJniInstance::OnConnectionState(
     protocol::ConnectionToHost::State state,
     protocol::ErrorCode error) {
@@ -128,11 +132,12 @@ void ChromotingJniInstance::ConnectToHostOnDisplayThread() {
   DCHECK(ChromotingJni::GetInstance()->
       display_task_runner()->BelongsToCurrentThread());
 
-  if (!frame_consumer_.get()) {
-    frame_consumer_ = new FrameConsumerProxy(
-        ChromotingJni::GetInstance()->display_task_runner());
-    // TODO(solb) Instantiate some FrameConsumer implementation and attach it.
-  }
+  frame_consumer_ = new FrameConsumerProxy(
+      ChromotingJni::GetInstance()->display_task_runner());
+  view_.reset(new JniFrameConsumer());
+  view_weak_factory_.reset(new base::WeakPtrFactory<JniFrameConsumer>(
+      view_.get()));
+  frame_consumer_->Attach(view_weak_factory_->GetWeakPtr());
 
   ChromotingJni::GetInstance()->network_task_runner()->PostTask(
       FROM_HERE,
@@ -153,7 +158,6 @@ void ChromotingJniInstance::ConnectToHostOnNetworkThread() {
       this);
   client_config_->authentication_tag = host_id_;
 
-  // TODO(solb) Move these hardcoded values elsewhere:
   client_config_->authentication_methods.push_back(
       protocol::AuthenticationMethod::FromString("spake2_hmac"));
   client_config_->authentication_methods.push_back(
@@ -172,6 +176,8 @@ void ChromotingJniInstance::ConnectToHostOnNetworkThread() {
                                      frame_consumer_,
                                      scoped_ptr<AudioPlayer>()));
 
+  view_->set_frame_producer(client_->GetFrameProducer());
+
   signaling_config_.reset(new XmppSignalStrategy::XmppServerConfig());
   signaling_config_->host = CHAT_SERVER;
   signaling_config_->port = CHAT_PORT;
@@ -181,7 +187,7 @@ void ChromotingJniInstance::ConnectToHostOnNetworkThread() {
       ChromotingJni::GetInstance()->url_requester(),
       username_,
       auth_token_,
-      CHAT_AUTH_METHOD,
+      "oauth2",
       *signaling_config_));
 
   network_settings_.reset(new NetworkSettings(
@@ -194,6 +200,21 @@ void ChromotingJniInstance::ConnectToHostOnNetworkThread() {
   client_->Start(signaling_.get(), fact.Pass());
 }
 
+void ChromotingJniInstance::DisconnectFromHostOnNetworkThread() {
+  DCHECK(ChromotingJni::GetInstance()->
+      network_task_runner()->BelongsToCurrentThread());
+
+  username_ = "";
+  auth_token_ = "";
+  host_jid_ = "";
+  host_id_ = "";
+  host_pubkey_ = "";
+
+  // |client_| must be torn down before |signaling_|.
+  connection_.reset();
+  client_.reset();
+}
+
 void ChromotingJniInstance::FetchSecret(
     bool pairable,
     const protocol::SecretFetchedCallback& callback) {
diff --git a/remoting/client/jni/chromoting_jni_instance.h b/remoting/client/jni/chromoting_jni_instance.h
index c9e689a..2374e24 100644
--- a/remoting/client/jni/chromoting_jni_instance.h
+++ b/remoting/client/jni/chromoting_jni_instance.h
@@ -9,18 +9,19 @@
 
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
+#include "base/memory/weak_ptr.h"
 #include "base/message_loop/message_loop.h"
 #include "remoting/client/chromoting_client.h"
 #include "remoting/client/client_config.h"
 #include "remoting/client/client_context.h"
 #include "remoting/client/client_user_interface.h"
 #include "remoting/client/frame_consumer_proxy.h"
+#include "remoting/client/jni/jni_frame_consumer.h"
 #include "remoting/jingle_glue/network_settings.h"
 #include "remoting/jingle_glue/xmpp_signal_strategy.h"
 #include "remoting/protocol/connection_to_host.h"
 
 namespace remoting {
-class ChromotingJni;
 
 // ClientUserInterface that indirectly makes and receives JNI calls.
 class ChromotingJniInstance
@@ -44,6 +45,9 @@ class ChromotingJniInstance
   // but only after the UI has been asked to provide a PIN (via FetchSecret()).
   void ProvideSecret(const char* pin);
 
+  // Schedules a redraw on the display thread. May be called from any thread.
+  void RedrawDesktop();
+
   // ClientUserInterface implementation.
   virtual void OnConnectionState(
       protocol::ConnectionToHost::State state,
@@ -63,13 +67,18 @@ class ChromotingJniInstance
 
   void ConnectToHostOnDisplayThread();
   void ConnectToHostOnNetworkThread();
+  void DisconnectFromHostOnNetworkThread();
 
   // Notifies the user interface that the user needs to enter a PIN. The
   // current authentication attempt is put on hold until |callback| is invoked.
+  // May be called on any thread.
   void FetchSecret(bool pairable,
                    const protocol::SecretFetchedCallback& callback);
 
+  // This group of variables is to be used on the display thread.
   scoped_refptr<FrameConsumerProxy> frame_consumer_;
+  scoped_ptr<JniFrameConsumer> view_;
+  scoped_ptr<base::WeakPtrFactory<JniFrameConsumer> > view_weak_factory_;
 
   // This group of variables is to be used on the network thread.
   scoped_ptr<ClientConfig> client_config_;
diff --git a/remoting/client/jni/jni_frame_consumer.cc b/remoting/client/jni/jni_frame_consumer.cc
new file mode 100644
index 0000000..ee8ef94
--- /dev/null
+++ b/remoting/client/jni/jni_frame_consumer.cc
@@ -0,0 +1,133 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/client/jni/jni_frame_consumer.h"
+
+#include "base/android/jni_android.h"
+#include "base/logging.h"
+#include "base/synchronization/waitable_event.h"
+#include "remoting/client/frame_producer.h"
+#include "remoting/client/jni/chromoting_jni.h"
+#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
+
+namespace {
+
+// Allocates its buffer within a Java direct byte buffer, where it can be
+// accessed by both native and managed code.
+class DirectDesktopFrame : public webrtc::BasicDesktopFrame {
+ public:
+  DirectDesktopFrame(int width, int height);
+
+  virtual ~DirectDesktopFrame();
+
+  jobject buffer() const {
+    return buffer_;
+  }
+
+ private:
+  jobject buffer_;
+};
+
+DirectDesktopFrame::DirectDesktopFrame(int width, int height)
+    : webrtc::BasicDesktopFrame(webrtc::DesktopSize(width, height)) {
+  JNIEnv* env = base::android::AttachCurrentThread();
+  buffer_ = env->NewDirectByteBuffer(data(), stride()*height);
+}
+
+DirectDesktopFrame::~DirectDesktopFrame() {}
+
+}  // namespace
+
+namespace remoting {
+
+JniFrameConsumer::JniFrameConsumer()
+    : provide_buffer_(true),
+      frame_producer_(NULL) {
+}
+
+JniFrameConsumer::~JniFrameConsumer() {
+  // Stop giving the producer a buffer to work with.
+  provide_buffer_ = false;
+
+  // Don't destroy the object until we've deleted the buffer.
+  base::WaitableEvent done_event(true, false);
+  frame_producer_->RequestReturnBuffers(
+      base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event)));
+  done_event.Wait();
+}
+
+void JniFrameConsumer::set_frame_producer(FrameProducer* producer) {
+  frame_producer_ = producer;
+}
+
+void JniFrameConsumer::ApplyBuffer(const SkISize& view_size,
+                                   const SkIRect& clip_area,
+                                   webrtc::DesktopFrame* buffer,
+                                   const SkRegion& region) {
+  DCHECK(ChromotingJni::GetInstance()->
+      display_task_runner()->BelongsToCurrentThread());
+
+  ChromotingJni::GetInstance()->RedrawCanvas();
+
+  if (view_size.width() > view_size_.width() ||
+      view_size.height() > view_size_.height()) {
+    LOG(INFO) << "Existing buffer is too small";
+    view_size_ = view_size;
+    delete buffer;
+    AllocateBuffer();
+  }
+
+  // Supply |frame_producer_| with a buffer to render the next frame into.
+  if (provide_buffer_)
+    frame_producer_->DrawBuffer(buffer);
+}
+
+void JniFrameConsumer::ReturnBuffer(webrtc::DesktopFrame* buffer) {
+  DCHECK(ChromotingJni::GetInstance()->
+      display_task_runner()->BelongsToCurrentThread());
+  LOG(INFO) << "Returning image buffer";
+  delete buffer;
+}
+
+void JniFrameConsumer::SetSourceSize(const SkISize& source_size,
+                                     const SkIPoint& dpi) {
+  DCHECK(ChromotingJni::GetInstance()->
+      display_task_runner()->BelongsToCurrentThread());
+
+  // We currently render the desktop 1:1 and perform pan/zoom scaling
+  // and cropping on the managed canvas.
+  view_size_ = source_size;
+  clip_area_ = SkIRect::MakeSize(view_size_);
+  frame_producer_->SetOutputSizeAndClip(view_size_, clip_area_);
+
+  // Unless being destructed, allocate buffer and start drawing frames onto it.
+  frame_producer_->RequestReturnBuffers(base::Bind(
+      &JniFrameConsumer::AllocateBuffer, base::Unretained(this)));
+}
+
+void JniFrameConsumer::AllocateBuffer() {
+  // Only do anything if we're not being destructed.
+  if (provide_buffer_) {
+    if (!ChromotingJni::GetInstance()->
+        display_task_runner()->BelongsToCurrentThread()) {
+      ChromotingJni::GetInstance()->display_task_runner()->PostTask(FROM_HERE,
+          base::Bind(&JniFrameConsumer::AllocateBuffer,
+                     base::Unretained(this)));
+      return;
+    }
+
+    DirectDesktopFrame* buffer = new DirectDesktopFrame(view_size_.width(),
+                                                        view_size_.height());
+
+    // Update Java's reference to the buffer and record of its dimensions.
+    ChromotingJni::GetInstance()->UpdateImageBuffer(
+        view_size_.width(),
+        view_size_.height(),
+        buffer->buffer());
+
+    frame_producer_->DrawBuffer(buffer);
+  }
+}
+
+}  // namespace remoting
diff --git a/remoting/client/jni/jni_frame_consumer.h b/remoting/client/jni/jni_frame_consumer.h
new file mode 100644
index 0000000..b74681f
--- /dev/null
+++ b/remoting/client/jni/jni_frame_consumer.h
@@ -0,0 +1,58 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef REMOTING_CLIENT_JNI_JNI_FRAME_CONSUMER_H_
+#define REMOTING_CLIENT_JNI_JNI_FRAME_CONSUMER_H_
+
+#include "remoting/client/frame_consumer.h"
+
+#include "base/compiler_specific.h"
+
+namespace webrtc {
+class DesktopFrame;
+}  // namespace webrtc
+
+namespace remoting {
+class ChromotingJni;
+class FrameProducer;
+
+// FrameConsumer implementation that draws onto a JNI direct byte buffer.
+class JniFrameConsumer : public FrameConsumer {
+ public:
+  JniFrameConsumer();
+  virtual ~JniFrameConsumer();
+
+  // This must be called once before the producer's source size is set.
+  void set_frame_producer(FrameProducer* producer);
+
+  // FrameConsumer implementation.
+  virtual void ApplyBuffer(const SkISize& view_size,
+                           const SkIRect& clip_area,
+                           webrtc::DesktopFrame* buffer,
+                           const SkRegion& region) OVERRIDE;
+  virtual void ReturnBuffer(webrtc::DesktopFrame* buffer) OVERRIDE;
+  virtual void SetSourceSize(const SkISize& source_size,
+                             const SkIPoint& dpi) OVERRIDE;
+
+ private:
+  // Variables are to be used from the display thread.
+
+  // Whether to allocate/provide the producer with a buffer when able. This
+  // goes to false during destruction so that we don't leak memory.
+  bool provide_buffer_;
+
+  FrameProducer* frame_producer_;
+  SkISize view_size_;
+  SkIRect clip_area_;
+
+  // If |provide_buffer_|, allocates a new buffer of |view_size_|, informs
+  // Java about it, and tells the producer to draw onto it. Otherwise, no-op.
+  void AllocateBuffer();
+
+  DISALLOW_COPY_AND_ASSIGN(JniFrameConsumer);
+};
+
+}  // namespace remoting
+
+#endif
diff --git a/remoting/client/jni/jni_interface.cc b/remoting/client/jni/jni_interface.cc
index d8f03c9..ad42cc7 100644
--- a/remoting/client/jni/jni_interface.cc
+++ b/remoting/client/jni/jni_interface.cc
@@ -20,7 +20,7 @@
 
 // Class and package name of the Java class that declares this file's functions.
 #define JNI_IMPLEMENTATION(method) \
-        Java_org_chromium_chromoting_jni_JNIInterface_##method
+        Java_org_chromium_chromoting_jni_JniInterface_##method
 
 extern "C" {
 
@@ -107,4 +107,10 @@ JNIEXPORT void JNICALL JNI_IMPLEMENTATION(authenticationResponse)(
   env->ReleaseStringUTFChars(pin_jstr, pin_cstr);
 }
 
+JNIEXPORT void JNICALL JNI_IMPLEMENTATION(scheduleRedrawNative)(
+    JNIEnv* env,
+    jobject that) {
+  remoting::ChromotingJni::GetInstance()->session()->RedrawDesktop();
+}
+
 }  // extern "C"
diff --git a/remoting/remoting.gyp b/remoting/remoting.gyp
index 28363d6..d6d5bd3 100644
--- a/remoting/remoting.gyp
+++ b/remoting/remoting.gyp
@@ -1824,6 +1824,8 @@
             'client/jni/chromoting_jni.h',
             'client/jni/chromoting_jni_instance.cc',
             'client/jni/chromoting_jni_instance.h',
+            'client/jni/jni_frame_consumer.cc',
+            'client/jni/jni_frame_consumer.h',
             'client/jni/jni_interface.cc',
           ],
         },  # end of target 'remoting_client_jni'