Chromoting client: video renderer based on MediaSource API.

The new render is not enabled yet by default, but can be enabled using
remoting.settings.USE_MEDIA_SOURCE_RENDERING flag in the webapp.

BUG=321825

Review URL: https://codereview.chromium.org/134163005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@249338 0039d316-1c4b-4281-b951-d872f2087c98

7 files changed, 303 insertions, 35 deletions

diff --git a/remoting/webapp/client_plugin.js b/remoting/webapp/client_plugin.js
index 7ef05c7..60cb6cb 100644
--- a/remoting/webapp/client_plugin.js
+++ b/remoting/webapp/client_plugin.js
@@ -43,6 +43,7 @@ remoting.ClientPlugin = function(plugin, onExtensionMessage) {
   this.onConnectionStatusUpdateHandler = function(state, error) {};
   /** @param {boolean} ready Connection ready state. */
   this.onConnectionReadyHandler = function(ready) {};
+
   /**
    * @param {string} tokenUrl Token-request URL, received from the host.
    * @param {string} hostPublicKey Public key for the host.
@@ -55,6 +56,9 @@ remoting.ClientPlugin = function(plugin, onExtensionMessage) {
   this.onSetCapabilitiesHandler = function (capabilities) {};
   this.fetchPinHandler = function (supportsPairing) {};
 
+  /** @type {remoting.MediaSourceRenderer} */
+  this.mediaSourceRenderer_ = null;
+
   /** @type {number} */
   this.pluginApiVersion_ = -1;
   /** @type {Array.<string>} */
@@ -97,7 +101,8 @@ remoting.ClientPlugin.Feature = {
   THIRD_PARTY_AUTH: 'thirdPartyAuth',
   TRAP_KEY: 'trapKey',
   PINLESS_AUTH: 'pinlessAuth',
-  EXTENSION_MESSAGE: 'extensionMessage'
+  EXTENSION_MESSAGE: 'extensionMessage',
+  MEDIA_SOURCE_RENDERING: 'mediaSourceRendering'
 };
 
 /**
@@ -121,14 +126,18 @@ remoting.ClientPlugin.prototype.API_VERSION_ = 6;
 remoting.ClientPlugin.prototype.API_MIN_VERSION_ = 5;
 
 /**
- * @param {string} messageStr Message from the plugin.
+ * @param {string|{method:string, data:Object.<string,*>}}
+ *    rawMessage Message from the plugin.
+ * @private
  */
-remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
-  var message = /** @type {{method:string, data:Object.<string,string>}} */
-      jsonParseSafe(messageStr);
+remoting.ClientPlugin.prototype.handleMessage_ = function(rawMessage) {
+  var message =
+      /** @type {{method:string, data:Object.<string,*>}} */
+      ((typeof(rawMessage) == 'string') ? jsonParseSafe(rawMessage)
+                                        : rawMessage);
 
   if (!message || !('method' in message) || !('data' in message)) {
-    console.error('Received invalid message from the plugin: ' + messageStr);
+    console.error('Received invalid message from the plugin:', rawMessage);
     return;
   }
 
@@ -149,18 +158,18 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
     this.plugin.height = 0;
     if (typeof message.data['apiVersion'] != 'number' ||
         typeof message.data['apiMinVersion'] != 'number') {
-      console.error('Received invalid hello message: ' + messageStr);
+      console.error('Received invalid hello message:', rawMessage);
       return;
     }
     this.pluginApiVersion_ = /** @type {number} */ message.data['apiVersion'];
 
     if (this.pluginApiVersion_ >= 7) {
       if (typeof message.data['apiFeatures'] != 'string') {
-        console.error('Received invalid hello message: ' + messageStr);
+        console.error('Received invalid hello message:', rawMessage);
         return;
       }
       this.pluginApiFeatures_ =
-          /** @type {Array.<string>} */ tokenize(message.data['apiFeatures']);
+          tokenize((/** @type {string} */ message.data['apiFeatures']));
 
       // Negotiate capabilities.
 
@@ -168,20 +177,22 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
       var requestedCapabilities = [];
       if ('requestedCapabilities' in message.data) {
         if (typeof message.data['requestedCapabilities'] != 'string') {
-          console.error('Received invalid hello message: ' + messageStr);
+          console.error('Received invalid hello message:', rawMessage);
           return;
         }
-        requestedCapabilities = tokenize(message.data['requestedCapabilities']);
+        requestedCapabilities = tokenize(
+            (/** @type {string} */ message.data['requestedCapabilities']));
       }
 
       /** @type {!Array.<string>} */
       var supportedCapabilities = [];
       if ('supportedCapabilities' in message.data) {
         if (typeof message.data['supportedCapabilities'] != 'string') {
-          console.error('Received invalid hello message: ' + messageStr);
+          console.error('Received invalid hello message:', rawMessage);
           return;
         }
-        supportedCapabilities = tokenize(message.data['supportedCapabilities']);
+        supportedCapabilities = tokenize(
+            (/** @type {string} */ message.data['requestedCapabilities']));
       }
 
       // At the moment the webapp does not recognize any of
@@ -213,22 +224,22 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
     }
   } else if (message.method == 'sendOutgoingIq') {
     if (typeof message.data['iq'] != 'string') {
-      console.error('Received invalid sendOutgoingIq message: ' + messageStr);
+      console.error('Received invalid sendOutgoingIq message:', rawMessage);
       return;
     }
-    this.onOutgoingIqHandler(message.data['iq']);
+    this.onOutgoingIqHandler((/** @type {string} */ message.data['iq']));
   } else if (message.method == 'logDebugMessage') {
     if (typeof message.data['message'] != 'string') {
-      console.error('Received invalid logDebugMessage message: ' + messageStr);
+      console.error('Received invalid logDebugMessage message:', rawMessage);
       return;
     }
-    this.onDebugMessageHandler(message.data['message']);
+    this.onDebugMessageHandler((/** @type {string} */ message.data['message']));
   } else if (message.method == 'onConnectionStatus') {
     if (typeof message.data['state'] != 'string' ||
         !remoting.ClientSession.State.hasOwnProperty(message.data['state']) ||
         typeof message.data['error'] != 'string') {
-      console.error('Received invalid onConnectionState message: ' +
-                    messageStr);
+      console.error('Received invalid onConnectionState message:',
+                    rawMessage);
       return;
     }
 
@@ -247,7 +258,7 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
   } else if (message.method == 'onDesktopSize') {
     if (typeof message.data['width'] != 'number' ||
         typeof message.data['height'] != 'number') {
-      console.error('Received invalid onDesktopSize message: ' + messageStr);
+      console.error('Received invalid onDesktopSize message:', rawMessage);
       return;
     }
     this.desktopWidth = /** @type {number} */ message.data['width'];
@@ -265,7 +276,7 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
         typeof message.data['decodeLatency'] != 'number' ||
         typeof message.data['renderLatency'] != 'number' ||
         typeof message.data['roundtripLatency'] != 'number') {
-      console.error('Received incorrect onPerfStats message: ' + messageStr);
+      console.error('Received incorrect onPerfStats message:', rawMessage);
       return;
     }
     this.perfStats_ =
@@ -277,8 +288,9 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
       return;
     }
     if (remoting.clipboard) {
-      remoting.clipboard.fromHost(message.data['mimeType'],
-                                  message.data['item']);
+      remoting.clipboard.fromHost(
+          (/** @type {string} */ message.data['mimeType']),
+          (/** @type {string} */ message.data['item']));
     }
   } else if (message.method == 'onFirstFrameReceived') {
     if (remoting.clientSession) {
@@ -313,7 +325,8 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
     }
 
     /** @type {!Array.<string>} */
-    var capabilities = tokenize(message.data['capabilities']);
+    var capabilities =
+        tokenize((/** @type {string} */ message.data['capabilities']));
     this.onSetCapabilitiesHandler(capabilities);
   } else if (message.method == 'fetchThirdPartyToken') {
     if (typeof message.data['tokenUrl'] != 'string' ||
@@ -352,6 +365,28 @@ remoting.ClientPlugin.prototype.handleMessage_ = function(messageStr) {
                       message.data['type'] + ': ' + message.data['data']);
         }
     }
+  } else if (message.method == 'mediaSourceReset') {
+    if (typeof(message.data['format']) != 'string') {
+      console.error('Invalid mediaSourceReset message:', message.data);
+      return;
+    }
+    if (!this.mediaSourceRenderer_) {
+      console.error('Unexpected mediaSourceReset.');
+      return;
+    }
+    this.mediaSourceRenderer_.reset(
+        (/** @type {string} */ message.data['format']));
+  } else if (message.method == 'mediaSourceData') {
+    if (!(message.data['buffer'] instanceof ArrayBuffer)) {
+      console.error('Invalid mediaSourceData message:', message.data);
+      return;
+    }
+    if (!this.mediaSourceRenderer_) {
+      console.error('Unexpected mediaSourceData.');
+      return;
+    }
+    this.mediaSourceRenderer_.onIncomingData(
+        (/** @type {ArrayBuffer} */ message.data['buffer']));
   }
 };
 
@@ -565,8 +600,9 @@ remoting.ClientPlugin.prototype.notifyClientResolution =
  */
 remoting.ClientPlugin.prototype.pauseVideo =
     function(pause) {
-  if (!this.hasFeature(remoting.ClientPlugin.Feature.PAUSE_VIDEO))
+  if (!this.hasFeature(remoting.ClientPlugin.Feature.PAUSE_VIDEO)) {
     return;
+  }
   this.plugin.postMessage(JSON.stringify(
       { method: 'pauseVideo', data: { pause: pause }}));
 };
@@ -578,8 +614,9 @@ remoting.ClientPlugin.prototype.pauseVideo =
  */
 remoting.ClientPlugin.prototype.pauseAudio =
     function(pause) {
-  if (!this.hasFeature(remoting.ClientPlugin.Feature.PAUSE_AUDIO))
+  if (!this.hasFeature(remoting.ClientPlugin.Feature.PAUSE_AUDIO)) {
     return;
+  }
   this.plugin.postMessage(JSON.stringify(
       { method: 'pauseAudio', data: { pause: pause }}));
 };
@@ -658,6 +695,21 @@ remoting.ClientPlugin.prototype.sendClientMessage =
 };
 
 /**
+ * Request MediaStream-based rendering.
+ *
+ * @param {remoting.MediaSourceRenderer} mediaSourceRenderer
+ */
+remoting.ClientPlugin.prototype.enableMediaSourceRendering =
+    function(mediaSourceRenderer) {
+  if (!this.hasFeature(remoting.ClientPlugin.Feature.MEDIA_SOURCE_RENDERING)) {
+    return;
+  }
+  this.mediaSourceRenderer_ = mediaSourceRenderer;
+  this.plugin.postMessage(JSON.stringify(
+      { method: 'enableMediaSourceRendering', data: {} }));
+};
+
+/**
  * If we haven't yet received a "hello" message from the plugin, change its
  * size so that the user can confirm it if click-to-play is enabled, or can
  * see the "this plugin is disabled" message if it is actually disabled.
diff --git a/remoting/webapp/client_session.js b/remoting/webapp/client_session.js
index 021446a..3a78b4e 100644
--- a/remoting/webapp/client_session.js
+++ b/remoting/webapp/client_session.js
@@ -124,6 +124,9 @@ remoting.ClientSession = function(accessCode, fetchPin, fetchThirdPartyToken,
       document.getElementById('send-keys-menu')
   );
 
+  /** @type {HTMLMediaElement} @private */
+  this.video_ = null;
+
   /** @type {HTMLElement} @private */
   this.resizeToClientButton_ =
       document.getElementById('screen-resize-to-client');
@@ -448,6 +451,26 @@ remoting.ClientSession.prototype.onPluginInitialized_ = function(initialized) {
     this.applyRemapKeys_(true);
   }
 
+  // Enable MediaSource-based rendering if available.
+  if (remoting.settings.USE_MEDIA_SOURCE_RENDERING &&
+      this.plugin_.hasFeature(
+          remoting.ClientPlugin.Feature.MEDIA_SOURCE_RENDERING)) {
+    this.video_ = /** @type {HTMLMediaElement} */(
+        document.getElementById('mediasource-video-output'));
+    // Make sure that the <video> element is hidden until we get the first
+    // frame.
+    this.video_.style.width = '0px';
+    this.video_.style.height = '0px';
+
+    var renderer = new remoting.MediaSourceRenderer(this.video_);
+    this.plugin_.enableMediaSourceRendering(renderer);
+    /** @type {HTMLElement} */(document.getElementById('video-container'))
+        .classList.add('mediasource-rendering');
+  } else {
+    /** @type {HTMLElement} */(document.getElementById('video-container'))
+        .classList.remove('mediasource-rendering');
+  }
+
   /** @param {string} msg The IQ stanza to send. */
   this.plugin_.onOutgoingIqHandler = this.sendIq_.bind(this);
   /** @param {string} msg The message to log. */
@@ -493,6 +516,11 @@ remoting.ClientSession.prototype.removePlugin = function() {
 
   // In case the user had selected full-screen mode, cancel it now.
   document.webkitCancelFullScreen();
+
+  // Remove mediasource-rendering class from video-contained - this will also
+  // hide the <video> element.
+  /** @type {HTMLElement} */(document.getElementById('video-container'))
+      .classList.remove('mediasource-rendering');
 };
 
 /**
@@ -1095,13 +1123,18 @@ remoting.ClientSession.prototype.updateDimensions = function() {
     }
   }
 
-  var pluginWidth = desktopWidth * scale;
-  var pluginHeight = desktopHeight * scale;
+  var pluginWidth = Math.round(desktopWidth * scale);
+  var pluginHeight = Math.round(desktopHeight * scale);
+
+  if (this.video_) {
+    this.video_.style.width = pluginWidth + 'px';
+    this.video_.style.height = pluginHeight + 'px';
+  }
 
   // Resize the plugin if necessary.
   // TODO(wez): Handle high-DPI to high-DPI properly (crbug.com/135089).
-  this.plugin_.element().width = pluginWidth;
-  this.plugin_.element().height = pluginHeight;
+  this.plugin_.element().style.width = pluginWidth + 'px';
+  this.plugin_.element().style.height = pluginHeight + 'px';
 
   // Position the container.
   // Note that clientWidth/Height take into account scrollbars.
diff --git a/remoting/webapp/js_proto/dom_proto.js b/remoting/webapp/js_proto/dom_proto.js
index 0ab4e4a..64f330f 100644
--- a/remoting/webapp/js_proto/dom_proto.js
+++ b/remoting/webapp/js_proto/dom_proto.js
@@ -85,12 +85,16 @@ Node.prototype.value;
 Node.prototype.style;
 
 
-/** @constructor
-    @param {function(Array.<MutationRecord>):void} callback */
+/**
+ * @constructor
+ * @param {function(Array.<MutationRecord>):void} callback
+ */
 var MutationObserver = function(callback) {};
 
-/** @param {Element} element
-    @param {Object} options */
+/**
+ * @param {Element} element
+ * @param {Object} options
+ */
 MutationObserver.prototype.observe = function(element, options) {};
 
 
@@ -109,3 +113,57 @@ MutationRecord.prototype.type;
 
 /** @type {{getRandomValues: function((Uint16Array|Uint8Array)):void}} */
 Window.prototype.crypto;
+
+
+/**
+ * @constructor
+ * @implements {EventTarget} */
+var EventTargetStub = function() {};
+
+/**
+ * @param {string} type
+ * @param {(EventListener|function(Event): (boolean|undefined|null))} listener
+ * @param {boolean=} opt_useCapture
+ */
+EventTargetStub.prototype.addEventListener =
+    function(type, listener, opt_useCapture) {}
+
+/**
+ * @param {string} type
+ * @param {(EventListener|function(Event): (boolean|undefined|null))} listener
+ * @param {boolean=} opt_useCapture
+ */
+EventTargetStub.prototype.removeEventListener =
+    function(type, listener, opt_useCapture) {}
+
+/**
+ * @param {Event} event
+ */
+EventTargetStub.prototype.dispatchEvent =
+    function(event) {}
+
+/**
+ * @constructor
+ * @extends {EventTargetStub}
+ */
+var SourceBuffer = function() {}
+
+/** @type {boolean} */
+SourceBuffer.prototype.updating;
+
+/**
+ * @param {ArrayBuffer} buffer
+ */
+SourceBuffer.prototype.appendBuffer = function(buffer) {}
+
+/**
+ * @constructor
+ * @extends {EventTargetStub}
+ */
+var MediaSource = function() {}
+
+/**
+ * @param {string} format
+ * @return {SourceBuffer}
+ */
+MediaSource.prototype.addSourceBuffer = function(format) {}
diff --git a/remoting/webapp/main.css b/remoting/webapp/main.css
index b5b75b6..7673498 100644
--- a/remoting/webapp/main.css
+++ b/remoting/webapp/main.css
@@ -739,3 +739,35 @@ html.apps-v2.scrollable {
   left: 0;
   width: 100%;
 }
+
+/* video-container needs relative position so that mediasource-video-output can
+ * be positioned relative to the parent with position:absolute. */
+#video-container {
+  position: relative;
+}
+
+/* mediasource-video-output is hidden by default. */
+#mediasource-video-output {
+  display: none;
+}
+
+/* Use absolute positioning for mediasource-video-output so that it's rendered
+ * at the same position as the plugin. */
+#video-container.mediasource-rendering #mediasource-video-output {
+  display: block;
+  position: absolute;
+  left: 0;
+  top: 0;
+  box-shadow: 0 0 8px 0 black;
+}
+
+/*
+ * With MediaSource-based rendering the plugin is transparent and is placed on
+ * top of the <video> element so that it can still receive mouse events.
+ *
+ * TODO(sergeyu): This is temporary solution. Ideally mouse and keyboard events
+ * should be captured on JS level and passed to the plugin.
+ */
+#video-container.mediasource-rendering #client-plugin-container {
+  opacity: 0;
+}
diff --git a/remoting/webapp/main.html b/remoting/webapp/main.html
index f99c7a8..9106c2d 100644
--- a/remoting/webapp/main.html
+++ b/remoting/webapp/main.html
@@ -39,6 +39,7 @@ found in the LICENSE file.
     <script src="identity.js"></script>
     <script src="l10n.js"></script>
     <script src="log_to_server.js"></script>
+    <script src="media_source_renderer.js"></script>
     <script src="menu_button.js"></script>
     <script src="oauth2.js"></script>
     <script src="oauth2_api.js"></script>
@@ -792,7 +793,10 @@ found in the LICENSE file.
       </div> <!-- session-toolbar -->
       <div class="vertically-centered">
         <div class="horizontally-centered">
-          <div id="client-plugin-container"></div>
+          <div id="video-container">
+            <video id="mediasource-video-output"></video>
+            <div id="client-plugin-container"></div>
+          </div>
         </div>
       </div>
     </div> <!-- session-mode -->
diff --git a/remoting/webapp/media_source_renderer.js b/remoting/webapp/media_source_renderer.js
new file mode 100644
index 0000000..a79173b
--- /dev/null
+++ b/remoting/webapp/media_source_renderer.js
@@ -0,0 +1,86 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+'use strict';
+
+/** @suppress {duplicate} */
+var remoting = remoting || {};
+
+/**
+ * @param {HTMLMediaElement} videoTag <video> tag to render to.
+ * @constructor
+ */
+remoting.MediaSourceRenderer = function(videoTag) {
+  /** @type {HTMLMediaElement} */
+  this.video_ = videoTag;
+
+  /** @type {MediaSource} */
+  this.mediaSource_ = null;
+
+  /** @type {SourceBuffer} */
+  this.sourceBuffer_ = null;
+
+  /** @type {!Array.<ArrayBuffer>} Queue of pending buffers that haven't been
+   * processed . */
+  this.buffers_ = [];
+}
+
+/**
+ * @param {string} format Format of the stream.
+ */
+remoting.MediaSourceRenderer.prototype.reset = function(format) {
+  // Create a new MediaSource instance.
+  this.sourceBuffer_ = null;
+  this.mediaSource_ = new MediaSource();
+  this.mediaSource_.addEventListener('sourceopen',
+                                     this.onSourceOpen_.bind(this, format));
+  this.mediaSource_.addEventListener('sourceclose', function(e) {
+    console.error("MediaSource closed unexpectedly.");
+  });
+  this.mediaSource_.addEventListener('sourceended', function(e) {
+    console.error("MediaSource ended unexpectedly.");
+  });
+
+  // Start playback from new MediaSource.
+  this.video_.src =
+      /** @type {string} */(
+          window.URL.createObjectURL(/** @type {!Blob} */(this.mediaSource_)));
+  this.video_.play();
+}
+
+/**
+ * @param {string} format
+ * @private
+ */
+remoting.MediaSourceRenderer.prototype.onSourceOpen_ = function(format) {
+  this.sourceBuffer_ =
+      this.mediaSource_.addSourceBuffer(format);
+
+  this.sourceBuffer_.addEventListener(
+      'updateend', this.processPendingData_.bind(this));
+  this.processPendingData_();
+}
+
+/**
+ * @private
+ */
+remoting.MediaSourceRenderer.prototype.processPendingData_ = function() {
+  if (this.sourceBuffer_) {
+    while (this.buffers_.length > 0 && !this.sourceBuffer_.updating) {
+      // TODO(sergeyu): Figure out the way to determine when a frame is rendered
+      // and use it to report performance statistics.
+      this.sourceBuffer_.appendBuffer(
+          /** @type {ArrayBuffer} */(this.buffers_.shift()));
+    }
+  }
+}
+
+/**
+ * @param {ArrayBuffer} data
+ */
+remoting.MediaSourceRenderer.prototype.onIncomingData = function(data) {
+  this.buffers_.push(data);
+  this.processPendingData_();
+}
+
diff --git a/remoting/webapp/plugin_settings.js b/remoting/webapp/plugin_settings.js
index 81e95bd..8d0a166 100644
--- a/remoting/webapp/plugin_settings.js
+++ b/remoting/webapp/plugin_settings.js
@@ -51,3 +51,6 @@ remoting.Settings.prototype.XMPP_SERVER_USE_TLS =
 /** @type {string} The third party auth redirect URI. */
 remoting.Settings.prototype.THIRD_PARTY_AUTH_REDIRECT_URI =
     'THIRD_PARTY_AUTH_REDIRECT_URL';
+
+// Whether to use MediaSource API for video rendering.
+remoting.Settings.prototype.USE_MEDIA_SOURCE_RENDERING = false;