Implement new video encoder performance tests, for VP9/VP8 comparison.

The old encoder perf tests (in VideoEncoderVpxTest) were not providing
very realistic results because:
 - They were measuring performance on frames with random data.
 - They were testing the case when the whole frame is changing every
   time.
 - They were using real clock, which affects results.

This change implements new tests that provide closer approximation to
how the codec is being used.

CyclicFrameGenerator added in this test will also be used to test
performance of the new WebRTC-based protocol.

Review URL: https://codereview.chromium.org/1601303002

Cr-Commit-Position: refs/heads/master@{#370491}

6 files changed, 389 insertions, 0 deletions

diff --git a/remoting/test/BUILD.gn b/remoting/test/BUILD.gn
index fd014ff..8bc132f 100644
--- a/remoting/test/BUILD.gn
+++ b/remoting/test/BUILD.gn
@@ -18,6 +18,8 @@ source_set("test_support") {
     "connection_setup_info.h",
     "connection_time_observer.cc",
     "connection_time_observer.h",
+    "cyclic_frame_generator.cc",
+    "cyclic_frame_generator.h",
     "fake_access_token_fetcher.cc",
     "fake_access_token_fetcher.h",
     "fake_app_remoting_report_issue_request.cc",
@@ -76,6 +78,7 @@ source_set("test_support") {
 
   deps = [
     "//google_apis",
+    "//skia",
     "//testing/gmock",
     "//testing/gtest",
     "//third_party/libjingle",
diff --git a/remoting/test/DEPS b/remoting/test/DEPS
index ef74945..485d290 100644
--- a/remoting/test/DEPS
+++ b/remoting/test/DEPS
@@ -8,4 +8,5 @@ include_rules = [
   "+remoting/signaling",
   "+ui/gfx",
   "+ui/events/keycodes/dom",
+  "+third_party/skia",
 ]
diff --git a/remoting/test/codec_perftest.cc b/remoting/test/codec_perftest.cc
new file mode 100644
index 0000000..e157b88
--- /dev/null
+++ b/remoting/test/codec_perftest.cc
@@ -0,0 +1,158 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/test/simple_test_tick_clock.h"
+#include "remoting/codec/video_encoder_vpx.h"
+#include "remoting/proto/video.pb.h"
+#include "remoting/test/cyclic_frame_generator.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
+
+namespace remoting {
+namespace test {
+
+static const int kIntervalBetweenFramesMs = 33;
+
+struct CodecParams {
+  CodecParams(bool use_vp9, bool lossless, bool lossless_color)
+      : use_vp9(use_vp9), lossless(lossless), lossless_color(lossless_color) {}
+
+  bool use_vp9;
+  bool lossless;
+  bool lossless_color;
+};
+
+class CodecPerfTest : public testing::Test,
+                      public testing::WithParamInterface<CodecParams> {
+ public:
+  void SetUp() override {
+    if (GetParam().use_vp9) {
+      encoder_ = VideoEncoderVpx::CreateForVP9();
+      encoder_->SetLosslessEncode(GetParam().lossless);
+      encoder_->SetLosslessColor(GetParam().lossless_color);
+    } else {
+      encoder_ = VideoEncoderVpx::CreateForVP8();
+    }
+    encoder_->SetTickClockForTests(&clock_);
+
+    frame_generator_ = CyclicFrameGenerator::Create();
+    frame_generator_->SetTickClock(&clock_);
+  }
+
+ protected:
+  base::SimpleTestTickClock clock_;
+  scoped_refptr<CyclicFrameGenerator> frame_generator_;
+  scoped_ptr<VideoEncoderVpx> encoder_;
+};
+
+INSTANTIATE_TEST_CASE_P(VP8,
+                        CodecPerfTest,
+                        ::testing::Values(CodecParams(false, false, false)));
+INSTANTIATE_TEST_CASE_P(VP9,
+                        CodecPerfTest,
+                        ::testing::Values(CodecParams(true, false, false)));
+INSTANTIATE_TEST_CASE_P(VP9Lossless,
+                        CodecPerfTest,
+                        ::testing::Values(CodecParams(true, true, false)));
+INSTANTIATE_TEST_CASE_P(VP9LosslessColor,
+                        CodecPerfTest,
+                        ::testing::Values(CodecParams(true, false, true)));
+
+TEST_P(CodecPerfTest, EncodeLatency) {
+  const int kTotalFrames = 300;
+  base::TimeDelta total_latency;
+
+  base::TimeDelta total_latency_big_frames;
+  int big_frame_count = 0;
+  base::TimeDelta total_latency_small_frames;
+  int small_frame_count = 0;
+  base::TimeDelta total_latency_empty_frames;
+  int empty_frame_count = 0;
+
+  int total_bytes = 0;
+
+  for (int i = 0; i < kTotalFrames; ++i) {
+    scoped_ptr<webrtc::DesktopFrame> frame =
+        frame_generator_->GenerateFrame(nullptr);
+    base::TimeTicks started = base::TimeTicks::Now();
+
+    scoped_ptr<VideoPacket> packet = encoder_->Encode(*frame);
+
+    base::TimeTicks ended = base::TimeTicks::Now();
+    base::TimeDelta latency = ended - started;
+
+    total_latency += latency;
+    if (packet)
+      total_bytes += packet->data().size();
+
+    switch (frame_generator_->last_frame_type()) {
+      case CyclicFrameGenerator::FrameType::EMPTY:
+        total_latency_empty_frames += latency;
+        ++empty_frame_count;
+        break;
+      case CyclicFrameGenerator::FrameType::FULL:
+        total_latency_big_frames += latency;
+        ++big_frame_count;
+        break;
+      case CyclicFrameGenerator::FrameType::CURSOR:
+        total_latency_small_frames += latency;
+        ++small_frame_count;
+        break;
+    }
+
+    clock_.Advance(base::TimeDelta::FromMilliseconds(kIntervalBetweenFramesMs));
+  }
+
+  VLOG(0) << "Total time: " << total_latency.InMillisecondsF();
+  VLOG(0) << "Average encode latency: "
+          << (total_latency / kTotalFrames).InMillisecondsF();
+
+  CHECK(big_frame_count);
+  VLOG(0) << "Average encode latency for big frames: "
+          << (total_latency_big_frames / big_frame_count).InMillisecondsF();
+
+  if (small_frame_count) {
+    VLOG(0) << "Average encode latency for small frames: "
+            << (total_latency_small_frames / small_frame_count)
+                   .InMillisecondsF();
+  }
+
+  if (empty_frame_count) {
+    VLOG(0) << "Average encode latency for empty frames: "
+            << (total_latency_empty_frames / empty_frame_count)
+                   .InMillisecondsF();
+  }
+
+  VLOG(0) << "Encoded bytes: " << total_bytes;
+}
+
+TEST_P(CodecPerfTest, MaxFramerate) {
+  const int kTotalFrames = 100;
+  base::TimeDelta total_latency;
+
+  // Update the whole screen on every frame.
+  frame_generator_->set_frame_cycle_period(
+      base::TimeDelta::FromMilliseconds(kIntervalBetweenFramesMs));
+
+  for (int i = 0; i < kTotalFrames; ++i) {
+    scoped_ptr<webrtc::DesktopFrame> frame =
+        frame_generator_->GenerateFrame(nullptr);
+    base::TimeTicks started = base::TimeTicks::Now();
+
+    scoped_ptr<VideoPacket> packet = encoder_->Encode(*frame);
+
+    base::TimeTicks ended = base::TimeTicks::Now();
+    base::TimeDelta latency = ended - started;
+
+    total_latency += latency;
+
+    clock_.Advance(base::TimeDelta::FromMilliseconds(kIntervalBetweenFramesMs));
+  }
+
+  VLOG(0) << "Max framerate: "
+          << (kTotalFrames * base::TimeDelta::FromSeconds(1) / total_latency);
+}
+
+}  // namespace test
+}  // namespace remoting
diff --git a/remoting/test/cyclic_frame_generator.cc b/remoting/test/cyclic_frame_generator.cc
new file mode 100644
index 0000000..2fb3753
--- /dev/null
+++ b/remoting/test/cyclic_frame_generator.cc
@@ -0,0 +1,119 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "remoting/test/cyclic_frame_generator.h"
+
+#include "base/base_paths.h"
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "base/time/default_tick_clock.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
+#include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
+#include "ui/gfx/codec/png_codec.h"
+
+namespace remoting {
+namespace test {
+
+namespace {
+
+scoped_ptr<webrtc::DesktopFrame> LoadDesktopFrameFromPng(
+    const base::FilePath& file_path) {
+  std::string file_content;
+  if (!base::ReadFileToString(file_path, &file_content))
+    LOG(FATAL) << "Failed to read " << file_path.MaybeAsASCII()
+               << ". Please run remoting/test/data/download.sh";
+  SkBitmap bitmap;
+  gfx::PNGCodec::Decode(reinterpret_cast<const uint8_t*>(file_content.data()),
+                        file_content.size(), &bitmap);
+  scoped_ptr<webrtc::DesktopFrame> frame(new webrtc::BasicDesktopFrame(
+      webrtc::DesktopSize(bitmap.width(), bitmap.height())));
+  bitmap.copyPixelsTo(frame->data(),
+                      frame->stride() * frame->size().height(),
+                      frame->stride());
+  return frame;
+}
+
+}  // namespace
+
+// static
+scoped_refptr<CyclicFrameGenerator> CyclicFrameGenerator::Create() {
+  base::FilePath test_data_path;
+  PathService::Get(base::DIR_SOURCE_ROOT, &test_data_path);
+  test_data_path = test_data_path.Append(FILE_PATH_LITERAL("remoting"));
+  test_data_path = test_data_path.Append(FILE_PATH_LITERAL("test"));
+  test_data_path = test_data_path.Append(FILE_PATH_LITERAL("data"));
+
+  std::vector<scoped_ptr<webrtc::DesktopFrame>> frames;
+  frames.push_back(
+      LoadDesktopFrameFromPng(test_data_path.AppendASCII("test_frame1.png")));
+  frames.push_back(
+      LoadDesktopFrameFromPng(test_data_path.AppendASCII("test_frame2.png")));
+  return new CyclicFrameGenerator(std::move(frames));
+}
+
+CyclicFrameGenerator::CyclicFrameGenerator(
+    std::vector<scoped_ptr<webrtc::DesktopFrame>> reference_frames)
+    : reference_frames_(std::move(reference_frames)),
+      clock_(&default_tick_clock_),
+      started_time_(clock_->NowTicks()) {
+  CHECK(!reference_frames_.empty());
+  screen_size_ = reference_frames_[0]->size();
+  for (const auto& frame : reference_frames_) {
+    CHECK(screen_size_.equals(frame->size()))
+        << "All reference frames should have the same size.";
+  }
+}
+CyclicFrameGenerator::~CyclicFrameGenerator() {}
+
+void CyclicFrameGenerator::SetTickClock(base::TickClock* tick_clock) {
+  clock_ = tick_clock;
+  started_time_ = clock_->NowTicks();
+}
+
+scoped_ptr<webrtc::DesktopFrame> CyclicFrameGenerator::GenerateFrame(
+    webrtc::DesktopCapturer::Callback* callback) {
+  base::TimeTicks now = clock_->NowTicks();
+  int reference_frame =
+      ((now - started_time_) / frame_cycle_period_) % reference_frames_.size();
+  bool cursor_state = ((now - started_time_) / cursor_blink_period_) % 2;
+
+  scoped_ptr<webrtc::DesktopFrame> frame(
+      new webrtc::BasicDesktopFrame(screen_size_));
+  frame->CopyPixelsFrom(*reference_frames_[reference_frame],
+                        webrtc::DesktopVector(),
+                        webrtc::DesktopRect::MakeSize(screen_size_));
+
+  // Render the cursor.
+  webrtc::DesktopRect cursor_rect =
+      webrtc::DesktopRect::MakeXYWH(20, 20, 2, 20);
+  if (cursor_state) {
+    for (int y = cursor_rect.top(); y < cursor_rect.bottom(); ++y) {
+      memset(frame->data() + y * frame->stride() +
+                 cursor_rect.left() * webrtc::DesktopFrame::kBytesPerPixel,
+             0, cursor_rect.width() * webrtc::DesktopFrame::kBytesPerPixel);
+    }
+  }
+
+  if (last_reference_frame_ != reference_frame) {
+    // The whole frame has changed.
+    frame->mutable_updated_region()->AddRect(
+        webrtc::DesktopRect::MakeSize(screen_size_));
+    last_frame_type_ = FrameType::FULL;
+  } else if (last_cursor_state_ != cursor_state) {
+    // Cursor state has changed.
+    frame->mutable_updated_region()->AddRect(cursor_rect);
+    last_frame_type_ = FrameType::CURSOR;
+  } else {
+    // No changes.
+    last_frame_type_ = FrameType::EMPTY;
+  }
+  last_reference_frame_ = reference_frame;
+  last_cursor_state_ = cursor_state;
+
+  return frame;
+}
+
+}  // namespace test
+}  // namespace remoting
diff --git a/remoting/test/cyclic_frame_generator.h b/remoting/test/cyclic_frame_generator.h
new file mode 100644
index 0000000..50acad0
--- /dev/null
+++ b/remoting/test/cyclic_frame_generator.h
@@ -0,0 +1,88 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef REMOTING_TEST_CYCLIC_FRAME_GENERATOR_H_
+#define REMOTING_TEST_CYCLIC_FRAME_GENERATOR_H_
+
+#include <vector>
+
+#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/time/default_tick_clock.h"
+#include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
+
+namespace remoting {
+namespace test {
+
+// CyclicFrameGenerator generates a sequence of frames that approximates
+// properties of a real video stream when using a desktop applications. It
+// loads a sequence of reference frames and switches between them with the
+// specified frequency (every 2 seconds by default). Between reference frames it
+// also generate frames with small changes which simulate a blinking cursor.
+class CyclicFrameGenerator
+    : public base::RefCountedThreadSafe<CyclicFrameGenerator> {
+ public:
+  enum class FrameType {
+    // Frame had no changes.
+    EMPTY,
+
+    // Whole screen changed.
+    FULL,
+
+    // Cursor state has changed.
+    CURSOR,
+  };
+
+  static scoped_refptr<CyclicFrameGenerator> Create();
+
+  CyclicFrameGenerator(
+      std::vector<scoped_ptr<webrtc::DesktopFrame>> reference_frames);
+
+  void set_frame_cycle_period(base::TimeDelta frame_cycle_period) {
+    frame_cycle_period_ = frame_cycle_period;
+  }
+
+  void set_cursor_blink_period(base::TimeDelta cursor_blink_period) {
+    cursor_blink_period_ = cursor_blink_period;
+  }
+
+  void SetTickClock(base::TickClock* tick_clock);
+
+  scoped_ptr<webrtc::DesktopFrame> GenerateFrame(
+      webrtc::DesktopCapturer::Callback* callback);
+
+  FrameType last_frame_type() { return last_frame_type_; }
+
+ private:
+  ~CyclicFrameGenerator();
+  friend class base::RefCountedThreadSafe<CyclicFrameGenerator>;
+
+  std::vector<scoped_ptr<webrtc::DesktopFrame>> reference_frames_;
+  base::DefaultTickClock default_tick_clock_;
+  base::TickClock* clock_;
+  webrtc::DesktopSize screen_size_;
+
+  // By default switch between reference frames every 2 seconds.
+  base::TimeDelta frame_cycle_period_ = base::TimeDelta::FromSeconds(2);
+
+  // By default blink the cursor 4 times per seconds.
+  base::TimeDelta cursor_blink_period_ = base::TimeDelta::FromMilliseconds(250);
+
+  // Index of the reference frame used to render the last generated frame.
+  int last_reference_frame_ = -1;
+
+  // True if the cursor was rendered on the last generated frame.
+  bool last_cursor_state_ = false;
+
+  FrameType last_frame_type_ = FrameType::EMPTY;
+
+  base::TimeTicks started_time_;
+
+  DISALLOW_COPY_AND_ASSIGN(CyclicFrameGenerator);
+};
+
+}  // namespace test
+}  // namespace remoting
+
+#endif  // REMOTING_TEST_CYCLIC_FRAME_GENERATOR_H_
diff --git a/remoting/test/data/download.sh b/remoting/test/data/download.sh
new file mode 100755
index 0000000..106a883
--- /dev/null
+++ b/remoting/test/data/download.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script downloads test files used by some remoting perf tests.The files
+# are stored on Google Cloud Storage.
+
+set -e
+
+SRC_DIR="$(readlink -f "$(dirname "$0")")"
+
+for file_index in 1 2; do
+  file_name=test_frame${file_index}.png
+  file_path="${SRC_DIR}/${file_name}"
+  if [ ! -e "${file_path}" ] ; then
+    curl -L "https://storage.googleapis.com/chromoting-test-data/${file_name}" \
+      > "${file_path}"
+  fi
+done