path: root/remoting/test/test_video_renderer_unittest.cc

// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "remoting/test/test_video_renderer.h"

#include <stdint.h>

#include <cmath>
#include <utility>

#include "base/macros.h"
#include "base/memory/scoped_vector.h"
#include "base/message_loop/message_loop.h"
#include "base/run_loop.h"
#include "base/thread_task_runner_handle.h"
#include "base/timer/timer.h"
#include "remoting/codec/video_encoder.h"
#include "remoting/codec/video_encoder_verbatim.h"
#include "remoting/codec/video_encoder_vpx.h"
#include "remoting/proto/video.pb.h"
#include "remoting/test/rgb_value.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
#include "third_party/webrtc/modules/desktop_capture/desktop_region.h"

namespace {

// Used to verify if image pattern is matched.
void ProcessPacketDoneHandler(const base::Closure& done_closure,
                              bool* handler_called) {
  *handler_called = true;
  done_closure.Run();
}

const int kDefaultScreenWidthPx = 1024;
const int kDefaultScreenHeightPx = 768;

// Default max error for encoding and decoding, measured in percent.
const double kDefaultErrorLimit = 0.02;

// Default expected rect for image pattern, measured in pixels.
const webrtc::DesktopRect kDefaultExpectedRect =
    webrtc::DesktopRect::MakeLTRB(100, 100, 200, 200);
}  // namespace

namespace remoting {
namespace test {

// Provides basic functionality for for the TestVideoRenderer Tests below.
// This fixture also creates an MessageLoop to test decoding video packets.
class TestVideoRendererTest : public testing::Test {
 public:
  TestVideoRendererTest();
  ~TestVideoRendererTest() override;

  // Handles creating a frame and sending to TestVideoRenderer for processing.
  void TestVideoPacketProcessing(int screen_width, int screen_height,
                                 double error_limit);

  // Handles setting an image pattern and sending a frame to TestVideoRenderer.
  // |expect_to_match| indicates if the image pattern is expected to match.
  void TestImagePatternMatch(int screen_width,
                             int screen_height,
                             const webrtc::DesktopRect& expected_rect,
                             bool expect_to_match);

  // Generate a basic desktop frame containing a gradient.
  scoped_ptr<webrtc::DesktopFrame> CreateDesktopFrameWithGradient(
      int screen_width, int screen_height) const;

 protected:
  // Used to post tasks to the message loop.
  scoped_ptr<base::RunLoop> run_loop_;

  // Used to set timeouts and delays.
  scoped_ptr<base::Timer> timer_;

  // Manages the decoder and process generated video packets.
  scoped_ptr<TestVideoRenderer> test_video_renderer_;

  // Used to encode desktop frames to generate video packets.
  scoped_ptr<VideoEncoder> encoder_;

 private:
  // testing::Test interface.
  void SetUp() override;

  // Set image pattern, send video packet and returns if the expected pattern is
  // matched.
  bool SendPacketAndWaitForMatch(scoped_ptr<VideoPacket> packet,
                                 const webrtc::DesktopRect& expected_rect,
                                 const RGBValue& expected_average_color);

  // Returns the average color value of pixels fall within |rect|.
  // NOTE: Callers should not release the objects.
  RGBValue CalculateAverageColorValueForFrame(
      const webrtc::DesktopFrame* frame,
      const webrtc::DesktopRect& rect) const;

  // Return the mean error of two frames over all pixels, where error at each
  // pixel is the root mean square of the errors in the R, G and B components,
  // each normalized to [0, 1].
  double CalculateError(const webrtc::DesktopFrame* original_frame,
                        const webrtc::DesktopFrame* decoded_frame) const;

  // Fill a desktop frame with a gradient.
  void FillFrameWithGradient(webrtc::DesktopFrame* frame) const;

  // The thread's message loop. Valid only when the thread is alive.
  scoped_ptr<base::MessageLoop> message_loop_;

  DISALLOW_COPY_AND_ASSIGN(TestVideoRendererTest);
};

TestVideoRendererTest::TestVideoRendererTest()
    : timer_(new base::Timer(true, false)) {}

TestVideoRendererTest::~TestVideoRendererTest() {}

void TestVideoRendererTest::SetUp() {
  if (!base::MessageLoop::current()) {
    // Create a temporary message loop if the current thread does not already
    // have one.
    message_loop_.reset(new base::MessageLoop);
  }
  test_video_renderer_.reset(new TestVideoRenderer());
}

void TestVideoRendererTest::TestVideoPacketProcessing(int screen_width,
                                                      int screen_height,
                                                      double error_limit) {
  DCHECK(encoder_);
  DCHECK(test_video_renderer_);

  // Generate a frame containing a gradient.
  scoped_ptr<webrtc::DesktopFrame> original_frame =
      CreateDesktopFrameWithGradient(screen_width, screen_height);
  EXPECT_TRUE(original_frame);

  scoped_ptr<VideoPacket> packet = encoder_->Encode(*original_frame.get());

  DCHECK(!run_loop_ || !run_loop_->running());
  DCHECK(!timer_->IsRunning());
  run_loop_.reset(new base::RunLoop());

  // Set an extremely long time: 10 min to prevent bugs from hanging the system.
  // NOTE: We've seen cases which take up to 1 min to process a packet, so an
  // extremely long time as 10 min is chosen to avoid being variable/flaky.
  timer_->Start(FROM_HERE, base::TimeDelta::FromMinutes(10),
                run_loop_->QuitClosure());

  // Wait for the video packet to be processed and rendered to buffer.
  test_video_renderer_->ProcessVideoPacket(std::move(packet),
                                           run_loop_->QuitClosure());

  run_loop_->Run();
  EXPECT_TRUE(timer_->IsRunning());
  timer_->Stop();
  run_loop_.reset();

  scoped_ptr<webrtc::DesktopFrame> buffer_copy =
      test_video_renderer_->GetCurrentFrameForTest();
  EXPECT_NE(buffer_copy, nullptr);

  // The original frame is compared to the decoded video frame to check that
  // the mean error over all pixels does not exceed a given limit.
  double error = CalculateError(original_frame.get(), buffer_copy.get());
  EXPECT_LT(error, error_limit);
}

bool TestVideoRendererTest::SendPacketAndWaitForMatch(
    scoped_ptr<VideoPacket> packet,
    const webrtc::DesktopRect& expected_rect,
    const RGBValue& expected_average_color) {
  DCHECK(!run_loop_ || !run_loop_->running());
  DCHECK(!timer_->IsRunning());
  run_loop_.reset(new base::RunLoop());

  // Set an extremely long time: 10 min to prevent bugs from hanging the system.
  // NOTE: We've seen cases which take up to 1 min to process a packet, so an
  // extremely long time as 10 min is chosen to avoid being variable/flaky.
  timer_->Start(FROM_HERE, base::TimeDelta::FromMinutes(10),
                run_loop_->QuitClosure());

  // Set expected image pattern.
  test_video_renderer_->ExpectAverageColorInRect(
      expected_rect, expected_average_color, run_loop_->QuitClosure());

  // Used to verify if the expected image pattern will be matched by |packet|.
  scoped_ptr<VideoPacket> packet_copy(new VideoPacket(*packet.get()));

  // Post first test packet: |packet|.
  test_video_renderer_->ProcessVideoPacket(std::move(packet),
                                           base::Bind(&base::DoNothing));

  // Second packet: |packet_copy| is posted, and |second_packet_done_callback|
  // will always be posted back to main thread, however, whether it will be
  // called depends on whether the expected pattern is matched or not.
  bool second_packet_done_is_called = false;
  base::Closure second_packet_done_callback =
      base::Bind(&ProcessPacketDoneHandler, run_loop_->QuitClosure(),
                 &second_packet_done_is_called);

  test_video_renderer_->ProcessVideoPacket(std::move(packet_copy),
                                           second_packet_done_callback);

  run_loop_->Run();
  EXPECT_TRUE(timer_->IsRunning());
  timer_->Stop();
  run_loop_.reset();

  // if expected image pattern is matched, the QuitClosure of |run_loop_| will
  // be called before |second_packet_done_callback|, which leaves
  // |second_packet_done_is_called| be false.
  bool image_pattern_is_matched = !second_packet_done_is_called;

  return image_pattern_is_matched;
}

void TestVideoRendererTest::TestImagePatternMatch(
    int screen_width,
    int screen_height,
    const webrtc::DesktopRect& expected_rect,
    bool expect_to_match) {
  DCHECK(encoder_);
  DCHECK(test_video_renderer_);

  scoped_ptr<webrtc::DesktopFrame> frame =
      CreateDesktopFrameWithGradient(screen_width, screen_height);
  RGBValue expected_average_color =
      CalculateAverageColorValueForFrame(frame.get(), expected_rect);
  scoped_ptr<VideoPacket> packet = encoder_->Encode(*frame.get());

  if (expect_to_match) {
    EXPECT_TRUE(SendPacketAndWaitForMatch(std::move(packet), expected_rect,
                                          expected_average_color));
  } else {
    // Shift each channel by 128.
    // e.g. (10, 127, 200) -> (138, 255, 73).
    // In this way, the error between expected color and true value is always
    // around 0.5.
    int red_shift = (expected_average_color.red + 128) % 255;
    int green_shift = (expected_average_color.green + 128) % 255;
    int blue_shift = (expected_average_color.blue + 128) % 255;

    RGBValue expected_average_color_shift =
        RGBValue(red_shift, green_shift, blue_shift);

    EXPECT_FALSE(SendPacketAndWaitForMatch(std::move(packet), expected_rect,
                                           expected_average_color_shift));
  }
}

RGBValue TestVideoRendererTest::CalculateAverageColorValueForFrame(
    const webrtc::DesktopFrame* frame,
    const webrtc::DesktopRect& rect) const {
  int red_sum = 0;
  int green_sum = 0;
  int blue_sum = 0;

  // Loop through pixels that fall within |accumulating_rect_| to obtain the
  // average color value.
  for (int y = rect.top(); y < rect.bottom(); ++y) {
    uint8_t* frame_pos =
        frame->data() + (y * frame->stride() +
                         rect.left() * webrtc::DesktopFrame::kBytesPerPixel);

    // Pixels of decoded video frame are presented in ARGB format.
    for (int x = 0; x < rect.width(); ++x) {
      red_sum += frame_pos[2];
      green_sum += frame_pos[1];
      blue_sum += frame_pos[0];
      frame_pos += 4;
    }
  }

  int area = rect.width() * rect.height();

  return RGBValue(red_sum / area, green_sum / area, blue_sum / area);
}

double TestVideoRendererTest::CalculateError(
    const webrtc::DesktopFrame* original_frame,
    const webrtc::DesktopFrame* decoded_frame) const {
  DCHECK(original_frame);
  DCHECK(decoded_frame);

  // Check size remains the same after encoding and decoding.
  EXPECT_EQ(original_frame->size().width(), decoded_frame->size().width());
  EXPECT_EQ(original_frame->size().height(), decoded_frame->size().height());
  EXPECT_EQ(original_frame->stride(), decoded_frame->stride());
  int screen_width = original_frame->size().width();
  int screen_height = original_frame->size().height();

  // Error is calculated as the sum of the square error at each pixel in the
  // R, G and B components, each normalized to [0, 1].
  double error_sum_squares = 0.0;

  // The mapping between the position of a pixel on 3-dimensional image
  // (origin at top left corner) and its position in 1-dimensional buffer.
  //
  //  _______________
  // |      |        |      stride = 4 * width;
  // |      |        |
  // |      | height |      height * stride + width + 0; Red channel.
  // |      |        |  =>  height * stride + width + 1; Green channel.
  // |-------        |      height * stride + width + 2; Blue channel.
  // | width         |
  // |_______________|
  //
  for (int height = 0; height < screen_height; ++height) {
    uint8_t* original_ptr = original_frame->data() +
                            height * original_frame->stride();
    uint8_t* decoded_ptr = decoded_frame->data() +
                           height * decoded_frame->stride();

    for (int width = 0; width < screen_width; ++width) {
      // Errors are calculated in the R, G, B components.
      for (int j = 0; j < 3; ++j) {
        int offset = webrtc::DesktopFrame::kBytesPerPixel * width + j;
        double original_value = static_cast<double>(*(original_ptr + offset));
        double decoded_value = static_cast<double>(*(decoded_ptr + offset));
        double error = original_value - decoded_value;

        // Normalize the error to [0, 1].
        error /= 255.0;
        error_sum_squares += error * error;
      }
    }
  }
  return sqrt(error_sum_squares / (3 * screen_width * screen_height));
}

scoped_ptr<webrtc::DesktopFrame>
    TestVideoRendererTest::CreateDesktopFrameWithGradient(
        int screen_width, int screen_height) const {
  webrtc::DesktopSize screen_size(screen_width, screen_height);
  scoped_ptr<webrtc::DesktopFrame> frame(
      new webrtc::BasicDesktopFrame(screen_size));
  frame->mutable_updated_region()->SetRect(
      webrtc::DesktopRect::MakeSize(screen_size));
  FillFrameWithGradient(frame.get());
  return frame;
}

void TestVideoRendererTest::FillFrameWithGradient(
    webrtc::DesktopFrame* frame) const {
  for (int y = 0; y < frame->size().height(); ++y) {
    uint8_t* p = frame->data() + y * frame->stride();
    for (int x = 0; x < frame->size().width(); ++x) {
      *p++ = (255.0 * x) / frame->size().width();
      *p++ = (164.0 * y) / frame->size().height();
      *p++ = (82.0 * (x + y)) /
          (frame->size().width() + frame->size().height());
      *p++ = 0;
    }
  }
}

// Verify video decoding for VP8 Codec.
TEST_F(TestVideoRendererTest, VerifyVideoDecodingForVP8) {
  encoder_ = VideoEncoderVpx::CreateForVP8();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::CODEC_VP8);
  TestVideoPacketProcessing(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                            kDefaultErrorLimit);
}

// Verify video decoding for VP9 Codec.
TEST_F(TestVideoRendererTest, VerifyVideoDecodingForVP9) {
  encoder_ = VideoEncoderVpx::CreateForVP9();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::CODEC_VP9);
  TestVideoPacketProcessing(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                            kDefaultErrorLimit);
}


// Verify video decoding for VERBATIM Codec.
TEST_F(TestVideoRendererTest, VerifyVideoDecodingForVERBATIM) {
  encoder_.reset(new VideoEncoderVerbatim());
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::CODEC_VERBATIM);
  TestVideoPacketProcessing(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                            kDefaultErrorLimit);
}

// Verify a set of video packets are processed correctly.
TEST_F(TestVideoRendererTest, VerifyMultipleVideoProcessing) {
  encoder_ = VideoEncoderVpx::CreateForVP8();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::CODEC_VP8);

  // Post multiple tasks to |test_video_renderer_|, and it should not crash.
  // 20 is chosen because it's large enough to make sure that there will be
  // more than one task on the video decode thread, while not too large to wait
  // for too long for the unit test to complete.
  const int task_num = 20;
  ScopedVector<VideoPacket> video_packets;
  for (int i = 0; i < task_num; ++i) {
    scoped_ptr<webrtc::DesktopFrame> original_frame =
        CreateDesktopFrameWithGradient(kDefaultScreenWidthPx,
                                       kDefaultScreenHeightPx);
    video_packets.push_back(encoder_->Encode(*original_frame.get()));
  }

  for (int i = 0; i < task_num; ++i) {
    // Transfer ownership of video packet.
    VideoPacket* packet = video_packets[i];
    video_packets[i] = nullptr;
    test_video_renderer_->ProcessVideoPacket(make_scoped_ptr(packet),
                                             base::Bind(&base::DoNothing));
  }
}

// Verify video packet size change is handled properly.
TEST_F(TestVideoRendererTest, VerifyVideoPacketSizeChange) {
  encoder_ = VideoEncoderVpx::CreateForVP8();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VP8);

  TestVideoPacketProcessing(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                            kDefaultErrorLimit);

  TestVideoPacketProcessing(2 * kDefaultScreenWidthPx,
                            2 * kDefaultScreenHeightPx, kDefaultErrorLimit);

  TestVideoPacketProcessing(kDefaultScreenWidthPx / 2,
                            kDefaultScreenHeightPx / 2, kDefaultErrorLimit);
}

// Verify setting expected image pattern doesn't break video packet processing.
TEST_F(TestVideoRendererTest, VerifySetExpectedImagePattern) {
  encoder_ = VideoEncoderVpx::CreateForVP8();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VP8);

  DCHECK(encoder_);
  DCHECK(test_video_renderer_);

  scoped_ptr<webrtc::DesktopFrame> frame = CreateDesktopFrameWithGradient(
      kDefaultScreenWidthPx, kDefaultScreenHeightPx);

  // Since we don't care whether expected image pattern is matched or not in
  // this case, an expected color is chosen arbitrarily.
  RGBValue black_color = RGBValue();

  // Set expected image pattern.
  test_video_renderer_->ExpectAverageColorInRect(
      kDefaultExpectedRect, black_color, base::Bind(&base::DoNothing));

  // Post test video packet.
  test_video_renderer_->ProcessVideoPacket(encoder_->Encode(*frame.get()),
                                           base::Bind(&base::DoNothing));
}

// Verify correct image pattern can be matched for VP8.
TEST_F(TestVideoRendererTest, VerifyImagePatternMatchForVP8) {
  encoder_ = VideoEncoderVpx::CreateForVP8();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VP8);
  TestImagePatternMatch(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                        kDefaultExpectedRect, true);
}

// Verify expected image pattern can be matched for VP9.
TEST_F(TestVideoRendererTest, VerifyImagePatternMatchForVP9) {
  encoder_ = VideoEncoderVpx::CreateForVP9();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VP9);
  TestImagePatternMatch(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                        kDefaultExpectedRect, true);
}

// Verify expected image pattern can be matched for VERBATIM.
TEST_F(TestVideoRendererTest, VerifyImagePatternMatchForVERBATIM) {
  encoder_.reset(new VideoEncoderVerbatim());
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VERBATIM);
  TestImagePatternMatch(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                        kDefaultExpectedRect, true);
}

// Verify incorrect image pattern shouldn't be matched for VP8.
TEST_F(TestVideoRendererTest, VerifyImagePatternNotMatchForVP8) {
  encoder_ = VideoEncoderVpx::CreateForVP8();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VP8);
  TestImagePatternMatch(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                        kDefaultExpectedRect, false);
}

// Verify incorrect image pattern shouldn't be matched for VP9.
TEST_F(TestVideoRendererTest, VerifyImagePatternNotMatchForVP9) {
  encoder_ = VideoEncoderVpx::CreateForVP9();
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VP9);
  TestImagePatternMatch(kDefaultScreenWidthPx, kDefaultScreenWidthPx,
                        kDefaultExpectedRect, false);
}

// Verify incorrect image pattern shouldn't be matched for VERBATIM.
TEST_F(TestVideoRendererTest, VerifyImagePatternNotMatchForVERBATIM) {
  encoder_.reset(new VideoEncoderVerbatim());
  test_video_renderer_->SetCodecForDecoding(
      protocol::ChannelConfig::Codec::CODEC_VERBATIM);
  TestImagePatternMatch(kDefaultScreenWidthPx, kDefaultScreenHeightPx,
                        kDefaultExpectedRect, false);
}

}  // namespace test
}  // namespace remoting