path: root/chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc

// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/base64.h"
#include "base/command_line.h"
#include "base/environment.h"
#include "base/files/file.h"
#include "base/files/file_util.h"
#include "base/files/scoped_temp_dir.h"
#include "base/path_service.h"
#include "base/process/launch.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_split.h"
#include "base/strings/stringprintf.h"
#include "base/test/test_timeouts.h"
#include "base/time/time.h"
#include "chrome/browser/chrome_notification_types.h"
#include "chrome/browser/infobars/infobar_service.h"
#include "chrome/browser/media/media_stream_infobar_delegate.h"
#include "chrome/browser/media/webrtc_browsertest_base.h"
#include "chrome/browser/media/webrtc_browsertest_common.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/browser/ui/browser_tabstrip.h"
#include "chrome/browser/ui/tabs/tab_strip_model.h"
#include "chrome/common/chrome_switches.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "chrome/test/base/ui_test_utils.h"
#include "components/infobars/core/infobar.h"
#include "content/public/browser/notification_service.h"
#include "content/public/test/browser_test_utils.h"
#include "media/base/media_switches.h"
#include "net/test/embedded_test_server/embedded_test_server.h"
#include "net/test/python_utils.h"
#include "testing/perf/perf_test.h"
#include "ui/gl/gl_switches.h"

static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
#if defined(OS_WIN)
    FILE_PATH_LITERAL("frame_analyzer.exe");
#else
    FILE_PATH_LITERAL("frame_analyzer");
#endif

static const base::FilePath::CharType kArgbToI420ConverterExecutable[] =
#if defined(OS_WIN)
    FILE_PATH_LITERAL("rgba_to_i420_converter.exe");
#else
    FILE_PATH_LITERAL("rgba_to_i420_converter");
#endif

static const base::FilePath::CharType kCapturedYuvFileName[] =
    FILE_PATH_LITERAL("captured_video.yuv");
static const base::FilePath::CharType kStatsFileName[] =
    FILE_PATH_LITERAL("stats.txt");
static const char kMainWebrtcTestHtmlPage[] =
    "/webrtc/webrtc_jsep01_test.html";
static const char kCapturingWebrtcHtmlPage[] =
    "/webrtc/webrtc_video_quality_test.html";

static const struct VideoQualityTestConfig {
  const char* test_name;
  int width;
  int height;
  const base::FilePath::CharType* reference_video;
  const char* constraints;
} kVideoConfigurations[] = {
  { "360p", 640, 360,
    test::kReferenceFileName360p,
    WebRtcTestBase::kAudioVideoCallConstraints360p },
  { "720p", 1280, 720,
    test::kReferenceFileName720p,
    WebRtcTestBase::kAudioVideoCallConstraints720p },
};

// Test the video quality of the WebRTC output.
//
// Prerequisites: This test case must run on a machine with a chrome playing
// the video from the reference files located in GetReferenceFilesDir().
// The file kReferenceY4mFileName.kY4mFileExtension is played using a
// FileVideoCaptureDevice and its sibling with kYuvFileExtension is used for
// comparison.
//
// You must also compile the chromium_builder_webrtc target before you run this
// test to get all the tools built.
//
// The external compare_videos.py script also depends on two external
// executables which must be located in the PATH when running this test.
// * zxing (see the CPP version at https://code.google.com/p/zxing)
// * ffmpeg 0.11.1 or compatible version (see http://www.ffmpeg.org)
//
// The test runs several custom binaries - rgba_to_i420 converter and
// frame_analyzer. Both tools can be found under third_party/webrtc/tools. The
// test also runs a stand alone Python implementation of a WebSocket server
// (pywebsocket) and a barcode_decoder script.
class WebRtcVideoQualityBrowserTest : public WebRtcTestBase,
    public testing::WithParamInterface<VideoQualityTestConfig> {
 public:
  WebRtcVideoQualityBrowserTest()
      : environment_(base::Environment::Create()) {
    test_config_ = GetParam();
  }

  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.

    ASSERT_TRUE(temp_working_dir_.CreateUniqueTempDir());
  }

  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
    // Set up the command line option with the expected file name. We will check
    // its existence in HasAllRequiredResources().
    webrtc_reference_video_y4m_ = test::GetReferenceFilesDir()
        .Append(test_config_.reference_video)
        .AddExtension(test::kY4mFileExtension);
    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
                                   webrtc_reference_video_y4m_);
    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);

    // The video playback will not work without a GPU, so force its use here.
    command_line->AppendSwitch(switches::kUseGpuInTests);
  }

  // Writes all frames we've captured so far by grabbing them from the
  // javascript and writing them to the temporary work directory.
  void WriteCapturedFramesToWorkingDir(content::WebContents* capturing_tab) {
    int num_frames = 0;
    std::string response =
        ExecuteJavascript("getTotalNumberCapturedFrames()", capturing_tab);
    ASSERT_TRUE(base::StringToInt(response, &num_frames)) <<
        "Failed to retrieve frame count: got " << response;
    ASSERT_NE(0, num_frames) << "Failed to capture any frames.";

    for (int i = 0; i < num_frames; i++) {
      std::string base64_encoded_frame =
          ExecuteJavascript(base::StringPrintf("getOneCapturedFrame(%d)", i),
                            capturing_tab);
      std::string decoded_frame;
      ASSERT_TRUE(base::Base64Decode(base64_encoded_frame, &decoded_frame))
          << "Failed to decode frame data '" << base64_encoded_frame << "'.";

      std::string file_name = base::StringPrintf("frame_%04d", i);
      base::File frame_file(GetWorkingDir().AppendASCII(file_name),
                            base::File::FLAG_CREATE | base::File::FLAG_WRITE);
      size_t written = frame_file.Write(0, decoded_frame.c_str(),
                                        decoded_frame.length());
      ASSERT_EQ(decoded_frame.length(), written);
    }
  }

  // Runs the RGBA to I420 converter on the video in |capture_video_filename|,
  // which should contain frames of size |width| x |height|.
  //
  // The rgba_to_i420_converter is part of the webrtc_test_tools target which
  // should be build prior to running this test. The resulting binary should
  // live next to Chrome.
  bool RunARGBtoI420Converter(int width,
                              int height,
                              const base::FilePath& captured_video_filename) {
    base::FilePath path_to_converter = base::MakeAbsoluteFilePath(
        GetBrowserDir().Append(kArgbToI420ConverterExecutable));

    if (!base::PathExists(path_to_converter)) {
      LOG(ERROR) << "Missing ARGB->I420 converter: should be in "
          << path_to_converter.value();
      return false;
    }

    CommandLine converter_command(path_to_converter);
    converter_command.AppendSwitchPath("--frames_dir", GetWorkingDir());
    converter_command.AppendSwitchPath("--output_file",
                                       captured_video_filename);
    converter_command.AppendSwitchASCII("--width",
                                        base::StringPrintf("%d", width));
    converter_command.AppendSwitchASCII("--height",
                                        base::StringPrintf("%d", height));
    converter_command.AppendSwitchASCII("--delete_frames", "true");

    // We produce an output file that will later be used as an input to the
    // barcode decoder and frame analyzer tools.
    VLOG(0) << "Running " << converter_command.GetCommandLineString();
    std::string result;
    bool ok = base::GetAppOutput(converter_command, &result);
    VLOG(0) << "Output was:\n\n" << result;
    return ok;
  }

  // Compares the |captured_video_filename| with the |reference_video_filename|.
  //
  // The barcode decoder decodes the captured video containing barcodes overlaid
  // into every frame of the video (produced by rgba_to_i420_converter). It
  // produces a set of PNG images and a |stats_file| that maps each captured
  // frame to a frame in the reference video. The frames should be of size
  // |width| x |height|.
  // All measurements calculated are printed as perf parsable numbers to stdout.
  bool CompareVideosAndPrintResult(
      const char* test_label,
      int width,
      int height,
      const base::FilePath& captured_video_filename,
      const base::FilePath& reference_video_filename,
      const base::FilePath& stats_file) {

    base::FilePath path_to_analyzer = base::MakeAbsoluteFilePath(
        GetBrowserDir().Append(kFrameAnalyzerExecutable));
    base::FilePath path_to_compare_script = GetSourceDir().Append(
        FILE_PATH_LITERAL("third_party/webrtc/tools/compare_videos.py"));

    if (!base::PathExists(path_to_analyzer)) {
      LOG(ERROR) << "Missing frame analyzer: should be in "
          << path_to_analyzer.value();
      return false;
    }
    if (!base::PathExists(path_to_compare_script)) {
      LOG(ERROR) << "Missing video compare script: should be in "
          << path_to_compare_script.value();
      return false;
    }

    // Note: don't append switches to this command since it will mess up the
    // -u in the python invocation!
    CommandLine compare_command(CommandLine::NO_PROGRAM);
    EXPECT_TRUE(GetPythonCommand(&compare_command));

    compare_command.AppendArgPath(path_to_compare_script);
    compare_command.AppendArg(base::StringPrintf("--label=%s", test_label));
    compare_command.AppendArg("--ref_video");
    compare_command.AppendArgPath(reference_video_filename);
    compare_command.AppendArg("--test_video");
    compare_command.AppendArgPath(captured_video_filename);
    compare_command.AppendArg("--frame_analyzer");
    compare_command.AppendArgPath(path_to_analyzer);
    compare_command.AppendArg("--yuv_frame_width");
    compare_command.AppendArg(base::StringPrintf("%d", width));
    compare_command.AppendArg("--yuv_frame_height");
    compare_command.AppendArg(base::StringPrintf("%d", height));
    compare_command.AppendArg("--stats_file");
    compare_command.AppendArgPath(stats_file);

    VLOG(0) << "Running " << compare_command.GetCommandLineString();
    std::string output;
    bool ok = base::GetAppOutput(compare_command, &output);
    // Print to stdout to ensure the perf numbers are parsed properly by the
    // buildbot step.
    printf("Output was:\n\n%s\n", output.c_str());
    return ok;
  }

 protected:
  VideoQualityTestConfig test_config_;

  base::FilePath GetWorkingDir() {
    return temp_working_dir_.path();
  }

 private:
  base::FilePath GetSourceDir() {
    base::FilePath source_dir;
    PathService::Get(base::DIR_SOURCE_ROOT, &source_dir);
    return source_dir;
  }

  base::FilePath GetBrowserDir() {
    base::FilePath browser_dir;
    EXPECT_TRUE(PathService::Get(base::DIR_MODULE, &browser_dir));
    return browser_dir;
  }

  scoped_ptr<base::Environment> environment_;
  base::FilePath webrtc_reference_video_y4m_;
  base::ScopedTempDir temp_working_dir_;
};

INSTANTIATE_TEST_CASE_P(
    WebRtcVideoQualityBrowserTests,
    WebRtcVideoQualityBrowserTest,
    testing::ValuesIn(kVideoConfigurations));

IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
                       MANUAL_TestVideoQuality) {
  if (OnWinXp())
    return;  // Fails on XP. http://crbug.com/353078.

  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
      "This is a long-running test; you must specify "
      "--ui-test-action-max-timeout to have a value of at least 150000.";
  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());

  content::WebContents* left_tab =
      OpenPageAndGetUserMediaInNewTabWithConstraints(
          embedded_test_server()->GetURL(kMainWebrtcTestHtmlPage),
          test_config_.constraints);
  content::WebContents* right_tab =
      OpenPageAndGetUserMediaInNewTabWithConstraints(
          embedded_test_server()->GetURL(kCapturingWebrtcHtmlPage),
          test_config_.constraints);

  SetupPeerconnectionWithLocalStream(left_tab);
  SetupPeerconnectionWithLocalStream(right_tab);

  NegotiateCall(left_tab, right_tab);

  // Poll slower here to avoid flooding the log with messages: capturing and
  // sending frames take quite a bit of time.
  int polling_interval_msec = 1000;

  EXPECT_TRUE(test::PollingWaitUntil(
      "doneFrameCapturing()", "done-capturing", right_tab,
      polling_interval_msec));

  HangUp(left_tab);

  WriteCapturedFramesToWorkingDir(right_tab);

  // Shut everything down to avoid having the javascript race with the analysis
  // tools. For instance, dont have console log printouts interleave with the
  // RESULT lines from the analysis tools (crbug.com/323200).
  chrome::CloseWebContents(browser(), left_tab, false);
  chrome::CloseWebContents(browser(), right_tab, false);

  ASSERT_TRUE(RunARGBtoI420Converter(
      test_config_.width, test_config_.height,
      GetWorkingDir().Append(kCapturedYuvFileName)));
  ASSERT_TRUE(CompareVideosAndPrintResult(
      test_config_.test_name,
      test_config_.width,
      test_config_.height,
      GetWorkingDir().Append(kCapturedYuvFileName),
      test::GetReferenceFilesDir()
          .Append(test_config_.reference_video)
          .AddExtension(test::kYuvFileExtension),
      GetWorkingDir().Append(kStatsFileName)));
}