Continues https://codereview.chromium.org/405093003, so general description is there.

Now the testing mode has appeared: ScreenshotTester can now load the golden 
screenshot from the disk and compare it to the screenshot taken right now.
It still should be turned on with switches. To turn testing with screenshots on, 
--enable-screenshot-testing-with-mode=MODE is used, where MODE can be "test" or "update". 
"update" updates golden screenshot and requires --golden-screenshots-dir=DIR: place to store golden screenshots. "test" runs comparing golden 
screenshot with current one, and --artifacts-dir=DIR can be used, where DIR is 
the place to store screenshots that are different from golden ones if they exist,
 and also an image representing difference between screenshots. If no 
--artifacts-dir switch is found, these bugproofs are saved to the directory with 
golden screenshots.

BUG=395653

Review URL: https://codereview.chromium.org/433873002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@286880 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 202 insertions, 52 deletions

diff --git a/chrome/browser/chromeos/login/login_ui_browsertest.cc b/chrome/browser/chromeos/login/login_ui_browsertest.cc
index 29bf936..12c91c4 100644
--- a/chrome/browser/chromeos/login/login_ui_browsertest.cc
+++ b/chrome/browser/chromeos/login/login_ui_browsertest.cc
@@ -148,8 +148,6 @@ class LoginUITest : public chromeos::LoginManagerTest {
     enable_test_screenshots_ = screenshot_tester.TryInitialize();
     if (enable_test_screenshots_) {
       animation_delay_handler.Initialize();
-    } else {
-      LOG(WARNING) << "Screenshots will not be taken";
     }
     LoginManagerTest::SetUpOnMainThread();
   }
diff --git a/chrome/browser/chromeos/login/screenshot_tester.cc b/chrome/browser/chromeos/login/screenshot_tester.cc
index 683f987..44602f5 100644
--- a/chrome/browser/chromeos/login/screenshot_tester.cc
+++ b/chrome/browser/chromeos/login/screenshot_tester.cc
@@ -8,6 +8,7 @@
 #include "base/base_export.h"
 #include "base/bind_internal.h"
 #include "base/command_line.h"
+#include "base/logging.h"
 #include "base/memory/weak_ptr.h"
 #include "base/prefs/pref_service.h"
 #include "base/run_loop.h"
@@ -15,13 +16,27 @@
 #include "chrome/common/pref_names.h"
 #include "chromeos/chromeos_switches.h"
 #include "content/public/browser/browser_thread.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "third_party/skia/include/core/SkCanvas.h"
 #include "ui/compositor/compositor_switches.h"
+#include "ui/gfx/codec/png_codec.h"
 #include "ui/gfx/image/image.h"
 #include "ui/snapshot/snapshot.h"
 
+namespace {
+
+// Sets test mode for screenshot testing.
+const char kTestMode[] = "test";
+
+// Sets update mode for screenshot testing.
+const char kUpdateMode[] = "update;"
+
+}  // namespace
+
 namespace chromeos {
 
-ScreenshotTester::ScreenshotTester() : weak_factory_(this) {
+ScreenshotTester::ScreenshotTester() : test_mode_(false), weak_factory_(this) {
 }
 
 ScreenshotTester::~ScreenshotTester() {
@@ -29,7 +44,7 @@ ScreenshotTester::~ScreenshotTester() {
 
 bool ScreenshotTester::TryInitialize() {
   CommandLine& command_line = *CommandLine::ForCurrentProcess();
-  if (!command_line.HasSwitch(switches::kEnableScreenshotTesting))
+  if (!command_line.HasSwitch(switches::kEnableScreenshotTestingWithMode))
     return false;
   if (!command_line.HasSwitch(::switches::kEnablePixelOutputInTests) ||
       !command_line.HasSwitch(::switches::kUIEnableImplSidePainting)) {
@@ -40,64 +55,172 @@ bool ScreenshotTester::TryInitialize() {
                << "screenshots";
     return false;
   }
-  if (!command_line.HasSwitch(switches::kScreenshotDestinationDir)) {
-    LOG(ERROR) << "No destination for screenshots specified";
-    return false;
+
+  std::string mode = command_line.GetSwitchValueASCII(
+      switches::kEnableScreenshotTestingWithMode);
+  if (mode != kUpdateMode && mode != kTestMode) {
+    CHECK(false) << "Invalid mode for screenshot testing: " << mode;
+  }
+
+  if (!command_line.HasSwitch(chromeos::switches::kGoldenScreenshotsDir)) {
+    CHECK(false) << "No directory for golden screenshots specified";
+  }
+
+  golden_screenshots_dir_ =
+      command_line.GetSwitchValuePath(switches::kGoldenScreenshotsDir);
+
+  if (mode == kTestMode) {
+    test_mode_ = true;
+    if (!command_line.HasSwitch(switches::kArtifactsDir)) {
+      artifacts_dir_ = golden_screenshots_dir_;
+      LOG(WARNING)
+          << "No directory for artifact storing specified. Artifacts will be"
+          << "saved at golden screenshots directory.";
+    } else {
+      artifacts_dir_ = command_line.GetSwitchValuePath(switches::kArtifactsDir);
+    }
   }
-  screenshot_dest_ = command_line.GetSwitchValuePath(
-      chromeos::switches::kScreenshotDestinationDir);
   return true;
 }
 
-void ScreenshotTester::Run(const std::string& file_name) {
-  TakeScreenshot();
-  PNGFile current_screenshot = screenshot_;
-  UpdateGoldenScreenshot(file_name, current_screenshot);
+void ScreenshotTester::Run(const std::string& test_name) {
+  test_name_ = test_name;
+  PNGFile current_screenshot = TakeScreenshot();
+  if (test_mode_) {
+    PNGFile golden_screenshot = LoadGoldenScreenshot();
+    VLOG(0) << "Loaded golden screenshot";
+    CompareScreenshots(golden_screenshot, current_screenshot);
+  } else {
+    UpdateGoldenScreenshot(current_screenshot);
+  }
 }
 
-void ScreenshotTester::UpdateGoldenScreenshot(const std::string& file_name,
-                                              PNGFile png_data) {
+void ScreenshotTester::UpdateGoldenScreenshot(PNGFile png_data) {
+  CHECK(SaveImage("golden_screenshot", golden_screenshots_dir_, png_data));
+}
+
+bool ScreenshotTester::SaveImage(const std::string& file_name,
+                                 const base::FilePath& screenshot_dir,
+                                 PNGFile png_data) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+  base::FilePath screenshot_path =
+      screenshot_dir.AppendASCII(test_name_ + "_" + file_name + ".png");
   if (!png_data) {
     LOG(ERROR) << "Can't take a screenshot";
-    return;
+    return false;
   }
-  base::FilePath golden_screenshot_path =
-      screenshot_dest_.AppendASCII(file_name + ".png");
-  if (!base::CreateDirectory(golden_screenshot_path.DirName())) {
-    LOG(ERROR) << "Can't create a directory ";
-    return;
+  if (!base::CreateDirectory(screenshot_path.DirName())) {
+    LOG(ERROR) << "Can't create a directory "
+               << screenshot_path.DirName().value();
+    return false;
   }
   if (static_cast<size_t>(
-          base::WriteFile(golden_screenshot_path,
+          base::WriteFile(screenshot_path,
                           reinterpret_cast<char*>(&(png_data->data()[0])),
                           png_data->size())) != png_data->size()) {
-    LOG(ERROR) << "Can't save screenshot";
+    LOG(ERROR) << "Can't save screenshot " << file_name;
+    return false;
   }
-  VLOG(0) << "Golden screenshot updated successfully";
+  VLOG(0) << "Screenshot " << file_name << ".png saved successfully to "
+          << screenshot_dir.value();
+  return true;
 }
 
-void ScreenshotTester::ReturnScreenshot(PNGFile png_data) {
-  // TODO(elizavetai): rewrite this function so that TakeScreenshot
-  // could return a |PNGFile| -- current screenshot.
+void ScreenshotTester::ReturnScreenshot(const PNGFile& screenshot,
+                                        PNGFile png_data) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
-  screenshot_ = png_data;
+  screenshot->data() = png_data->data();
   content::BrowserThread::PostTask(
       content::BrowserThread::UI, FROM_HERE, run_loop_quitter_);
 }
 
-void ScreenshotTester::TakeScreenshot() {
-  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+ScreenshotTester::PNGFile ScreenshotTester::TakeScreenshot() {
+  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::UI));
   aura::Window* primary_window = ash::Shell::GetPrimaryRootWindow();
   gfx::Rect rect = primary_window->bounds();
+  PNGFile screenshot = new base::RefCountedBytes;
   ui::GrabWindowSnapshotAsync(primary_window,
                               rect,
                               content::BrowserThread::GetBlockingPool(),
                               base::Bind(&ScreenshotTester::ReturnScreenshot,
-                                         weak_factory_.GetWeakPtr()));
+                                         weak_factory_.GetWeakPtr(),
+                                         screenshot));
   base::RunLoop run_loop;
   run_loop_quitter_ = run_loop.QuitClosure();
   run_loop.Run();
+  return screenshot;
+}
+
+ScreenshotTester::PNGFile ScreenshotTester::LoadGoldenScreenshot() {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  base::FilePath screenshot_path = golden_screenshots_dir_.AppendASCII(
+      test_name_ + "_golden_screenshot.png");
+  if (!base::PathExists(screenshot_path)) {
+    CHECK(false) << "Can't find a golden screenshot for this test";
+  }
+
+  size_t golden_screenshot_size;
+  base::GetFileSize(screenshot_path,
+                    reinterpret_cast<int64*>(&golden_screenshot_size));
+
+  if (golden_screenshot_size == -1) {
+    CHECK(false) << "Can't get golden screenshot size";
+  }
+  PNGFile png_data = new base::RefCountedBytes;
+  png_data->data().resize(golden_screenshot_size);
+  base::ReadFile(screenshot_path,
+                 reinterpret_cast<char*>(&(png_data->data()[0])),
+                 golden_screenshot_size);
+
+  return png_data;
+}
+
+void ScreenshotTester::CompareScreenshots(PNGFile model, PNGFile sample) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  ASSERT_TRUE(model.get());
+  ASSERT_TRUE(sample.get());
+
+  SkBitmap model_bitmap;
+  SkBitmap sample_bitmap;
+  gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&(model->data()[0])),
+                        model->data().size(),
+                        &model_bitmap);
+  gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&(sample->data()[0])),
+                        sample->data().size(),
+                        &sample_bitmap);
+
+  ASSERT_EQ(model_bitmap.config(), sample_bitmap.config());
+  ASSERT_EQ(model_bitmap.width(), sample_bitmap.width());
+  ASSERT_EQ(model_bitmap.height(), sample_bitmap.height());
+
+  bool screenshots_match = true;
+
+  SkCanvas diff_canvas(sample_bitmap);
+  for (int i = 0; i < model_bitmap.width(); i++)
+    for (int j = 0; j < model_bitmap.height(); j++) {
+      if (model_bitmap.getColor(i, j) == sample_bitmap.getColor(i, j)) {
+        diff_canvas.drawPoint(i, j, SK_ColorWHITE);
+      } else {
+        screenshots_match = false;
+        diff_canvas.drawPoint(i, j, SK_ColorRED);
+      }
+    }
+
+  if (!screenshots_match) {
+    CHECK(SaveImage("failed_screenshot", artifacts_dir_, sample));
+    gfx::PNGCodec::EncodeBGRASkBitmap(sample_bitmap, false, &sample->data());
+    CHECK(SaveImage("difference", artifacts_dir_, sample));
+    LOG(ERROR)
+        << "Screenshots testing failed. Screenshots differ in some pixels";
+    VLOG(0) << "Current screenshot and diff picture saved to "
+            << artifacts_dir_.value();
+  } else {
+    VLOG(0) << "Current screenshot matches the golden screenshot. Screenshot "
+               "testing passed.";
+  }
+  ASSERT_TRUE(screenshots_match);
 }
 
 }  // namespace chromeos
diff --git a/chrome/browser/chromeos/login/screenshot_tester.h b/chrome/browser/chromeos/login/screenshot_tester.h
index 5a1d052..d9319e8 100644
--- a/chrome/browser/chromeos/login/screenshot_tester.h
+++ b/chrome/browser/chromeos/login/screenshot_tester.h
@@ -5,6 +5,8 @@
 #ifndef CHROME_BROWSER_CHROMEOS_LOGIN_SCREENSHOT_TESTER_H_
 #define CHROME_BROWSER_CHROMEOS_LOGIN_SCREENSHOT_TESTER_H_
 
+#include <string>
+
 #include "base/base_export.h"
 #include "base/bind_internal.h"
 #include "base/files/file_path.h"
@@ -29,34 +31,56 @@ class ScreenshotTester {
 
   // Does all the work that has been stated through switches:
   // updates golden screenshot or takes a new screenshot and compares it
-  // with the golden one (this part is not implemented yet).
-  void Run(const std::string& file_name);
+  // with the golden one. |test_name| is the name of the test from which
+  // we run this method.
+  void Run(const std::string& test_name);
 
  private:
   typedef scoped_refptr<base::RefCountedBytes> PNGFile;
 
-  // Takes a screenshot and puts it to |screenshot_| field.
-  void TakeScreenshot();
+  // Takes a screenshot and returns it.
+  PNGFile TakeScreenshot();
+
+  // Saves |png_data| as a new golden screenshot for test |test_name_|.
+  void UpdateGoldenScreenshot(PNGFile png_data);
 
-  // Saves |png_data| as a new golden screenshot for this test.
-  void UpdateGoldenScreenshot(const std::string& file_name, PNGFile png_data);
+  // Saves an image |png_data|, assuming it is a .png file.
+  // Returns true if image was saved successfully.
+  bool SaveImage(const std::string& file_name,
+                 const base::FilePath& screenshot_dir,
+                 PNGFile png_data);
 
   // Saves |png_data| as a current screenshot.
-  void ReturnScreenshot(PNGFile png_data);
+  void ReturnScreenshot(const PNGFile& screenshot, PNGFile png_data);
+
+  // Loads golden screenshot from the disk. Fails if there is no
+  // golden screenshot for test |test_name_|.
+  PNGFile LoadGoldenScreenshot();
+
+  // Compares two given screenshots and saves |sample|
+  // and difference between |sample| and |model|, if they differ in any pixel.
+  void CompareScreenshots(PNGFile model, PNGFile sample);
+
+  // Name of the test from which Run() method has been called.
+  // Used for generating names for screenshot files.
+  std::string test_name_;
 
   // Path to the directory for golden screenshots.
-  base::FilePath screenshot_dest_;
+  base::FilePath golden_screenshots_dir_;
+
+  // Path to the directory where screenshots that failed comparing
+  // and difference between them and golden ones will be stored.
+  base::FilePath artifacts_dir_;
 
   // |run_loop_| and |run_loop_quitter_| are used to synchronize
   // with ui::GrabWindowSnapshotAsync.
   base::RunLoop run_loop_;
   base::Closure run_loop_quitter_;
 
-  // Current screenshot.
-  PNGFile screenshot_;
+  // Is true when we're in test mode:
+  // comparing golden screenshots and current ones.
+  bool test_mode_;
 
-  // Is true when we running updating golden screenshots mode.
-  bool update_golden_screenshot_;
   base::WeakPtrFactory<ScreenshotTester> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(ScreenshotTester);
diff --git a/chromeos/chromeos_switches.cc b/chromeos/chromeos_switches.cc
index d027ef0..471c250 100644
--- a/chromeos/chromeos_switches.cc
+++ b/chromeos/chromeos_switches.cc
@@ -96,13 +96,14 @@ const char kEnableNetworkPortalNotification[] =
 // Enables activation of voice search by saying 'Ok Google'.
 const char kEnableOkGoogleVoiceSearch[] = "enable-ok-google-voice-search";
 
-// Enables using screenshots in tests.
-const char kEnableScreenshotTesting[] = "enable-screenshot-testing";
-
 // Enables touchpad three-finger-click as middle button.
 const char kEnableTouchpadThreeFingerClick[]
     = "enable-touchpad-three-finger-click";
 
+// Enables using screenshots in tests and seets mode.
+const char kEnableScreenshotTestingWithMode[] =
+    "enable-screenshot-testing-with-mode";
+
 // Enable Kiosk mode for ChromeOS. Note this switch refers to retail mode rather
 // than the kiosk app mode.
 const char kEnableKioskMode[] = "enable-kiosk-mode";
@@ -240,12 +241,15 @@ const char kEnableFirstRunUITransitions[] = "enable-first-run-ui-transitions";
 // Forces first-run UI to be shown for every login.
 const char kForceFirstRunUI[] = "force-first-run-ui";
 
-// Turns on screenshot testing and specifies the directory where the
-// golden screenshots are stored.
-const char kScreenshotDestinationDir[] = "screenshot-destination-dir";
-
 // Enables testing for auto update UI.
 const char kTestAutoUpdateUI[] = "test-auto-update-ui";
 
+// Screenshot testing: specifies the directory where the golden screenshots are
+// stored.
+const char kGoldenScreenshotsDir[] = "golden-screenshots-dir";
+
+// Screenshot testing: specifies the directoru where artifacts will be stored.
+const char kArtifactsDir[] = "artifacts-dir";
+
 }  // namespace switches
 }  // namespace chromeos
diff --git a/chromeos/chromeos_switches.h b/chromeos/chromeos_switches.h
index b7bc3b3..1c3a9ed 100644
--- a/chromeos/chromeos_switches.h
+++ b/chromeos/chromeos_switches.h
@@ -80,8 +80,9 @@ CHROMEOS_EXPORT extern const char kShowHostPairingDemo[];
 CHROMEOS_EXPORT extern const char kSmsTestMessages[];
 CHROMEOS_EXPORT extern const char kStubCrosSettings[];
 CHROMEOS_EXPORT extern const char kTestAutoUpdateUI[];
-CHROMEOS_EXPORT extern const char kEnableScreenshotTesting[];
-CHROMEOS_EXPORT extern const char kScreenshotDestinationDir[];
+CHROMEOS_EXPORT extern const char kEnableScreenshotTestingWithMode[];
+CHROMEOS_EXPORT extern const char kGoldenScreenshotsDir[];
+CHROMEOS_EXPORT extern const char kArtifactsDir[];
 
 }  // namespace switches
 }  // namespace chromeos