path: root/o3d/tests/selenium/main.py

#!/usr/bin/python2.4
# Copyright 2009, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


"""Selenium tests for the O3D plugin.

Sets up a local Selenium Remote Control server and a static file
server that serves files off the o3d directory.

Launches browsers to test the local build of the o3d plugin
and reports results back to the user.
"""

import os
import sys

script_dir = os.path.normpath(os.path.dirname(os.path.abspath(__file__)))
o3d_dir = os.path.dirname(os.path.dirname(script_dir))
src_dir = os.path.dirname(o3d_dir)
third_party_dir = os.path.join(src_dir, 'third_party')
gflags_dir = os.path.join(third_party_dir, 'gflags', 'python')
selenium_dir = os.path.join(third_party_dir, 'selenium_rc', 'files')
selenium_py_dir = os.path.join(selenium_dir, 'selenium-python-client-driver')
sys.path.append(gflags_dir)
sys.path.append(selenium_py_dir)

import re
import SimpleHTTPServer
import socket
import SocketServer
import subprocess
import threading
import time
import unittest
import gflags
import javascript_unit_tests
# Import custom testrunner for pulse
import pulse_testrunner
import selenium
import samples_tests
import selenium_constants
import selenium_utilities

if sys.platform == 'win32' or sys.platform == 'cygwin':
  default_java_exe = "java.exe"
else:
  default_java_exe = "java"

# Command line flags
FLAGS = gflags.FLAGS
gflags.DEFINE_boolean("verbose", False, "verbosity")
gflags.DEFINE_boolean("screenshots", False, "takes screenshots")
gflags.DEFINE_string(
    "java",
    default_java_exe,
    "specifies the path to the java executable.")
gflags.DEFINE_string(
    "selenium_server",
    os.path.join(selenium_dir, 'selenium-server', 'selenium-server.jar'),
    "specifies the path to the selenium server jar.")
gflags.DEFINE_string(
    "product_dir",
    None,
    "specifies the path to the build output directory.")
gflags.DEFINE_string(
    "screencompare",
    "",
    "specifies the directory in which perceptualdiff resides.\n"
    "compares screenshots with reference images")
gflags.DEFINE_string(
    "screenshotsdir",
    selenium_constants.DEFAULT_SCREENSHOT_PATH,
    "specifies the directory in which screenshots will be stored.")
gflags.DEFINE_string(
    "referencedir",
    selenium_constants.DEFAULT_SCREENSHOT_PATH,
    "Specifies the directory where reference images will be read from.")
gflags.DEFINE_string(
    "testprefix", "Test",
    "specifies the prefix of tests to run")
gflags.DEFINE_string(
    "testsuffixes",
    "small,medium,large",
    "specifies the suffixes, separated by commas of tests to run")
gflags.DEFINE_string(
    "servertimeout",
    "30",
    "Specifies the timeout value, in seconds, for the selenium server.")

# Browsers to choose from (for browser flag).
# use --browser $BROWSER_NAME to run
# tests for that browser
gflags.DEFINE_list(
    "browser",
    "*firefox",
    "\n".join(["comma-separated list of browsers to test",
               "Options:"] +
              selenium_constants.SELENIUM_BROWSER_SET))
gflags.DEFINE_string(
    "browserpath",
    "",
    "specifies the path to the browser executable "
    "(for platforms that don't support MOZ_PLUGIN_PATH)")
gflags.DEFINE_string(
    "samplespath",
    "",
    "specifies the path from the web root to the samples.")


class MyRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  """Hook to handle HTTP server requests.

  Functions as a handler for logging and other utility functions.
  """

  def log_message(self, format, *args):
    """Logging hook for HTTP server."""

    # For now, just suppress logging.
    pass
    # TODO: might be nice to have a verbose option for debugging.


class LocalFileHTTPServer(threading.Thread):
  """Minimal HTTP server that serves local files.

  Members:
    http_alive: event to signal that http server is up and running
    http_port: the TCP port the server is using
  """

  START_PORT = 8100
  END_PORT = 8105

  def __init__(self, local_root=None):
    """Initializes the server.

    Initializes the HTTP server to serve static files from the
    local o3d directory

    Args:
      local_root: all files below this path are served.  If not specified,
        the current directory is the root.
    """
    threading.Thread.__init__(self)
    self._local_root = local_root
    self.http_alive = threading.Event()
    self.http_port = 0

  def run(self):
    """Runs the HTTP server.

    Server is started on an available port in the range of
    START_PORT to END_PORT
    """

    if self._local_root:
      os.chdir(self._local_root)

    for self.http_port in range(self.START_PORT, self.END_PORT):
      # Attempt to start the server
      try:
        httpd = SocketServer.TCPServer(("", self.http_port),
                                       MyRequestHandler)
      except socket.error:
        # Server didn't manage to start up, try another port.
        pass
      else:
        self.http_alive.set()
        httpd.serve_forever()

    if not self.http_alive.isSet():
      print("No available port found for HTTP server in the range %d to %d."
            % (self.START_PORT, self.END_PORT))
      self.http_port = 0

  @staticmethod
  def StartServer(local_root=None):
    """Create and start a LocalFileHTTPServer on a separate thread.

    Args:
      local_root: serve all static files below this directory.  If not
        specified, the current directory is the root.

    Returns:
      http_server: LocalFileHTTPServer() object
    """

    # Start up the Selenium Remote Control server
    http_server = LocalFileHTTPServer(local_root)
    http_server.setDaemon(True)
    http_server.start()

    time_out = 30.0

    # Wait till the Selenium RC Server is up
    print 'Waiting %d seconds for local HTTP server to start.' % (int(time_out))
    http_server.http_alive.wait(time_out)
    if not http_server.http_port:
      print 'Timed out.'
      return None

    print "LocalFileHTTPServer started on port %d" % http_server.http_port

    return http_server


class SeleniumRemoteControl(threading.Thread):
  """A thread that launches the Selenium Remote Control server.

  The Remote Control server allows us to launch a browser and remotely
  control it from a script.

  Members:
    selenium_alive: event to signal that selenium server is up and running
    selenium_port: the TCP port the server is using
    process: the subprocess.Popen instance for the server
  """

  START_PORT = 5430
  END_PORT = 5535

  def __init__(self, verbose, java_path, selenium_server, server_timeout):
    """Initializes the SeleniumRemoteControl class.

    Args:
      verbose: boolean verbose flag
      java_path: path to java used to run selenium.
      selenium_server: path to jar containing selenium server.
      server_timeout: server timeout value, in seconds.
    """
    self.selenium_alive = threading.Event()
    self.selenium_port = 0
    self.verbose = verbose
    self.java_path = java_path
    self.selenium_server = selenium_server
    self.timeout = server_timeout
    threading.Thread.__init__(self)

  def run(self):
    """Starts the selenium server.

    Server is started on an available port in the range of
    START_PORT to END_PORT
    """

    for self.selenium_port in range(self.START_PORT, self.END_PORT):
      # Attempt to start the selenium RC server from java
      self.process = subprocess.Popen(
          [self.java_path, "-jar", self.selenium_server, "-multiWindow",
           "-port", str(self.selenium_port), "-timeout", self.timeout],
          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

      for unused_i in range(1, 10):
        server_msg = self.process.stdout.readline()
        if self.verbose and server_msg is not None:
          # log if verbose flag is on
          print "sel_serv:" + server_msg

        # This status message indicates that the server has done
        # a bind to the port self.selenium_port successfully.
        if server_msg.find("INFO - Started SocketListener") != -1:
          self.selenium_alive.set()
          break

      # Error starting server on this port, try the next port.
      if not self.selenium_alive.isSet():
        continue

      # Loop and read from stdout
      while self.process.poll() is None:
        server_msg = self.process.stdout.readline()
        if self.verbose and server_msg is not None:
          # log if verbose flag is on
          print "sel_serv:" + server_msg

      # Finish.
      break

    if not self.selenium_alive.isSet():
      print("No available port found for Selenium RC Server "
            "in the range %d to %d."
            % (self.START_PORT, self.END_PORT))
      self.selenium_port = 0

  @staticmethod
  def StartServer(verbose, java_path, selenium_server, server_timeout):
    """Create and start the Selenium RC Server on a separate thread.

    Args:
      verbose: boolean verbose flag
      java_path: path to java used to run selenium.
      selenium_server: path to jar containing selenium server.
      server_timeout: server timeout value, in seconds

    Returns:
      selenium_server: SeleniumRemoteControl() object
    """

    # Start up the Selenium Remote Control server
    selenium_server = SeleniumRemoteControl(verbose,
                                            java_path,
                                            selenium_server,
                                            server_timeout)

    selenium_server.setDaemon(True)
    selenium_server.start()

    time_out = 30.0

    # Wait till the Selenium RC Server is up
    print 'Waiting %d seconds for Selenium RC to start.' % (int(time_out))
    selenium_server.selenium_alive.wait(time_out)
    if not selenium_server.selenium_port:
      print 'Timed out.'
      return None

    print("Selenium RC server started on port %d"
          % selenium_server.selenium_port)

    return selenium_server


class SeleniumSession(object):
  """A selenium browser session, with support servers.

    The support servers include a Selenium Remote Control server, and
    a local HTTP server to serve static test files.

  Members:
    session: a selenium() instance
    selenium_server: a SeleniumRemoteControl() instance
    http_server: a LocalFileHTTPServer() instance
    runner: a TestRunner() instance
  """

  def __init__(self, verbose, java_path, selenium_server, server_timeout,
               http_root=None):
    """Initializes a Selenium Session.

    Args:
      verbose: boolean verbose flag
      java_path: path to java used to run selenium.
      selenium_server: path to jar containing selenium server.
      server_timeout: server timeout value, in seconds.
      http_root: Serve http pages using this path as the document root.  When
      None, use the default.
    """
    # Start up a static file server, to serve the test pages.

    if not http_root:
        http_root = FLAGS.product_dir

    self.http_server = LocalFileHTTPServer.StartServer(http_root)

    if self.http_server:
      # Start up the Selenium Remote Control Server
      self.selenium_server = SeleniumRemoteControl.StartServer(verbose,
                                                               java_path,
                                                               selenium_server,
                                                               server_timeout)
    if not self.http_server or not self.selenium_server:
      return

    # Set up a testing runner
    self.runner = pulse_testrunner.PulseTestRunner()

    # Set up a phantom selenium session so we can call shutdown if needed.
    self.session = selenium.selenium(
        "localhost", self.selenium_server.selenium_port, "*firefox",
        "http://" + socket.gethostname() + ":" +
        str(self.http_server.http_port))

  def StartSession(self, browser):
    """Starts the Selenium Session and connects to the HTTP server.

    Args:
      browser: selenium browser name
    """

    if browser == "*googlechrome":
      # TODO: Replace socket.gethostname() with "localhost"
      #                 once Chrome local proxy fix is in.
      server_url = "http://" + socket.gethostname() + ":"
    else:
      server_url = "http://localhost:"
    server_url += str(self.http_server.http_port)

    browser_path_with_space = ""
    if FLAGS.browserpath:
      browser_path_with_space = " " + FLAGS.browserpath

    self.session = selenium.selenium("localhost",
                                     self.selenium_server.selenium_port,
                                     browser + browser_path_with_space,
                                     server_url)
    self.session.start()

  def CloseSession(self):
    """Closes the selenium sesssion."""
    self.session.stop()

  def TearDown(self):
    """Stops the selenium server."""
    self.session.shut_down_selenium_server()

  def TestBrowser(self, browser, test_list, test_prefix, test_suffixes,
                  server_timeout):
    """Runs Selenium tests for a specific browser.

    Args:
      browser: selenium browser name (eg. *iexplore, *firefox).
      test_list: list to add tests to.
      test_prefix: prefix of tests to run.
      test_suffixes: comma separated suffixes of tests to run.
      server_timeout: server timeout value, in milliseconds

    Returns:
      result: result of test runner.
    """
    print "Testing %s..." % browser
    self.StartSession(browser)
    self.session.set_timeout(server_timeout)
    self.runner.setBrowser(browser)

    try:
      result = self.runner.run(
          SeleniumSuite(self.session, browser, test_list,
                        test_prefix, test_suffixes))
    finally:
      self.CloseSession()

    return result


class LocalTestSuite(unittest.TestSuite):
  """Wrapper for unittest.TestSuite so we can collect the tests."""

  def __init__(self):
    unittest.TestSuite.__init__(self)
    self.test_list = []

  def addTest(self, name, test):
    """Adds a test to the TestSuite and records its name and test_path.

    Args:
      name: name of test.
      test: test to pass to unittest.TestSuite.
    """
    unittest.TestSuite.addTest(self, test)
    try:
      self.test_list.append((name, test.options))
    except AttributeError:
      self.test_list.append((name, []))


def MatchesSuffix(name, suffixes):
  """Checks if a name ends in one of the suffixes.

  Args:
    name: Name to test.
    suffixes: list of suffixes to test for.
  Returns:
    True if name ends in one of the suffixes or if suffixes is empty.
  """
  if suffixes:
    name_lower = name.lower()
    for suffix in suffixes:
      if name_lower.endswith(suffix):
        return True
    return False
  else:
    return True


def AddTests(test_suite, session, browser, module, filename, prefix,
             test_prefix_filter, test_suffixes, path_to_html):
  """Add tests defined in filename.

  Assumes module has a method "GenericTest" that uses self.args to run.

  Args:
    test_suite: A Selenium test_suite to add tests to.
    session: a Selenium instance.
    browser: browser name.
    module: module which will have method GenericTest() called to run each test.
    filename: filename of file with list of tests.
    prefix: prefix to add to the beginning of each test.
    test_prefix_filter: Only adds a test if it starts with this.
    test_suffixes: list of suffixes to filter by. An empty list = pass all.
    path_to_html: Path from server root to html
  """
  # See comments in that file for the expected format.
  # skip lines that are blank or have "#" or ";" as their first non whitespace
  # character.
  test_list_file = open(filename, "r")
  samples = test_list_file.readlines()
  test_list_file.close()

  for sample in samples:
    sample = sample.strip()
    if not sample or sample[0] == ";" or sample[0] == "#":
      continue

    arguments = sample.split()
    test_type = arguments[0].lower()
    test_path = arguments[1]
    options = arguments[2:]

    # TODO: Add filter based on test_type

    name = ("Test" + prefix + re.sub("\W", "_", test_path) +
            test_type.capitalize())

    # Only execute this test if the current browser is not in the list
    # of skipped browsers.
    test_skipped = False
    for option in options:
      if option.startswith("except"):
        skipped_platforms = selenium_utilities.GetArgument(option)
        if not skipped_platforms is None:
          skipped_platforms = skipped_platforms.split(",")
          if browser in skipped_platforms:
            test_skipped = True

    if not test_skipped:
      # Check if there is already a test function by this name in the module.
      if (test_path.startswith(test_prefix_filter) and
          hasattr(module, test_path) and callable(getattr(module, test_path))):
        test_suite.addTest(test_path, module(test_path, session, browser,
                                             path_to_html, options=options))
      elif (name.startswith(test_prefix_filter) and
            MatchesSuffix(name, test_suffixes)):
        # no, so add a method that will run a test generically.
        setattr(module, name, module.GenericTest)
        test_suite.addTest(name, module(name, session, browser, path_to_html,
                                        test_type, test_path, options))


def SeleniumSuite(session, browser, test_list, test_prefix, test_suffixes):
  """Creates a test suite to run the unit tests.

  Args:
    session: a selenium() instance
    browser: browser name
    test_list: list to add tests to.
    test_prefix: prefix of tests to run.
    test_suffixes: A comma separated string of suffixes to filter by.
  Returns:
    A selenium test suite.
  """

  test_suite = LocalTestSuite()

  suffixes = test_suffixes.split(",")

  # add sample tests.
  filename = os.path.abspath(os.path.join(script_dir, "sample_list.txt"))
  AddTests(test_suite,
           session,
           browser,
           samples_tests.SampleTests,
           filename,
           "Sample",
           test_prefix,
           suffixes,
           FLAGS.samplespath.replace("\\", "/"))

  # add javascript tests.
  filename = os.path.abspath(os.path.join(script_dir,
                                          "javascript_unit_test_list.txt"))
  AddTests(test_suite,
           session,
           browser,
           javascript_unit_tests.JavaScriptUnitTests,
           filename,
           "UnitTest",
           test_prefix,
           suffixes,
           '')

  test_list += test_suite.test_list

  return test_suite


def CompareScreenshots(browser, test_list, screencompare, screenshotsdir,
                       screencompare_tool, verbose):
  """Performs the image validation for test-case frame captures.

  Args:
    browser: selenium browser name
    test_list: list of tests that ran.
    screencompare: True to actually run tests.
    screenshotsdir: path of directory containing images to compare.
    screencompare_tool: path to image diff tool.
    verbose: If True then outputs verbose info.

  Returns:
    A Results object.
  """
  print "Validating captured frames against reference data..."

  class Results(object):
    """An object to return results of screenshot compares.

    Similar to unittest.TestResults.
    """

    def __init__(self):
      object.__init__(self)
      self.tests_run = 0
      self.current_test = None
      self.errors = []
      self.failures = []
      self.start_time = 0

    def StartTest(self, test):
      """Adds a test.

      Args:
        test: name of test.
      """
      self.start_time = time.time()
      self.tests_run += 1
      self.current_test = test

    def TimeTaken(self):
      """Returns the time since the last call to StartTest."""
      return time.time() - self.start_time

    def AddFailure(self, test, browser, message):
      """Adds a failure.

      Args:
        test: name of the test.
        browser: name of the browser.
        message: error message to print
      """
      self.failures.append(test)
      print "ERROR: ", message
      print("SELENIUMRESULT %s <%s> [%.3fs]: FAIL"
            % (test, browser, self.TimeTaken()))

    def AddSuccess(self, test):
      """Adds a success.

      Args:
        test: name of the test.
      """
      print("SELENIUMRESULT %s <%s> [%.3fs]: PASS"
            % (test, browser, self.TimeTaken()))

    def WasSuccessful(self):
      """Returns true if all tests were successful."""
      return not self.errors and not self.failures

  results = Results()

  if not screencompare:
    return results

  base_path = os.getcwd()

  reference_files = os.listdir(os.path.join(
      base_path,
      selenium_constants.REFERENCE_SCREENSHOT_PATH))

  generated_files = os.listdir(os.path.join(base_path, screenshotsdir))

  # Prep the test list for matching
  temp = []
  for (test, options) in test_list:
    test = selenium_utilities.StripTestTypeSuffix(test)
    temp.append((test.lower(), options))
  test_list = temp

  # Create regex object for filename
  # file is in format "FILENAME_reference.png"
  reference_file_name_regex = re.compile(r"^(.*)_reference\.png")
  generated_file_name_regex = re.compile(r"^(.*)\.png")

  # check that there is a reference file for each generated file.
  for file_name in generated_files:
    match = generated_file_name_regex.search(file_name)

    if match is None:
      # no matches
      continue

    # Generated file name without png extension
    actual_name = match.group(1)

    # Get full paths to reference and generated files
    reference_file = os.path.join(
        base_path,
        selenium_constants.REFERENCE_SCREENSHOT_PATH,
        actual_name + "_reference.png")
    generated_file = os.path.join(
        base_path,
        screenshotsdir,
        actual_name + ".png")

    test_name = "TestReferenceScreenshotExists_" + actual_name
    results.StartTest(test_name)
    if not os.path.exists(reference_file):
      # reference file does not exist
      results.AddFailure(
          test_name, browser,
          "Missing reference file %s for generated file %s." %
          (reference_file, generated_file))
    else:
      results.AddSuccess(test_name)

  # Assuming both the result and reference image sets are the same size,
  # verify that corresponding images are similar within tolerance.
  for file_name in reference_files:
    match = reference_file_name_regex.search(file_name)

    if match is None:
      # no matches
      continue

    # Generated file name without png extension
    actual_name = match.group(1)
    # Get full paths to reference and generated files
    reference_file = os.path.join(
        base_path,
        selenium_constants.REFERENCE_SCREENSHOT_PATH,
        file_name)
    platform_specific_reference_file = os.path.join(
        base_path,
        selenium_constants.PLATFORM_SPECIFIC_REFERENCE_SCREENSHOT_PATH,
        actual_name + "_reference.png")
    generated_file = os.path.join(
        base_path,
        screenshotsdir,
        actual_name + ".png")

    # Generate a test case name
    test_name = "TestScreenCompare_" + actual_name

    # skip the reference file if the test is not in the test list.
    basename = os.path.splitext(os.path.basename(file_name))[0]
    basename = re.sub("\d+_reference", "", basename).lower()
    basename = re.sub("\W", "_", basename)
    test_was_run = False
    test_options = []
    for (test, options) in test_list:
      if test.endswith(basename):
        test_was_run = True
        test_options = options or []
        break

    if test_was_run:
      results.StartTest(test_name)
    else:
      # test was not planned to run for this reference image.
      if os.path.exists(generated_file):
        # a generated file exists? The test name does not match the screenshot.
        results.StartTest(test_name)
        results.AddFailure(test_name, browser,
                           "Test name and screenshot name do not match.")
      continue

    # Check if there is a platform specific version of the reference image
    if os.path.exists(platform_specific_reference_file):
      reference_file = platform_specific_reference_file

    # Check if perceptual diff exists
    pdiff_path = os.path.join(base_path, screencompare_tool)
    if not os.path.exists(pdiff_path):
      # Perceptualdiff.exe does not exist, fail.
      results.AddFailure(
          test_name, browser,
          "Perceptual diff %s does not exist." % pdiff_path)
      continue

    pixel_threshold = "10"
    alpha_threshold = "1.0"
    use_colorfactor = False
    use_downsample = False
    use_edge = True
    edge_threshold = "5"

    # Find out if the test specified any options relating to perceptual diff
    # that will override the defaults.
    for opt in test_options:
      if opt.startswith("pdiff_threshold"):
        pixel_threshold = selenium_utilities.GetArgument(opt)
      elif (opt.startswith("pdiff_threshold_mac") and
            sys.platform == "darwin"):
        pixel_threshold = selenium_utilities.GetArgument(opt)
      elif (opt.startswith("pdiff_threshold_win") and
            sys.platform == 'win32' or sys.platform == "cygwin"):
        pixel_threshold = selenium_utilities.GetArgument(opt)
      elif (opt.startswith("pdiff_threshold_linux") and
            sys.platform[:5] == "linux"):
        pixel_threshold = selenium_utilities.GetArgument(opt)
      elif (opt.startswith("colorfactor")):
        colorfactor = selenium_utilities.GetArgument(opt)
        use_colorfactor = True
      elif (opt.startswith("downsample")):
        downsample_factor = selenium_utilities.GetArgument(opt)
        use_downsample = True
      elif (opt.startswith("pdiff_edge_ignore_off")):
        use_edge = False
      elif (opt.startswith("pdiff_edge_threshold")):
        edge_threshold = selenium_utilities.GetArgument(opt)

    # Check if file exists
    if os.path.exists(generated_file):
      diff_file = os.path.join(base_path, screenshotsdir,
                               "compare_%s.png" % actual_name)

      # Run perceptual diff
      arguments = [pdiff_path,
                   reference_file,
                   generated_file,
                   "-output", diff_file,
                   "-fov", "45",
                   "-alphaThreshold", alpha_threshold,
                   # Turn on verbose output for the percetual diff so we
                   # can see how far off we are on the threshold.
                   "-verbose",
                   # Set the threshold to zero so we can get a count
                   # of the different pixels.  This causes the program
                   # to return failure for most images, but we can compare
                   # the values ourselves below.
                   "-threshold", "0"]
      if use_colorfactor:
        arguments += ["-colorfactor", colorfactor]
      if use_downsample:
        arguments += ["-downsample", downsample_factor]
      if use_edge:
        arguments += ["-ignoreEdges", edge_threshold]

      # Print the perceptual diff command line so we can debug easier.
      if verbose:
        print " ".join(arguments)

      # diff tool should return 0 on success
      expected_result = 0

      pdiff_pipe = subprocess.Popen(arguments,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
      (pdiff_stdout, pdiff_stderr) = pdiff_pipe.communicate()
      result = pdiff_pipe.returncode

      # Find out how many pixels were different by looking at the output.
      pixel_re = re.compile("(\d+) pixels are different", re.DOTALL)
      pixel_match = pixel_re.search(pdiff_stdout)
      different_pixels = "0"
      if pixel_match:
        different_pixels = pixel_match.group(1)

      alpha_re = re.compile("max alpha delta of ([0-9\.]+)", re.DOTALL)
      alpha_delta = "0.0"
      alpha_match = alpha_re.search(pdiff_stdout)
      if alpha_match:
        alpha_delta = alpha_match.group(1)

      if (result == expected_result or (pixel_match and
          int(different_pixels) <= int(pixel_threshold))):
        # The perceptual diff passed.
        pass_re = re.compile("PASS: (.*?)\n", re.DOTALL)
        pass_match = pass_re.search(pdiff_stdout)
        reason = "Images are not perceptually different."
        if pass_match:
          reason = pass_match.group(1)
        print ("%s PASSED with %s different pixels "
               "(threshold %s) because: %s" % (test_name,
                                               different_pixels,
                                               pixel_threshold,
                                               reason))
        results.AddSuccess(test_name)
      else:
        # The perceptual diff failed.
        if pixel_match and int(different_pixels) > int(pixel_threshold):
          results.AddFailure(
              test_name, browser,
              ("Reference framebuffer (%s) does not match generated "
               "file (%s):  %s non-matching pixels, max alpha delta: %s, "
               "threshold: %s, alphaThreshold: %s." %
               (reference_file, generated_file, different_pixels, alpha_delta, 
                pixel_threshold, alpha_threshold)))
        else:
          # The perceptual diff failed for some reason other than
          # pixel differencing.
          fail_re = re.compile("FAIL: (.*?)\n", re.DOTALL)
          fail_match = fail_re.search(pdiff_stdout)
          reason = "Unknown failure"
          if fail_match:
            reason = fail_match.group(1)
          results.AddFailure(
              test_name, browser,
              ("Perceptual diff of reference (%s) and generated (%s) files "
               "failed because: %s" %
               (reference_file, generated_file, reason)))
    else:
      # Generated file does not exist
      results.AddFailure(test_name, browser,
                         "File %s does not exist." % generated_file)

  return results


def main(unused_argv):
  # Boolean to record if all tests passed.
  all_tests_passed = True

  selenium_constants.REFERENCE_SCREENSHOT_PATH = os.path.join(
    FLAGS.referencedir,
    "reference",
    "")
  selenium_constants.PLATFORM_SPECIFIC_REFERENCE_SCREENSHOT_PATH = os.path.join(
    FLAGS.referencedir,
    selenium_constants.PLATFORM_SCREENSHOT_DIR,
    "")

  # Open a new session to Selenium Remote Control
  selenium_session = SeleniumSession(FLAGS.verbose, FLAGS.java,
                                     os.path.abspath(FLAGS.selenium_server),
                                     FLAGS.servertimeout)
  if not selenium_session.http_server or not selenium_session.selenium_server:
    return 1

  for browser in FLAGS.browser:
    if browser in set(selenium_constants.SELENIUM_BROWSER_SET):
      test_list = []
      result = selenium_session.TestBrowser(browser, test_list,
                                            FLAGS.testprefix,
                                            FLAGS.testsuffixes,
                                            int(FLAGS.servertimeout) * 1000)

      # Compare screenshots
      compare_result = CompareScreenshots(browser,
                                          test_list,
                                          FLAGS.screenshots,
                                          FLAGS.screenshotsdir,
                                          FLAGS.screencompare,
                                          FLAGS.verbose)
      if not result.wasSuccessful() or not compare_result.WasSuccessful():
        all_tests_passed = False
      # Log results
      print "Results for %s:" % browser
      print "  %d tests run." % (result.testsRun + compare_result.tests_run)
      print "  %d errors." % (len(result.errors) + len(compare_result.errors))
      print "  %d failures.\n" % (len(result.failures) +
                                  len(compare_result.failures))

    else:
      print "ERROR: Browser %s is invalid." % browser
      print "Run with --help to view list of supported browsers.\n"
      all_tests_passed = False

  # Wrap up session
  selenium_session.TearDown()

  if all_tests_passed:
    # All tests successful.
    return 0
  else:
    # Return error code 1.
    return 1

if __name__ == "__main__":
  remaining_argv = FLAGS(sys.argv)

  # Setup the environment for Firefox
  os.environ["MOZ_CRASHREPORTER_DISABLE"] = "1"
  os.environ["MOZ_PLUGIN_PATH"] = os.path.normpath(FLAGS.product_dir)

  # Setup the LD_LIBRARY_PATH on Linux.
  if sys.platform[:5] == "linux":
    if os.environ.get("LD_LIBRARY_PATH"):
      os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(
        [os.environ["LD_LIBRARY_PATH"], os.path.normpath(FLAGS.product_dir)])
    else:
      os.environ["LD_LIBRARY_PATH"] = os.path.normpath(FLAGS.product_dir)

  sys.exit(main(remaining_argv))