Adds support for recovering when a running test completes but killed the browser window. Before only test hangs were gracefully handled. Better timeouts for tests. Adds short delay between selenium commands for IE, to fix StressWindow test and others.

Review URL: http://codereview.chromium.org/266031

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@28563 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 442 insertions, 417 deletions

diff --git a/o3d/tests/selenium/main.py b/o3d/tests/selenium/main.py
index d8425a0..392e217 100644
--- a/o3d/tests/selenium/main.py
+++ b/o3d/tests/selenium/main.py
@@ -381,6 +381,10 @@ class SeleniumSessionBuilder:
     
     new_session.start()
     new_session.set_timeout(self.sel_timeout)
+    if browser == "*iexplore":
+      # This improves stability on IE, especially IE 6. It at least fixes the
+      # StressWindow test. It adds a 10ms delay between selenium commands.
+      new_session.set_speed(10)
     
     return new_session
 
diff --git a/o3d/tests/selenium/samples_tests.py b/o3d/tests/selenium/samples_tests.py
index 0a83eb5..06c52fb 100644
--- a/o3d/tests/selenium/samples_tests.py
+++ b/o3d/tests/selenium/samples_tests.py
@@ -180,8 +180,9 @@ class SampleTests(selenium_utilities.SeleniumTestCase):
     self.assertEqual("null", s.get_eval("window.undefined_symbol_xxxyyy"))
 
     # Try different views of the camera
-    # Set delay between each operation at 100ms.
-    s.set_speed(100)
+    # Set delay between each operation at 10ms.
+    speed = int(s.get_speed())
+    s.set_speed(10)
     s.type("eyeX", "5")
     s.type("eyeY", "5")
     s.type("eyeZ", "5")
@@ -193,8 +194,8 @@ class SampleTests(selenium_utilities.SeleniumTestCase):
     s.type("upX", "1")
     s.type("upY", "0")
     s.click("btnSet")
-    # Reset delay to default value, 0ms.
-    s.set_speed(0)
+    # Reset delay to previous value.
+    s.set_speed(speed)
 
     # Capture screenshot
     self.assertTrue(selenium_utilities.TakeScreenShot(s, self.browser,
diff --git a/o3d/tests/selenium/selenium_constants.py b/o3d/tests/selenium/selenium_constants.py
index 75b5116..ad2b166 100644
--- a/o3d/tests/selenium/selenium_constants.py
+++ b/o3d/tests/selenium/selenium_constants.py
@@ -69,6 +69,3 @@ SELENIUM_BROWSER_SET = ["*iexplore", "*firefox", "*googlechrome", "*safari"]
 # otherwise the OpenGL context will be clipped to the size of the window
 RESIZE_WIDTH = 1400
 RESIZE_HEIGHT = 1200
-
-# Time to wait (after load timeout) till assume the browser has crashed.
-MAX_SELENIUM_TEST_TIME = 60
diff --git a/o3d/tests/selenium/selenium_utilities.py b/o3d/tests/selenium/selenium_utilities.py
index ba28e1c..7c23ca9 100644
--- a/o3d/tests/selenium/selenium_utilities.py
+++ b/o3d/tests/selenium/selenium_utilities.py
@@ -243,6 +243,8 @@ class SeleniumTestCase(unittest.TestCase):
       path_to_html: path to html from server root
       test_type: Type of test ("small", "medium", "large")
       sample_path: Path to test.
+      load_timeout: Time to wait for page to load (ms).
+      run_timeout: Time to wait for test to run.
       options: list of option strings.
     """
 
@@ -254,26 +256,38 @@ class SeleniumTestCase(unittest.TestCase):
     self.sample_path = sample_path
     self.path_to_html = path_to_html
     self.screenshots = []
-    self.timeout = 10000
+    self.load_timeout = 10000
+    self.run_timeout = None
     self.client = "g_client"
     # parse options
     for option in options:
-      if option.startswith("screenshot"):
+      if option.startswith("screenshots"):
+        for i in range(int(GetArgument(option))):
+          self.screenshots.append("27.5")
+      elif option.startswith("screenshot"):
         clock = GetArgument(option)
         if clock is None:
           clock = "27.5"
         self.screenshots.append(clock)
       elif option.startswith("timeout"):
-        self.timeout = int(GetArgument(option))
+        self.load_timeout = int(GetArgument(option))
       elif option.startswith("client"):
         self.client = GetArgument(option)
-        
+      elif option.startswith("run_time"):
+        self.run_timeout = int(GetArgument(option))
+    
+    if self.run_timeout is None:
+      # Estimate how long this test needs to run.    
+      time_per_screenshot = 10000
+      if browser == "*iexplore":
+        time_per_screenshot = 60000
+      self.run_timeout = 25000 + len(self.screenshots) * time_per_screenshot
 
   def SetSession(self, session):
     self.session = session
 
   def GetTestTimeout(self):
-    return self.timeout
+    return self.load_timeout + self.run_timeout
 
   def GetURL(self, url):
     """Gets a URL for the test."""
@@ -306,7 +320,7 @@ class SeleniumTestCase(unittest.TestCase):
     g_client which is the o3d client object for that sample.  This is
     used to take a screenshot.
     """
-    self.assertTrue(not self.timeout is None)
+    self.assertTrue(not self.load_timeout is None)
     self.assertTrue(not self.client is None)
     self.assertTrue(self.test_type in ["small", "medium", "large"])
 
@@ -316,7 +330,7 @@ class SeleniumTestCase(unittest.TestCase):
     self.session.open(url)
 
     # wait for it to initialize.
-    self.session.wait_for_condition(ready_condition, self.timeout)
+    self.session.wait_for_condition(ready_condition, self.load_timeout)
 
     self.session.run_script(
         "if (window.o3d_prepForSelenium) { window.o3d_prepForSelenium(); }")
diff --git a/o3d/tests/selenium/test_runner.py b/o3d/tests/selenium/test_runner.py
index 213f8c4..458b4a5 100644
--- a/o3d/tests/selenium/test_runner.py
+++ b/o3d/tests/selenium/test_runner.py
@@ -1,403 +1,412 @@
-#!/usr/bin/python2.4

-# Copyright 2009, Google Inc.

-# All rights reserved.

-#

-# Redistribution and use in source and binary forms, with or without

-# modification, are permitted provided that the following conditions are

-# met:

-#

-#     * Redistributions of source code must retain the above copyright

-# notice, this list of conditions and the following disclaimer.

-#     * Redistributions in binary form must reproduce the above

-# copyright notice, this list of conditions and the following disclaimer

-# in the documentation and/or other materials provided with the

-# distribution.

-#     * Neither the name of Google Inc. nor the names of its

-# contributors may be used to endorse or promote products derived from

-# this software without specific prior written permission.

-#

-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

-

-

-"""Test runners and associated classes.

-

-Each test runner has its own thread, which attempts to perform a given test.

-If a test hangs, the test runner can be aborted or exited.

-

-"""

-

-import os

-import sys

-

-import socket

-import subprocess

-import threading

-import time

-import unittest

-import gflags

-import selenium

-import selenium_constants

-import Queue

-import thread

-import copy

-

-class StringBuffer:

-  """Primitive string buffer.

-

-  Members:

-    data: the contents of the buffer

-  """

-  def __init__(self):

-    self.data = ""

-  def write(self, data):

-    self.data += str(data)

-  def writeln(self, data=None):

-    if data is not None:

-      self.write(data)

-    self.write("\n")

-  def get(self):

-    get_data = self.data

-    self.data = ""

-    return get_data

-

-class TestResult(unittest.TestResult):

-  """A specialized class that prints formatted text results to a stream.

-

-  """

-  separator1 = "=" * 30

-  separator2 = "-" * 30

-

-  def __init__(self, stream, browser, verbose):

-    unittest.TestResult.__init__(self)

-    self.stream = stream

-    # Dictionary of start times

-    self.start_times = {}

-    # Dictionary of results

-    self.results = {}

-    self.browser = browser

-    self.verbose = verbose

-

-  def getDescription(self, test):

-    """Gets description of test."""

-    return test.shortDescription() or str(test)

-

-  def startTest(self, test):

-    """Starts test."""

-    # Records the start time

-    self.start_times[test] = time.time()

-    # Default testresult if success not called

-    self.results[test] = "FAIL"

-    unittest.TestResult.startTest(self, test)

-    if self.verbose:

-      self.stream.writeln()

-      self.stream.writeln(self.separator2)

-      self.stream.write(self.getDescription(test))

-      self.stream.writeln(" ... ")

-

-  def stopTest(self, test):

-    """Called when test is ended."""

-    time_taken = time.time() - self.start_times[test]

-    result = self.results[test]

-    self.stream.writeln("SELENIUMRESULT %s <%s> [%.3fs]: %s"

-                        % (test, self.browser, time_taken, result))

-    self.printErrors()

-

-  def addSuccess(self, test):

-    """Adds success result to TestResult."""

-    unittest.TestResult.addSuccess(self, test)

-    self.results[test] = "PASS"

-

-  def addError(self, test, err):

-    """Adds error result to TestResult."""

-    unittest.TestResult.addError(self, test, err)

-    self.results[test] = "FAIL"

-

-  def addFailure(self, test, err):

-    """Adds failure result to TestResult."""

-    unittest.TestResult.addFailure(self, test, err)

-    self.results[test] = "FAIL"

-

-  def noResponse(self, test):

-    """Configures the result for a test that did not respond."""

-    self.results[test] = "FAIL"

-    self.testsRun += 1

-    self.errors.append("No response from test")

-

-    if self.verbose:

-      self.stream.writeln()

-      self.stream.writeln(self.separator2)

-      self.stream.write(self.getDescription(test))

-      self.stream.writeln(" ... ")

-    self.stream.writeln("SELENIUMRESULT %s <%s> [0s]: FAIL"

-                        % (test, self.browser))

-    self.stream.writeln("Test was aborted due to timeout")

-

-  def printErrors(self):

-    """Prints all errors and failures."""

-    if self.errors:

-      self.printErrorList("ERROR", self.errors)

-    if self.failures:

-      self.printErrorList("FAIL", self.failures)

-

-  def printErrorList(self, flavour, errors):

-    """Prints a given list of errors."""

-    for test, err in errors:

-      self.stream.writeln("%s:" % flavour)

-      self.stream.writeln("%s" % err)

-

-  def printAll(self, stream):

-    """Prints the entire stream to the given stream."""

-    stream.write(self.stream.data)

-

-  def merge(self, result):

-    """Merges the given result into this resultl."""

-    self.testsRun += result.testsRun

-    for key, entry in result.results.iteritems():

-      self.results[key] = entry

-    for error in result.errors:

-      self.errors.append(error)

-    for failure in result.failures:

-      self.failures.append(failure)

-    self.stream.write(result.stream)

-

-

-class TestRunnerThread(threading.Thread):

-  """Abstract test runner class.  Launches its own thread for running tests.

-  Formats test results.

-

-  Members:

-    completely_done_event: event that occurs just before thread exits.

-    test: the currently running test.

-    browser: selenium_name of browser that will be tested.

-  """

-  def __init__(self, verbose):

-    threading.Thread.__init__(self)

-    # This thread is a daemon so that the program can exit even if the

-    # thread has not finished.

-    self.setDaemon(True)

-    self.completely_done_event = threading.Event()

-    self.test = None

-    self.browser = "default_browser"

-    self.verbose = verbose

-

-  def IsCompletelyDone(self):

-    """Returns true if this test runner is completely done."""

-    return self.completely_done_event.isSet()

-

-  def run(self):

-    pass

-    

-  def SetBrowser(self, browser):

-    """Sets the browser name."""

-    self.browser = browser

-

-  def GetNoResponseResult(self):

-    """Returns a generic no response result for last test."""

-    result = TestResult(StringBuffer(), self.browser, self.verbose)

-    result.noResponse(self.test)

-    return result

-

-  def RunTest(self, test):

-    "Run the given test case or test suite."

-    self.test = test

-

-    stream = StringBuffer()

-    result = TestResult(stream, self.browser, self.verbose)

-    startTime = time.time()

-    test(result)

-    stopTime = time.time()

-    timeTaken = stopTime - startTime

-    if self.verbose:

-      result.printErrors()

-    run = result.testsRun

-    return result

-  

-

-class PDiffTestRunner(TestRunnerThread):

-  """Test runner for Perceptual Diff tests. Polls a test queue and launches

-  given tests. Adds result to given queue.

-

-  Members:

-    pdiff_queue: list of tests to run, when they arrive.

-    result_queue: queue of our tests results.

-    browser: selenium name of browser to be tested.

-    end_testing_event: event that occurs when we are guaranteed no more tests

-      will be added to the queue.

-  """

-  def __init__(self, pdiff_queue, result_queue, browser, verbose):

-    TestRunnerThread.__init__(self, verbose)

-    self.pdiff_queue = pdiff_queue

-    self.result_queue = result_queue

-    self.browser = browser

-    

-    self.end_testing_event = threading.Event()

-

-  def EndTesting(self):

-    """Called to notify thread that no more tests will be added to the test

-    queue."""

-    self.end_testing_event.set()

-

-  def run(self):

-    while True:

-      try:

-        test = self.pdiff_queue.get_nowait()

-

-        result = self.RunTest(test)

-        

-        self.result_queue.put(result)

-        

-      except Queue.Empty:

-        if self.end_testing_event.isSet() and self.pdiff_queue.empty():

-          break

-        else:

-          time.sleep(1)

-

-    self.completely_done_event.set()

-    

-

-class SeleniumTestRunner(TestRunnerThread):

-  """Test runner for Selenium tests. Takes a test from a test queue and launches

-  it. Tries to handle hung/crashed tests gracefully.

-

-  Members:

-    testing_event: event that occurs when the runner is testing.

-    finished_event: event that occurs when thread has finished testing and

-      before it starts its next test.

-    can_continue_lock: lock for |can_continue|.

-    can_continue: is True when main thread permits the test runner to continue.

-    sel_builder: builder that constructs new selenium sessions, as needed.

-    browser: selenium name of browser to be tested.

-    session: current selenium session being used in tests, can be None.

-    test_queue: queue of tests to run.

-    pdiff_queue: queue of perceptual diff tests to run. We add a perceptual

-      diff test to the queue when the related selenium test passes.

-    deadline: absolute time of when the test should be done.

-  """

-  def __init__(self, sel_builder, browser, test_queue, pdiff_queue, verbose):

-    TestRunnerThread.__init__(self, verbose)

-

-    # Synchronization.

-    self.testing_event = threading.Event()

-    self.finished_event = threading.Event()

-    self.can_continue_lock = threading.Lock()

-    self.can_continue = False

-    

-    # Selenium variables.

-    self.sel_builder = sel_builder

-    self.browser = browser

-

-    # Test variables.

-    self.test_queue = test_queue

-    self.pdiff_queue = pdiff_queue

-    

-    self.deadline = 0

-    

-  def IsPastDeadline(self):

-    if time.time() > self.deadline:

-      return True

-    return False

-

-  def IsTesting(self):

-    return self.testing_event.isSet()

-

-  def DidFinishTest(self):

-    return self.finished_event.isSet()

-  

-  def Continue(self):

-    """Signals to thread to continue testing.

-

-    Returns:

-      result: the result for the recently finished test.

-    """

-

-    self.finished_event.clear()

-    

-    self.can_continue_lock.acquire()

-    self.can_continue = True

-    result = self.result

-    self.can_continue_lock.release()

-    

-    return result

-    

-  def AbortTest(self):

-    self._StopSession()

-    self._StartSession()

-    

-  def _StartSession(self):

-    self.session = self.sel_builder.NewSeleniumSession(self.browser)

-    # Copy the session so we can shut down safely on a different thread.

-    self.shutdown_session = copy.deepcopy(self.session)

-

-  def _StopSession(self):

-    if self.session is not None:

-      self.session = None

-      try:

-        # This can cause an exception on some browsers.

-        # Silenly disregard the exception.

-        self.shutdown_session.stop()

-      except:

-        pass

-  

-  def run(self):

-    self._StartSession()

-    

-    while not self.test_queue.empty():

-      try:

-        # Grab test from queue.

-        test_obj = self.test_queue.get_nowait()

-        if type(test_obj) == tuple:

-          test = test_obj[0]

-          pdiff_test = test_obj[1]

-        else:

-          test = test_obj

-          pdiff_test = None

-

-        self.can_continue = False

-

-        # Deadline is the time to load page timeout plus a constant.

-        self.deadline = (time.time() + (test.GetTestTimeout() / 1000.0) +

-                         selenium_constants.MAX_SELENIUM_TEST_TIME)

-        # Supply test with necessary selenium session.

-        test.SetSession(self.session)

-

-        # Run test.

-        self.testing_event.set()

-        self.result = self.RunTest(test)

-

-        if time.time() > self.deadline:

-          self.result = self.GetNoResponseResult()

-        

-        self.testing_event.clear()

-        self.finished_event.set()

-

-        # Wait for instruction from the main thread.

-        while True:

-          self.can_continue_lock.acquire()

-          can_continue = self.can_continue

-          self.can_continue_lock.release()

-          if can_continue:

-            break

-          time.sleep(.5)        

-

-        if self.pdiff_queue is not None and pdiff_test is not None:

-          if self.result.wasSuccessful():

-            # Add the dependent perceptual diff test.

-            self.pdiff_queue.put(pdiff_test)       

-        

-      except Queue.Empty:

-        break

-

-    self._StopSession()      

-    self.completely_done_event.set()

- 

-

+#!/usr/bin/python2.4
+# Copyright 2009, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+"""Test runners and associated classes.
+
+Each test runner has its own thread, which attempts to perform a given test.
+If a test hangs, the test runner can be aborted or exited.
+
+"""
+
+import os
+import sys
+
+import socket
+import subprocess
+import threading
+import time
+import unittest
+import gflags
+import selenium
+import selenium_constants
+import Queue
+import thread
+import copy
+
+class StringBuffer:
+  """Primitive string buffer.
+
+  Members:
+    data: the contents of the buffer
+  """
+  def __init__(self):
+    self.data = ""
+  def write(self, data):
+    self.data += str(data)
+  def writeln(self, data=None):
+    if data is not None:
+      self.write(data)
+    self.write("\n")
+  def get(self):
+    get_data = self.data
+    self.data = ""
+    return get_data
+
+class TestResult(unittest.TestResult):
+  """A specialized class that prints formatted text results to a stream.
+
+  """
+  separator1 = "=" * 30
+  separator2 = "-" * 30
+
+  def __init__(self, stream, browser, verbose):
+    unittest.TestResult.__init__(self)
+    self.stream = stream
+    # Dictionary of start times
+    self.start_times = {}
+    # Dictionary of results
+    self.results = {}
+    self.browser = browser
+    self.verbose = verbose
+
+  def getDescription(self, test):
+    """Gets description of test."""
+    return test.shortDescription() or str(test)
+
+  def startTest(self, test):
+    """Starts test."""
+    # Records the start time
+    self.start_times[test] = time.time()
+    # Default testresult if success not called
+    self.results[test] = "FAIL"
+    unittest.TestResult.startTest(self, test)
+    if self.verbose:
+      self.stream.writeln()
+      self.stream.writeln(self.separator2)
+      self.stream.write(self.getDescription(test))
+      self.stream.writeln(" ... ")
+
+  def stopTest(self, test):
+    """Called when test is ended."""
+    time_taken = time.time() - self.start_times[test]
+    result = self.results[test]
+    self.stream.writeln("SELENIUMRESULT %s <%s> [%.3fs]: %s"
+                        % (test, self.browser, time_taken, result))
+    self.printErrors()
+
+  def addSuccess(self, test):
+    """Adds success result to TestResult."""
+    unittest.TestResult.addSuccess(self, test)
+    self.results[test] = "PASS"
+
+  def addError(self, test, err):
+    """Adds error result to TestResult."""
+    unittest.TestResult.addError(self, test, err)
+    self.results[test] = "FAIL"
+
+  def addFailure(self, test, err):
+    """Adds failure result to TestResult."""
+    unittest.TestResult.addFailure(self, test, err)
+    self.results[test] = "FAIL"
+
+  def noResponse(self, test):
+    """Configures the result for a test that did not respond."""
+    self.results[test] = "FAIL"
+    self.testsRun += 1
+    self.errors.append("No response from test")
+
+    if self.verbose:
+      self.stream.writeln()
+      self.stream.writeln(self.separator2)
+      self.stream.write(self.getDescription(test))
+      self.stream.writeln(" ... ")
+    self.stream.writeln("SELENIUMRESULT %s <%s> [0s]: FAIL"
+                        % (test, self.browser))
+    self.stream.writeln("Test was aborted due to timeout")
+
+  def printErrors(self):
+    """Prints all errors and failures."""
+    if self.errors:
+      self.printErrorList("ERROR", self.errors)
+    if self.failures:
+      self.printErrorList("FAIL", self.failures)
+
+  def printErrorList(self, flavour, errors):
+    """Prints a given list of errors."""
+    for test, err in errors:
+      self.stream.writeln("%s:" % flavour)
+      self.stream.writeln("%s" % err)
+
+  def printAll(self, stream):
+    """Prints the entire stream to the given stream."""
+    stream.write(self.stream.data)
+
+  def merge(self, result):
+    """Merges the given result into this resultl."""
+    self.testsRun += result.testsRun
+    for key, entry in result.results.iteritems():
+      self.results[key] = entry
+    for error in result.errors:
+      self.errors.append(error)
+    for failure in result.failures:
+      self.failures.append(failure)
+    self.stream.write(result.stream)
+
+
+class TestRunnerThread(threading.Thread):
+  """Abstract test runner class.  Launches its own thread for running tests.
+  Formats test results.
+
+  Members:
+    completely_done_event: event that occurs just before thread exits.
+    test: the currently running test.
+    browser: selenium_name of browser that will be tested.
+  """
+  def __init__(self, verbose):
+    threading.Thread.__init__(self)
+    # This thread is a daemon so that the program can exit even if the
+    # thread has not finished.
+    self.setDaemon(True)
+    self.completely_done_event = threading.Event()
+    self.test = None
+    self.browser = "default_browser"
+    self.verbose = verbose
+
+  def IsCompletelyDone(self):
+    """Returns true if this test runner is completely done."""
+    return self.completely_done_event.isSet()
+
+  def run(self):
+    pass
+    
+  def SetBrowser(self, browser):
+    """Sets the browser name."""
+    self.browser = browser
+
+  def GetNoResponseResult(self):
+    """Returns a generic no response result for last test."""
+    result = TestResult(StringBuffer(), self.browser, self.verbose)
+    result.noResponse(self.test)
+    return result
+
+  def RunTest(self, test):
+    "Run the given test case or test suite."
+    self.test = test
+
+    stream = StringBuffer()
+    result = TestResult(stream, self.browser, self.verbose)
+    startTime = time.time()
+    test(result)
+    stopTime = time.time()
+    timeTaken = stopTime - startTime
+    if self.verbose:
+      result.printErrors()
+    run = result.testsRun
+    return result
+  
+
+class PDiffTestRunner(TestRunnerThread):
+  """Test runner for Perceptual Diff tests. Polls a test queue and launches
+  given tests. Adds result to given queue.
+
+  Members:
+    pdiff_queue: list of tests to run, when they arrive.
+    result_queue: queue of our tests results.
+    browser: selenium name of browser to be tested.
+    end_testing_event: event that occurs when we are guaranteed no more tests
+      will be added to the queue.
+  """
+  def __init__(self, pdiff_queue, result_queue, browser, verbose):
+    TestRunnerThread.__init__(self, verbose)
+    self.pdiff_queue = pdiff_queue
+    self.result_queue = result_queue
+    self.browser = browser
+    
+    self.end_testing_event = threading.Event()
+
+  def EndTesting(self):
+    """Called to notify thread that no more tests will be added to the test
+    queue."""
+    self.end_testing_event.set()
+
+  def run(self):
+    while True:
+      try:
+        test = self.pdiff_queue.get_nowait()
+
+        result = self.RunTest(test)
+        
+        self.result_queue.put(result)
+        
+      except Queue.Empty:
+        if self.end_testing_event.isSet() and self.pdiff_queue.empty():
+          break
+        else:
+          time.sleep(1)
+
+    self.completely_done_event.set()
+    
+
+class SeleniumTestRunner(TestRunnerThread):
+  """Test runner for Selenium tests. Takes a test from a test queue and launches
+  it. Tries to handle hung/crashed tests gracefully.
+
+  Members:
+    testing_event: event that occurs when the runner is testing.
+    finished_event: event that occurs when thread has finished testing and
+      before it starts its next test.
+    can_continue_lock: lock for |can_continue|.
+    can_continue: is True when main thread permits the test runner to continue.
+    sel_builder: builder that constructs new selenium sessions, as needed.
+    browser: selenium name of browser to be tested.
+    session: current selenium session being used in tests, can be None.
+    test_queue: queue of tests to run.
+    pdiff_queue: queue of perceptual diff tests to run. We add a perceptual
+      diff test to the queue when the related selenium test passes.
+    deadline: absolute time of when the test should be done.
+  """
+  def __init__(self, sel_builder, browser, test_queue, pdiff_queue, verbose):
+    TestRunnerThread.__init__(self, verbose)
+
+    # Synchronization.
+    self.testing_event = threading.Event()
+    self.finished_event = threading.Event()
+    self.can_continue_lock = threading.Lock()
+    self.can_continue = False
+    
+    # Selenium variables.
+    self.sel_builder = sel_builder
+    self.browser = browser
+
+    # Test variables.
+    self.test_queue = test_queue
+    self.pdiff_queue = pdiff_queue
+    
+    self.deadline = 0
+    
+  def IsPastDeadline(self):
+    if time.time() > self.deadline:
+      return True
+    return False
+
+  def IsTesting(self):
+    return self.testing_event.isSet()
+
+  def DidFinishTest(self):
+    return self.finished_event.isSet()
+  
+  def Continue(self):
+    """Signals to thread to continue testing.
+
+    Returns:
+      result: the result for the recently finished test.
+    """
+
+    self.finished_event.clear()
+    
+    self.can_continue_lock.acquire()
+    self.can_continue = True
+    result = self.result
+    self.can_continue_lock.release()
+    
+    return result
+    
+  def AbortTest(self):
+    self._StopSession()
+    self._StartSession()
+    
+  def _StartSession(self):
+    self.session = self.sel_builder.NewSeleniumSession(self.browser)
+    # Copy the session so we can shut down safely on a different thread.
+    self.shutdown_session = copy.deepcopy(self.session)
+
+  def _StopSession(self):
+    if self.session is not None:
+      self.session = None
+      try:
+        # This can cause an exception on some browsers.
+        # Silenly disregard the exception.
+        self.shutdown_session.stop()
+      except:
+        pass
+  
+  def run(self):
+    self._StartSession()
+    
+    while not self.test_queue.empty():
+      try:
+        # Grab test from queue.
+        test_obj = self.test_queue.get_nowait()
+        if type(test_obj) == tuple:
+          test = test_obj[0]
+          pdiff_test = test_obj[1]
+        else:
+          test = test_obj
+          pdiff_test = None
+
+        self.can_continue = False
+        
+        # Check the current selenium session. Particularly, we are
+        # interested if the previous test ran to completion, but the
+        # browser window is closed.
+        try:
+          # This will generate an exception if the window is closed.
+          self.session.window_focus()
+        except Exception:
+          self._StopSession()
+          self._StartSession()
+
+        # Deadline is the time to load page plus test run time.
+        self.deadline = time.time() + (test.GetTestTimeout() / 1000.0)
+        # Supply test with necessary selenium session.
+        test.SetSession(self.session)
+
+        # Run test.
+        self.testing_event.set()
+        self.result = self.RunTest(test)
+
+        if time.time() > self.deadline:
+          self.result = self.GetNoResponseResult()
+        
+        self.testing_event.clear()
+        self.finished_event.set()
+
+        # Wait for instruction from the main thread.
+        while True:
+          self.can_continue_lock.acquire()
+          can_continue = self.can_continue
+          self.can_continue_lock.release()
+          if can_continue:
+            break
+          time.sleep(.5)        
+
+        if self.pdiff_queue is not None and pdiff_test is not None:
+          if self.result.wasSuccessful():
+            # Add the dependent perceptual diff test.
+            self.pdiff_queue.put(pdiff_test)       
+        
+      except Queue.Empty:
+        break
+
+    self._StopSession()      
+    self.completely_done_event.set()
+ 
+