path: root/o3d/tests/lab/run_lab_test.py

#!/usr/bin/python2.6.2
# Copyright 2009, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


"""Prepares to run the selenium lab tests and finally invokes the selenium
runner main.py.                      

Usage:
  run_lab_test.py [test_config_path]

Args:
  browser_config_path: path to test config file (optional)
"""

import copy
import logging
import optparse
import os
import shutil
import subprocess
import sys
import threading
import time

import configure_ie
import runner_constants as const
import runner_util as run_util
import util

# Resolution to configure video card to before running tests.
SCREEN_WIDTH = 1280
SCREEN_HEIGHT = 1024
SCREEN_BPP = 32

join = os.path.join

if util.IsWindows():
  IMAGE_DIFF_PATH = join(const.BASE_PATH, 'third_party', 'pdiff', 'files',
                         'bin', 'win', 'perceptualdiff.exe')
elif util.IsMac():
  IMAGE_DIFF_PATH = join(const.BASE_PATH, 'third_party', 'pdiff', 'files',
                         'bin', 'mac', 'perceptualdiff')
else:
  IMAGE_DIFF_PATH = join(const.BASE_PATH, 'third_party', 'pdiff', 'files',
                         'bin', 'linux', 'perceptualdiff')

SELENIUM_TEST_RUNNER_PATH = join(const.TEST_PATH, 'selenium', 'main.py')

SELENIUM_JAR_PATH = join(const.BASE_PATH, 'third_party', 'selenium_rc', 'files',
                         'selenium-server', 'selenium-server.jar')

O3D_REFERENCE_IMAGES_PATH = join(const.O3D_PATH, 'o3d_assets', 'tests',
                                 'screenshots')

SCREENSHOTS_PATH = join(const.RESULTS_PATH,'screenshots')

# Set total test timeout to 90 minutes.
TEST_TIMEOUT_SECS = 60 * 90.0

SELENIUM_BROWSER_PREFIXES = {
  'ie': '*iexplore',
  'ff': '*firefox',
  'chr': '*googlechrome',
  'saf': '*safari',
}


class TestRunningThread(threading.Thread):
  """Runs test in separate thread.  Allows timeout if test blocks."""

  def __init__(self, command):
    threading.Thread.__init__(self)
    # Make the test running thread a daemon thread.  It blocks waiting for
    # output from the test.  By being a daemon this program can exit even
    # if the runner thread is deadlocked waiting for output from the test.
    self.setDaemon(True)
    self.command = command
    self.return_code = None
    self.has_started_event = threading.Event()
    self.finished_event = threading.Event()
    
  def run(self):
    logging.info('Running tests:')
    logging.info(' '.join(self.command))
    self.test_process = subprocess.Popen(args=self.command,
                                         stdout=subprocess.PIPE,
                                         stderr=subprocess.STDOUT,
                                         universal_newlines=True)

    self.has_started_event.set()
    logging.info('Output from running test follows:')

    while True:
      line = self.test_process.stdout.readline()
      if line:
        print line.strip('\n')
      else:
        break
      
    self.finished_event.set()
        
  
  def HasFinished(self):
    # The main thread must poll the test thread because it is possible
    # for test_process.readline() to hang. This can happen when a debug
    # window opens from a crashing program. This way, we can know when
    # our process has exited, and then try to kill all the windows.
    if not self.has_started_event.isSet():
      # The tests cannot have finished if they have not even started.
      return False
    
    self.test_process.poll()
    
    code = self.test_process.returncode
    
    if code is not None and util.IsWindows():
      # Wait for the test runner to exit safely, if it is able.
      time.sleep(5)
      
      if not self.finished_event.isSet():
        # Something is preventing proper exiting of test runner.
        # Try to kill all the debug windows.
        logging.info('Trying to clean up after the test. Ignore errors here.')
        os.system('TASKKILL /F /IM dwwin.exe')
        os.system('TASKKILL /F /IM iedw.exe')
  
        # Windows Error Reporting (on Vista)
        os.system('TASKKILL /F /IM WerFault.exe')
        
        # Browsers.
        os.system('TASKKILL /F /IM chrome.exe')
        os.system('TASKKILL /F /IM iexplore.exe')
        os.system('TASKKILL /F /IM firefox.exe')
        
        # Wait and see if the test is allowed to finish now.
        time.sleep(5)
      
      if not self.finished_event.isSet():
        logging.error('Could not kill all the extra processes created by the' +
                      'test run.')
      else:
        logging.info('Test process exited succesfully.')
        
    return code is not None
  
  
  def GetReturnCode(self):
    """Returns the exit code from the test runner, or special code 128 if 
    test runner did not exit."""
    
    if not self.has_started_event.isSet():
      code = 128
    else:
      code = self.test_process.returncode
      if code is None:
        code = 128
    return code
    


def RunTest(browser):
  """Runs tests on |browser|.
  Args:
    browser: the browser to test.
  Returns:
    True on success.
  """
  if util.IsWindows(): 
    if not run_util.EnsureWindowsScreenResolution(SCREEN_WIDTH, SCREEN_HEIGHT, 
                                                  SCREEN_BPP):
      logging.warn('Could not detect/change screen resolution.')

  # Clear all screenshots.
  logging.info('** Deleting previous screenshots.')
  if os.path.exists(SCREENSHOTS_PATH):
    shutil.rmtree(SCREENSHOTS_PATH)

  os.makedirs(SCREENSHOTS_PATH)
  
  logging.info('** Running selenium tests...')

  # -u for unbuffered output.
  # Use Python2.4 for two reasons.  First, this is more or less the standard.
  # Second, if we use Python2.6 or later, we must manually shutdown the
  # httpserver, or the next run will overlap ports, which causes
  # some strange problems/exceptions.
  args = [const.PYTHON, '-u', SELENIUM_TEST_RUNNER_PATH]

  browser_parts = browser.split(' ', 1)
  args.append('--browser=' + browser_parts[0])
  if len(browser_parts) > 1:
    args.append('--browserpath=' + browser_parts[1])
  
  args.append('--servertimeout=80')
  args.append('--product_dir=' + const.PRODUCT_DIR_PATH)
  args.append('--verbose=0')
  args.append('--screenshots')
  args.append('--screencompare=' + IMAGE_DIFF_PATH)
  args.append('--screenshotsdir=' + SCREENSHOTS_PATH)
  args.append('--referencedir=' + O3D_REFERENCE_IMAGES_PATH)
  args.append('--selenium_server=' + SELENIUM_JAR_PATH)
  args.append('--testprefix=Test')
  args.append('--testsuffixes=small,medium,large')

  runner = TestRunningThread(args)
  runner.start()

  timeout_time = time.time() + TEST_TIMEOUT_SECS
  while not runner.HasFinished():
    if time.time() > timeout_time:
      break
    time.sleep(5)
  
  return runner.GetReturnCode()
    

def main(argv):
  logfile = os.path.join(os.path.dirname(os.path.abspath(__file__)), 
                         'log_run_lab_test.txt')
  util.ConfigureLogging(logfile)
  logging.info('Running on machine: ' + util.IpAddress())
  
  if len(argv) > 2:
    logging.error('Usage: run_lab_test.py [test_config_file]')
    return 1
  
  if len(argv) == 2:
    # Use given config file.
    config_path = argv[1]
  else:
    # Use default config file.
    config_path = os.path.join(const.HOME, 'test_config.txt')

  # Uninstall/Install plugin.
  if not run_util.UninstallO3DPlugin():
    logging.error('Could not successfully uninstall O3D. Tests will not be run.')
    return 1
  if not run_util.InstallO3DPlugin():
    logging.error('Unable to install O3D plugin. Tests will not be run.')
    return 1
    
  # Grab test configuration info from config file.
  if not os.path.exists(config_path):
    logging.error('Browser config file "%s" could not be found.' % config_path)
    return 1
  else:
    config_file = open(config_path, 'rU')
    test_browsers = []
    while True:
      browser = config_file.readline().strip()
      if len(browser) == 0:
        # No more lines in the file, go ahead and break.
        break
      test_browsers += [browser]

    config_file.close()

  if len(test_browsers) == 0:
    logging.warn('No browsers found in config file. No tests will run.')

  # Test browsers.
  all_test_passed = True
  for browser in test_browsers:
    # Get the Selenium name of the browser. The config file can contain
    # a particular browser name like "ie6", a selenium name like "*firefox",
    # or a selenium name like "*googlechrome C:/Chrome/chrome.exe".
    sel_name = browser
    for prefix in SELENIUM_BROWSER_PREFIXES.keys():
      if browser.startswith(prefix):
        sel_name = SELENIUM_BROWSER_PREFIXES[prefix]

    # Configure IE.
    if sel_name.startswith('*iexplore'):
      if not configure_ie.ConfigureIE():
        logging.error('Failed to configure IE.')
        all_test_passed = False
        continue

    # Run selenium tests.
    if RunTest(sel_name) != 0:
      all_test_passed = False
    
  if all_test_passed:
    logging.info('All tests passed.')
    return 0
  else:
    logging.info('Tests failed.')
    return 1

if __name__ == '__main__':
  code = main(sys.argv)
  sys.exit(code)