path: root/chrome/tools/test/smoketests.py

#!/usr/bin/python
# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Runs all the available unit tests, layout tests, page-cycler tests, etc.
for a build of Chrome, imitating a buildbot.

Usage examples:
  smoketests.py
  smoketests.py --target=debug --build-type=kjs
  smoketests.py --nopage-cycler
  smoketests.py --tests=unit,ui --verbose

For a full list of options, pass the '--help' switch.

[Alternatively, this script will kill all the tests' executables, in case one
got orphaned during a previous run of this script.  (This generally only
happens during script development.)]

"""

import errno
import optparse
import os
import subprocess
import sys
import time

# We have a chicken-and-egg problem here, since the utils we'd like to use to
# find the paths we need are located in the utils files we're trying to find.
# So we'll make a rough attempt based on the current repository setup, and
# give an error message if it still doesn't work.
this_script_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
# This script lives in chrome/tools/test. The utils live in tools/python.
python_google_path = os.path.join(this_script_dir[:-len('chrome/tools/test')],
                                                  'tools', 'python')
sys.path.insert(0, python_google_path)

try:
  import google.httpd_utils
  import google.path_utils
  import google.process_utils
except ImportError:
  print ("\n>> You must have your local path of trunk/src/tools/python added"
         " to your PYTHONPATH.<<\n")
  raise

# Keep a global httpd object so it can be killed in the event of errors.
_httpd = None

# All the available commands, by name. Items in the command-line lists may
# contain various keywords, listed in the "Substitutions" section below.
# The target build directory will be prepended to the first item in each
# command list.
COMMANDS = {'ipc':               ['ipc_tests.exe'],
            'unit':              ['unit_tests.exe'],
            'ui':                ['ui_tests.exe', '%(page_heap)s'],
            'ui-single':         ['ui_tests.exe', '--single-process'],
            'test_shell':        ['test_shell_tests.exe'],
            'page-cycler-moz':   ['page_cycler_tests.exe',
                                  '--gtest_filter=PageCycler*.MozFile'],
            'page-cycler-moz-http': ['page_cycler_tests.exe',
                                     '--gtest_filter=PageCycler*.MozHttp'],
            'page-cycler-intl1': ['page_cycler_tests.exe',
                                  '--gtest_filter=PageCycler*.Intl1File'],
            'page-cycler-intl2': ['page_cycler_tests.exe',
                                  '--gtest_filter=PageCycler*.Intl2File'],
            'page-cycler-bloat-http': ['page_cycler_tests.exe',
                                       '--gtest_filter=PageCycler*.BloatHttp'],
            'startup':           ['startup_tests.exe',
                                  '--gtest_filter=Startup*.*'],
            'dest-startup':      ['startup_tests.exe',
                                  '--gtest_filter=DestinationsStartupTest.*'],
            'selenium':          ['selenium_tests.exe'],
            'plugin':            ['plugin_tests.exe'],
            'installer':         ['installer_util_unittests.exe'],
            'webkit':            ['%(python)s',
                                  '%(slave_scripts)s/layout_test_wrapper.py',
                                  '--build-type', '%(build_type)s',
                                  '--target', '%(target)s',
                                  '%(page_heap)s'],
           }

# Certain tests are not run for each build type.
SKIPPED = {'Release': ['plugin'],
           'Debug':   ['selenium', 'webkit']}

def _BuildbotScriptPath(sub_dir):
  """Returns the full path to the given subdir of tools/buildbot/scripts,
  or None if that path cannot be found.
  """
  this_script_dir = google.path_utils.ScriptDir()
  # Most tests don't actually need this, so defer failure until something
  # tries to use the result.
  try:
    buildbot_path = google.path_utils.FindUpward(this_script_dir,
                                                 'tools',
                                                 'buildbot',
                                                 'scripts',
                                                  sub_dir)
  except google.path_utils.PathNotFound:
    buildbot_path = None
  return buildbot_path


def _MakeSubstitutions(list, options):
  """Makes substitutions in each item of a list and returns the resulting list.

  Args:
    list: a list of strings, optionally containing certain %()s substitution
        tags listed below
    options: options as returned by optparse

  Raises:
    google.path_utils.PathNotFound if slave_scripts substitution is needed
        but not available
  """
  this_script_dir = google.path_utils.ScriptDir()
  python_path = google.path_utils.FindUpward(this_script_dir,
                                             'third_party',
                                             'python_24',
                                             'python_slave.exe')

  substitutions = {'target':        options.target,
                   'build_type':    options.build_type,
                   'page_heap':     '',
                   'python':        python_path,
                   'slave_scripts': _BuildbotScriptPath('slave'),
                  }
  if options.build_type == 'kjs':
    substitutions['page_heap'] = '--enable-pageheap'
  # If we need the slave_scripts substitution but don't have it, raise an
  # exception. This allows running most of the tests without checking out all
  # the buildbot infrastructure.
  if not substitutions['slave_scripts']:
    for word in list:
      if word.find('%(slave_scripts)s') != -1:
        raise google.path_utils.PathNotFound('Unable to find buildbot scripts')
  return [word % substitutions for word in list]


def RunTestsInShards(test_command, verbose=True):
  """Runs a test in shards. The number of shards is equal to
  NUMBER_OF_PROCESSORS.

  Args:
    test_command: the test command to run, which is a list of one or more
                  strings.
    verbose: if True, combines stdout and stderr into stdout.
             Otherwise, prints only the command's stderr to stdout.

  Returns:
    The first shard process's exit status.

  Raises:
    CommandNotFound if the command executable could not be found.
  """
  processor_count = 2
  try:
    processor_count = int(os.environ['NUMBER_OF_PROCESSORS'])
  except KeyError:
    print 'No NUMBER_OF_PROCESSORS defined. Use 2 instances.'

  commands = []
  for i in xrange(processor_count):
    command = [test_command[j] for j in xrange(len(test_command))]
    # To support sharding, the test executable needs to provide --batch-count
    # --batch-index command line switches.
    command.append('--batch-count=%s' % processor_count)
    command.append('--batch-index=%d' % i)
    commands.append(command)
  return google.process_utils.RunCommandsInParallel(commands, verbose)[0][0]


def main(options, args):
  """Runs all the selected tests for the given build type and target."""
  options.build_type = options.build_type.lower()
  options.target = options.target.title()

  this_script_dir = google.path_utils.ScriptDir()
  test_path = google.path_utils.FindUpward(this_script_dir,
                                           'chrome', options.target)

  # Add the buildbot script paths to the module search path.
  sys.path.insert(0, _BuildbotScriptPath('slave'))
  sys.path.insert(0, _BuildbotScriptPath('common'))

  # Collect list of tests to run.
  if options.tests == '':
    tests = sorted(COMMANDS.keys())
  else:
    tests = set()
    requested_tests = options.tests.lower().split(',')
    for test in requested_tests:
      if test in COMMANDS:
        tests.add(test)
      else:
        print 'Ignoring unknown test "%s"' % test

  # Check page-cycler data, since the tests choke if it isn't available.
  try:
    page_cycler_data = google.path_utils.FindUpward(this_script_dir,
                                                    'data',
                                                    'page_cycler')
  except google.path_utils.PathNotFound:
    # Were we going to run any page-cycler tests?
    if (not options.nopage_cycler and
        len([x for x in tests if x.startswith('page-cycler')])):
      print 'Skipping page-cycler tests (no data)'
    options.nopage_cycler = True

  # Start an httpd if needed.
  http_tests = [x for x in tests if x.endswith('-http')]
  if http_tests and not options.nopage_cycler and not options.nohttp:
    try:
      _httpd = google.httpd_utils.StartServer(document_root=page_cycler_data)
    except google.httpd_utils.HttpdNotStarted:
      print 'Skipping http tests (httpd failed to start)'
      options.nohttp = True

  # Remove tests not desired.
  if options.nopage_cycler:
    tests = [x for x in tests if not x.startswith('page-cycler')]
  if options.nowebkit and 'webkit' in tests:
    tests.remove('webkit')
  if options.nohttp:
    tests = [x for x in tests if not x.endswith('-http')]

  # Remove tests skipped for this build target.
  for skip in SKIPPED[options.target]:
    if skip in tests:
      print 'Skipping %s for %s build' % (skip, options.target)
      tests.remove(skip)

  if not len(tests):
    print 'No tests to run.'
    return 0

  # Run each test, substituting strings as needed.
  failures = []
  start_time = time.time()
  for test in tests:
    test_start_time = time.time()
    try:
      command = _MakeSubstitutions(COMMANDS[test], options)
    except google.path_utils.PathNotFound, e:
      print 'Skipping %s: %s' % (test, e)
      failures.append(test)
      continue
    command[0] = os.path.join(test_path, command[0])
    if options.verbose:
      print
    print 'Running %s:' % test,
    try:
      if test == 'ui':
        result = RunTestsInShards(command, options.verbose)
      else:
        result = google.process_utils.RunCommand(command, options.verbose)
    except google.process_utils.CommandNotFound, e:
      print '%s' % e
      raise
    if options.verbose:
      print test,
    print '(%ds)' % (time.time() - test_start_time),
    if result:
      print 'FAIL'
      failures.append(test)
    else:
      print 'PASS'

  print 'Total time: %ds' % (time.time() - start_time)
  if len(failures):
    print 'Failed tests:'
    print os.linesep.join(failures)
  else:
    print 'All tests passed. Hurrah!'

  return len(failures)

if '__main__' == __name__:
  option_parser = optparse.OptionParser()
  option_parser.add_option('', '--target', default='Release',
                           help='build target (Debug or Release)')
  option_parser.add_option('', '--build-type', default='v8',
                           help='build type (V8 or KJS), used by webkit tests')
  option_parser.add_option('', '--verbose', action='store_true', default=False,
                           help='show full output from every command')
  option_parser.add_option('', '--nopage-cycler', action='store_true',
                           default=False, help='disable page-cycler tests')
  option_parser.add_option('', '--nowebkit', action='store_true',
                           default=False, help='disable webkit (layout) tests')
  option_parser.add_option('', '--nohttp', action='store_true',
                           default=False,
                           help="don't run tests (e.g. page_cycler) with http")
  option_parser.add_option('', '--tests', default='',
                           help='comma-separated list of tests to run, from '
                                '{%s}' % ', '.join(sorted(COMMANDS.keys())))
  option_parser.add_option('', '--killall', action='store_true', default=False,
                           help='kill all test executables (and run no tests)')
  options, args = option_parser.parse_args()

  if options.killall:
    kill_list = _MakeSubstitutions([COMMANDS[x][0] for x in COMMANDS.keys()],
                                   options)
    kill_list = set([os.path.basename(x) for x in kill_list])
    sys.exit(google.process_utils.KillAll(kill_list))

  try:
    result = main(options, args)
  finally:
    # Kill the httpd.
    if _httpd:
      _httpd.StopServer(force=True)
  sys.exit(result)