path: root/webkit/tools/leak_tests/run_node_leak_test.py

#!/usr/bin/python
# Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run node leak tests using the test_shell.

TODO(pjohnson): Add a way for layout tests (and other local files in the
working copy) to be easily run by specifying paths relative to webkit (or
something similar).
"""

import logging
import optparse
import os
import random
import re
import sys

import google.logging_utils
import google.path_utils
import google.platform_utils
import google.process_utils

# Magic exit code to indicate a new fix.
REBASELINE_EXIT_CODE = -88

# Status codes.
PASS, FAIL, REBASELINE = range(3)

# The test list files are found in this subdirectory, which must be a sibling
# to this script itself.
TEST_FILE_DIR = 'test_lists'

# TODO(pjohnson): Find a way to avoid this duplicate code.  This function has
# been shamelessly taken from layout_tests/layout_package.
_webkit_root = None

def WebKitRoot():
  """Returns the full path to the directory containing webkit.sln.  Raises
  PathNotFound if we're unable to find webkit.sln.
  """

  global _webkit_root
  if _webkit_root:
    return _webkit_root
  webkit_sln_path = google.path_utils.FindUpward(google.path_utils.ScriptDir(),
                                                 'webkit.sln')
  _webkit_root = os.path.dirname(webkit_sln_path)
  return _webkit_root

def GetAbsolutePath(path):
  platform_util = google.platform_utils.PlatformUtility(WebKitRoot())
  return platform_util.GetAbsolutePath(path)

# TODO(pjohnson): Find a way to avoid this duplicated code.  This function has
# been mostly copied from another function, TestShellBinary, in
# layout_tests/layout_package.
def TestShellTestBinary(target):
  """Gets the full path to the test_shell_tests binary for the target build
  configuration.  Raises PathNotFound if the file doesn't exist.
  """

  full_path = os.path.join(WebKitRoot(), target, 'test_shell_tests.exe')
  if not os.path.exists(full_path):
    # Try chrome's output directory in case test_shell was built by chrome.sln.
    full_path = google.path_utils.FindUpward(WebKitRoot(), 'chrome', target,
                                             'test_shell_tests.exe')
    if not os.path.exists(full_path):
      raise PathNotFound('unable to find test_shell_tests at %s' % full_path)
  return full_path

class NodeLeakTestRunner:
  """A class for managing running a series of node leak tests.
  """

  def __init__(self, options, urls):
    """Collect a list of URLs to test.

    Args:
      options: a dictionary of command line options
      urls: a list of URLs in the format:
            (url, expected_node_leaks, expected_js_leaks) tuples
    """

    self._options = options
    self._urls = urls

    self._test_shell_test_binary = TestShellTestBinary(options.target)

    self._node_leak_matcher = re.compile('LEAK: (\d+) Node')
    self._js_leak_matcher = re.compile('Leak (\d+) JS wrappers')

  def RunCommand(self, command):
    def FindMatch(line, matcher, group_number):
      match = matcher.match(line)
      if match:
        return int(match.group(group_number))
      return 0

    (code, output) = google.process_utils.RunCommandFull(command, verbose=True,
                                                         collect_output=True,
                                                         print_output=False)
    node_leaks = 0
    js_leaks = 0

    # Print a row of dashes.
    if code != 0:
      print '-' * 75
      print 'OUTPUT'
      print

    for line in output:
      # Sometimes multiple leak lines are printed out, which is why we
      # accumulate them here.
      node_leaks += FindMatch(line, self._node_leak_matcher, 1)
      js_leaks += FindMatch(line, self._js_leak_matcher, 1)

      # If the code indicates there was an error, print the output to help
      # figure out what happened.
      if code != 0:
        print line

    # Print a row of dashes.
    if code != 0:
      print '-' * 75
      print

    return (code, node_leaks, js_leaks)

  def RunUrl(self, test_url, expected_node_leaks, expected_js_leaks):
    shell_args = ['--gtest_filter=NodeLeakTest.*TestURL',
                  '--time-out-ms=' + str(self._options.time_out_ms),
                  '--test-url=' + test_url,
                  '--playback-mode']

    if self._options.cache_dir != '':
      shell_args.append('--cache-dir=' + self._options.cache_dir)

    command = [self._test_shell_test_binary] + shell_args
    (exit_code, actual_node_leaks, actual_js_leaks) = self.RunCommand(command)

    logging.info('%s\n' % test_url)

    if exit_code != 0:
      # There was a crash, or something else went wrong, so duck out early.
      logging.error('Test returned: %d\n' % exit_code)
      return FAIL

    result = ('TEST RESULT\n'
              '    Node Leaks: %d (actual), %d (expected)\n'
              '    JS Leaks: %d (actual), %d (expected)\n' %
              (actual_node_leaks, expected_node_leaks,
               actual_js_leaks, expected_js_leaks))

    success = (actual_node_leaks <= expected_node_leaks and
               actual_js_leaks <= expected_js_leaks)

    if success:
      logging.info(result)
    else:
      logging.error(result)
      logging.error('Unexpected leaks found!\n')
      return FAIL

    if (expected_node_leaks > actual_node_leaks or
        expected_js_leaks > actual_js_leaks):
       logging.warn('Expectations may need to be re-baselined.\n')
       # TODO(pjohnson): Return REBASELINE here once bug 1177263 is fixed and
       # the expectations have been lowered again.

    return PASS

  def Run(self):
    status = PASS
    results = [0, 0, 0]
    failed_urls = []
    rebaseline_urls = []

    for (test_url, expected_node_leaks, expected_js_leaks) in self._urls:
      result = self.RunUrl(test_url, expected_node_leaks, expected_js_leaks)
      if result == PASS:
        results[PASS] += 1
      elif result == FAIL:
        results[FAIL] += 1
        failed_urls.append(test_url)
        status = FAIL
      elif result == REBASELINE:
        results[REBASELINE] += 1
        rebaseline_urls.append(test_url)
        if status != FAIL:
          status = REBASELINE
    return (status, results, failed_urls, rebaseline_urls)

def main(options, args):
  if options.seed != None:
    random.seed(options.seed)

  # Set up logging so any messages below logging.WARNING are sent to stdout,
  # otherwise they are sent to stderr.
  google.logging_utils.config_root(level=logging.INFO,
                                   threshold=logging.WARNING)

  if options.url_list == '':
    logging.error('URL test list required')
    sys.exit(1)

  url_list = os.path.join(os.path.dirname(sys.argv[0]), TEST_FILE_DIR,
                          options.url_list)
  url_list = GetAbsolutePath(url_list);

  lines = []
  file = None
  try:
    file = open(url_list, 'r')
    lines = file.readlines()
  finally:
    if file != None:
      file.close()

  expected_matcher = re.compile('(\d+)\s*,\s*(\d+)')

  urls = []
  for line in lines:
    line = line.strip()
    if len(line) == 0 or line.startswith('#'):
      continue
    list = line.rsplit('=', 1)
    if len(list) < 2:
      logging.error('Line "%s" is not formatted correctly' % line)
      continue
    match = expected_matcher.match(list[1].strip())
    if not match:
      logging.error('Line "%s" is not formatted correctly' % line)
      continue
    urls.append((list[0].strip(), int(match.group(1)), int(match.group(2))))

  random.shuffle(urls)
  runner = NodeLeakTestRunner(options, urls)
  (status, results, failed_urls, rebaseline_urls) = runner.Run()

  logging.info('SUMMARY\n'
               '    %d passed\n'
               '    %d failed\n'
               '    %d re-baseline\n' %
               (results[0], results[1], results[2]))

  if len(failed_urls) > 0:
    failed_string = '\n'.join('    ' + url for url in failed_urls)
    logging.error('FAILED URLs\n%s\n' % failed_string)

  if len(rebaseline_urls) > 0:
    rebaseline_string = '\n'.join('    ' + url for url in rebaseline_urls)
    logging.warn('RE-BASELINE URLs\n%s\n' % rebaseline_string)

  if status == FAIL:
    return 1
  elif status == REBASELINE:
    return REBASELINE_EXIT_CODE
  return 0

if '__main__' == __name__:
  option_parser = optparse.OptionParser()
  option_parser.add_option('', '--target', default='Debug',
                           help='build target (Debug or Release)')
  option_parser.add_option('', '--cache-dir', default='',
                           help='use a specified cache directory')
  option_parser.add_option('', '--url-list', default='',
                           help='URL input file, with leak expectations, '
                                 'relative to webkit/tools/leak_tests')
  option_parser.add_option('', '--time-out-ms', default=30000,
                           help='time out for each test')
  option_parser.add_option('', '--seed', default=None,
                           help='seed for random number generator, use to '
                                'reproduce the exact same order for a '
                                'specific run')
  options, args = option_parser.parse_args()
  sys.exit(main(options, args))