2 files changed, 515 insertions, 0 deletions
diff --git a/ceee/tools/smoke_test.py b/ceee/tools/smoke_test.py
new file mode 100644
index 0000000..1ad5e3f
--- /dev/null
+++ b/ceee/tools/smoke_test.py
@@ -0,0 +1,212 @@
+#!python
+# Copyright (c) 2010 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+'''This scripts builds and runs the CEEE unittests and medium tests that
+make our smoke tests.
+
+It exits with a non-zero exit status on error.
+'''
+try:
+  import ctypes
+  import pythoncom
+  import win32com.client
+except ImportError:
+  print "This script requires Python 2.6 with PyWin32 installed."
+  raise
+
+import datetime
+import os.path
+import subprocess
+import sys
+import time
+import verifier
+
+# The top-level source directory.
+# All other paths in this file are relative to this directory.
+_SRC_DIR = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../..'))
+
+
+def _AbsolutePath(path):
+  '''Convert path to an absolute path.
+
+  Args:
+    path: a path relative to _SRC_DIR.
+
+  Returns: Path as an absolute, normalized path.
+  '''
+  return os.path.abspath(os.path.join(_SRC_DIR, path))
+
+
+_CHROME_SOLUTION = _AbsolutePath('chrome/chrome.sln')
+
+
+# List of project files to build.
+_PROJECTS_TO_BUILD = [
+    # Common unittest
+    _AbsolutePath('ceee/common/ceee_common_unittests.vcproj'),
+
+    # IE unittests and medium tests.
+    _AbsolutePath('ceee/ie/ie_unittests.vcproj'),
+    _AbsolutePath('ceee/ie/mediumtest_ie.vcproj'),
+]
+
+# List of test executables to run.
+_TESTS_TO_RUN = [
+  'ceee_common_unittests',
+  'ie_unittests',
+  'mediumtest_ie',
+]
+
+# A list of configurations we build and run.
+_CONFIGURATIONS = [
+  'Debug',
+  'Release',
+]
+
+class TestRunner(object):
+  def __init__(self, solution, projects, tests):
+    self._projects = projects
+    self._tests = tests
+    self._saved_autohides = None
+    self._solution = win32com.client.GetObject(solution)
+    self._dte = self._GetDte(self._solution)
+
+    self._EnsureVisible()
+
+  def __del__(self):
+    self._RestoreVisibility()
+
+  def _EnsureVisible(self):
+    dte = self._dte
+    # Make sure the UI is visible.
+    dte.MainWindow.Visible = True
+
+    # Get the output window, disable its autohide and give it focus.
+    self._output = dte.Windows['Output']
+    self._saved_autohides = self._output.AutoHides
+    self._output.AutoHides = False
+    self._output.SetFocus()
+
+  def _RestoreVisibility(self):
+    if self._saved_autohides != None:
+      self._output.AutoHides = self._saved_autohides
+
+  def _GetDte(self, solution):
+    '''Sometimes invoking solution.DTE will fail with an exception during
+    Visual Studio initialization. To work around this, we try a couple of
+    times with an intervening sleep to give VS time to settle.'''
+    # Attempt ten times under a try block.
+    for i in range(0, 10):
+      try:
+        return solution.DTE
+      except pythoncom.com_error, e:
+        # Sleep for 2 seconds
+        print 'Exception from solution.DTE "%s", retrying: ' % str(e)
+        time.sleep(2.0)
+
+    # Final try, pass the exception to the caller on failure here.
+    return solution.DTE
+
+  def BuildConfig(self, config):
+    '''Builds all projects for a given config.
+
+    Args:
+      config: the name of the build configuration to build projecs for.
+
+    Returns:
+      The number of failures.
+    '''
+    builder = self._solution.SolutionBuild
+    for project in self._projects:
+      print 'Building project "%s" in "%s" configuration' % (project, config)
+      wait_for_build = True
+      # See http://goo.gl/Dy8x for documentation on this method.
+      builder.BuildProject(config, project, wait_for_build)
+      errors = builder.LastBuildInfo
+      if errors != 0:
+        return errors
+
+    return 0
+
+  def RunConfigTests(self, config):
+    '''Runs all tests for a given config and writes success files.
+
+    For each succeeding test '<path>/foo.exe', this function writes a
+    corresponding success file '<path>/foo.success'. If the corresponding
+    success file is newer than the test executable, the test executable will
+    not run again.
+    If all tests succeed, this function writes an overall success file
+    named 'ceee.success'.
+
+    Args:
+      config: the name of the build configuration to run tests for.
+
+    Returns:
+      The number of failures.
+    '''
+    failures = 0
+    for test in self._tests:
+      test_exe = os.path.join(_SRC_DIR,
+                              'chrome',
+                              config,
+                              test) + '.exe'
+      success_file = os.path.join(_SRC_DIR,
+                                  'chrome',
+                                  config,
+                                  test) + '.success'
+      if (os.path.exists(success_file) and
+          os.stat(test_exe).st_mtime <= os.stat(success_file).st_mtime):
+        print '"%s" is unchanged since last successful run' % test_exe
+      else:
+        if verifier.HasAppVerifier():
+          (exit, errors) = verifier.RunTestWithVerifier(test_exe)
+        else:
+          exit = subprocess.call(test_exe)
+
+        # Create a success file if the test succeded.
+        if exit == 0:
+          open(success_file, 'w').write(str(datetime.datetime.now()))
+        else:
+          failures = failures + 1
+          print '"%s" failed with exit status %d' % (test_exe, exit)
+
+    # Create the overall success file if everything succeded.
+    if failures == 0:
+      success_file = os.path.join(_SRC_DIR,
+                                  'chrome',
+                                  config,
+                                  'ceee') + '.success'
+      open(success_file, 'w').write(str(datetime.datetime.now()))
+
+    return failures
+
+
+def Main():
+  if not ctypes.windll.shell32.IsUserAnAdmin():
+    print ("Please run the smoke tests in an admin prompt "
+           "(or AppVerifier won't work).")
+    return 1
+
+  runner = TestRunner(_CHROME_SOLUTION, _PROJECTS_TO_BUILD, _TESTS_TO_RUN)
+
+  failures = 0
+  for config in _CONFIGURATIONS:
+    failures += runner.BuildConfig(config)
+
+  if failures != 0:
+    print '%d build errors' % failures
+    return failures
+
+  for config in _CONFIGURATIONS:
+    failures += runner.RunConfigTests(config)
+
+  if failures != 0:
+    print '%d unittests failed' % failures
+
+  return failures
+
+
+if __name__ == '__main__':
+  sys.exit(Main())
diff --git a/ceee/tools/verifier.py b/ceee/tools/verifier.py
new file mode 100644
index 0000000..31fc6d2
--- /dev/null
+++ b/ceee/tools/verifier.py
@@ -0,0 +1,303 @@
+# Copyright (c) 2010 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+'''Utilities to automate running tests under app verifier.'''
+
+import os
+import os.path
+import re
+import subprocess
+import sys
+import tempfile
+import win32com.client.gencache as gencache
+import xml.sax
+import xml.sax.handler
+
+
+TRACE_RE_ = re.compile('(?P<module>\w+)'  # Module
+    '(?:'
+      '!'  # Separator
+      '(?P<symbol>[^\+]+)'  # Symbol
+      '(?:\+(?P<offset>[0-9a-f]+))?'  # Optional hex offset
+    ')?'  # Sometimes we only get a module
+    '(?:\s+\('  # Optional file and line
+      '(?P<file>[^@]*)'  # file
+      '\s+@\s*'
+      '(?P<line>\d+)'  # line
+    '\))?'
+    , re.I)
+
+
+# This is application verifier's automation library id.
+LIBID_AppVerifier = '{DAB52BCB-6990-464A-AC61-F60C8EF60E24}'
+try:
+  VerifierLib = gencache.EnsureModule(LIBID_AppVerifier, 0, 1, 0)
+except:
+  VerifierLib = None
+
+
+class LogTrace:
+  def __init__(self, text):
+    d = TRACE_RE_.match(text).groupdict()
+    self.module = d['module']
+    self.symbol = d['symbol']
+    self.offset = d['offset']
+    self.file = d['file']
+    self.line = d['line']
+
+  def __str__(self):
+    ret = ''
+    if self.file and self.line:
+      ret = '%s (%s): AppVerifier warning C1: ' % (self.file, self.line)
+
+    ret = '%s%s!%s' % (ret, self.module, self.symbol)
+
+    if self.offset:
+      ret = "%s+%s" % (ret, self.offset)
+
+    return ret
+
+
+class LogEntry:
+  def __init__(self, stopcode, layer, severity):
+    self.stopcode = stopcode
+    self.layer = layer
+    self.severity = severity
+    self.message = ''
+    self.trace = []
+
+  def __str__(self):
+    return "%s: %s\n\t%s" % (self.severity,
+                             self.message,
+                             '\n\t'.join(map(str,self.trace)))
+
+
+class VerifierSaxHandler(xml.sax.handler.ContentHandler):
+  '''Sax event handler to parse the verifier log file.
+
+  The XML generated by verifier appears not to be standards compliant, so
+  e.g. xml.dom.minidom falls down while parsing it, hence this tedious
+  implementation.
+  '''
+  def __init__(self):
+    self.log_entry = None
+    self.stack = []
+    self.element = None
+    self.errors = 0
+    self.text = ''
+    self.log_entries = []
+
+  def startElement(self, name, attrs):
+    self.stack.append((self.element, self.text))
+    self.element = name
+    self.text = []
+
+    if name == 'avrf:logEntry':
+      self.log_entry = LogEntry(attrs.getValue('StopCode'),
+                               attrs.getValue('LayerName'),
+                               attrs.getValue('Severity'))
+      self.errors += 1
+
+  def endElement(self, name):
+    if name == 'avrf:logEntry':
+      self.log_entries.append(self.log_entry)
+      self.log_entry = None
+    elif name == 'avrf:message':
+      self.log_entry.message = ''.join(self.text)
+    elif name == 'avrf:trace':
+      self.log_entry.trace.append(LogTrace(''.join(self.text)))
+
+    (self.element, self.text) = self.stack.pop()
+
+  def characters(self, content):
+    self.text.append(content)
+
+
+class AppverifierTestRunner:
+  '''Encapsulates logic to run an executable under app verifier.
+
+  Interacts with application verifiers automation library to set default
+  verifier settings for a given target, process logs etc.
+  '''
+  # Checks we want enabled.
+  default_checks_ = [
+      # Basics group
+      "Core",
+      "Exceptions",
+      "Handles",
+      "Heaps",
+      "InputOutput",
+      "Locks",
+      "Memory",
+      "TLS",
+
+      # Misc group
+      "DangerousAPIs",
+      "DirtyStacks",
+      "TimeRollOver",
+      ]
+
+  def __init__(self, break_on_stop):
+    self.manager = VerifierLib.AppVerifierManager()
+    self.break_on_stop = break_on_stop
+
+  def SetStopBreaks(self, check):
+    '''Configures all the stops for check to log only.
+
+      Arguments:
+        check: an app verifier check
+    '''
+    error_reporting = VerifierLib.constants.VerifierErrorReportingNoBreak
+    error_flags = (VerifierLib.constants.VerifierErrorFlagLogToFile |
+                   VerifierLib.constants.VerifierErrorFlagLogStackTrace)
+    if self.break_on_stop:
+      error_reporting = VerifierLib.constants.VerifierErrorReportingBreakpoint
+
+    try:
+      for stop in check.Stops:
+        stop.ErrorReporting = error_reporting
+        stop.ErrorFlags = error_flags
+    except:
+      # Accessing or enumerating Stops fails for some checks.
+      print 'Exception setting options for check', check.Name
+
+  def ResetImage(self, image_name):
+    '''Removes all verifier settings for image_name.
+    '''
+    # Reset the settings for our image
+    try:
+      self.manager.Images.Remove(image_name)
+    except:
+      # this fails if verifier had no settings for the image
+      pass
+
+  def SetImageDefaults(self, image_name):
+    '''Configures a default set of tests for image_name
+
+      Arguments:
+        image_name: the basename of a test, e.g. 'common_unittest.exe'
+    '''
+    self.ResetImage(image_name)
+
+    image = self.manager.Images.Add(image_name)
+    for check in image.Checks:
+      if check.Name in self.default_checks_:
+        check.Enabled = True
+        self.SetStopBreaks(check)
+
+  def ClearImageLogs(self, image_name):
+    '''Deletes all app verifier logs for image_name.'''
+    logs = self.manager.Logs(image_name)
+    if logs:
+      while logs.Count:
+        logs.Remove(0)
+
+  def ProcessLog(self, log_path):
+    '''Process the verifier log at log_path.
+
+      Returns: a list of the log entries in the log.
+    '''
+    handler = VerifierSaxHandler()
+    doc = xml.sax.parse(open(log_path, 'rb'), handler)
+
+    return handler.log_entries
+
+  def SaveLog(self, log):
+    '''Saves log to an XML file and returns the path to the file.
+    '''
+    (fd, path) = tempfile.mkstemp('.xml', 'verifier_log')
+    os.close(fd)
+    try:
+      log.SaveAsXML(path, '')
+    except:
+      os.remove(path)
+      return None
+
+    return path
+
+  def ProcessLogs(self, test_name):
+    '''Processes all logs for image test_name.
+
+      Arguments:
+        test_name: the base name of the test executable.
+
+      Returns: A list of LogEntry instances for each error logged.
+    '''
+    logs = self.manager.Logs(test_name)
+    if not logs or not logs.Count:
+      return 0
+
+    errors = []
+    for log in logs:
+      path = self.SaveLog(log)
+      if path:
+        errors.extend(self.ProcessLog(path))
+        os.remove(path)
+
+    return errors
+
+  def RunTestWithVerifier(self, test_path):
+    '''Run a single test under verifier.
+
+      Arguments:
+        test_path: full or relative path to the test to run.
+
+      Returns:
+        A tuple with the test exit code and a list of verifier errors,
+        example:
+          (0, [error1, error2, ...])
+    '''
+    test_name = os.path.basename(test_path)
+
+    # Set up the verifier configuration
+    self.SetImageDefaults(test_name)
+    self.ClearImageLogs(test_name)
+
+    # run the test.
+    exit = subprocess.call(test_path)
+
+    # Clear the verifier settings for the image.
+    self.ResetImage(test_name)
+
+    # And process the logs.
+    errors = self.ProcessLogs(test_name)
+    return (exit, errors)
+
+  def DumpImages_(self):
+    for image in self.manager.Images:
+      print image.Name
+      for check in image.Checks:
+        print '\t', check.Name, check.Enabled
+
+  def DumpLogs_(self, image_name):
+    for l in self.manager.Logs(image_name):
+      print l
+
+
+def RunTestWithVerifier(test_path, break_on_stop = False):
+  '''Runs test_path under app verifier.
+
+    Returns: a tuple of the exit code and list of errors
+  '''
+  runner = AppverifierTestRunner(break_on_stop)
+  return runner.RunTestWithVerifier(test_path)
+
+
+def HasAppVerifier():
+  '''Returns true iff application verifier is installed.'''
+  if VerifierLib:
+    return True
+
+  return False
+
+
+def Main():
+  '''Runs all tests on command line under verifier with stops disabled.'''
+  for test in sys.argv[1:]:
+    (exit_code, errors) = RunTestWithVerifier(test)
+    for error in errors:
+      print error
+
+
+if __name__ == '__main__':
+  Main()