#!/usr/bin/python # Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # valgrind_test.py '''Runs an exe through Valgrind and puts the intermediate files in a directory. ''' import datetime import glob import logging import optparse import os import re import shutil import stat import sys import tempfile import time import common import memcheck_analyze import tsan_analyze import logging_utils class ValgrindTool(object): """Abstract class for running Valgrind. Always subclass this and implement ValgrindCommand() with platform specific stuff. """ TMP_DIR = "valgrind.tmp" def __init__(self): # If we have a valgrind.tmp directory, we failed to cleanup last time. if os.path.exists(self.TMP_DIR): shutil.rmtree(self.TMP_DIR) os.mkdir(self.TMP_DIR) def UseXML(self): # Override if tool prefers nonxml output return True def SelfContained(self): # Returns true iff the tool is distibuted as a self-contained # .sh script (e.g. ThreadSanitizer) return False def ToolName(self): raise RuntimeError, "This method should be implemented " \ "in the tool-specific subclass" def CreateOptionParser(self): self._parser = optparse.OptionParser("usage: %prog [options] ") self._parser.add_option("-t", "--timeout", dest="timeout", metavar="TIMEOUT", default=10000, help="timeout in seconds for the run (default 10000)") self._parser.add_option("", "--source_dir", help="path to top of source tree for this build" "(used to normalize source paths in baseline)") self._parser.add_option("", "--gtest_filter", default="", help="which test case to run") self._parser.add_option("", "--gtest_repeat", help="how many times to run each test") self._parser.add_option("", "--gtest_print_time", action="store_true", default=False, help="show how long each test takes") self._parser.add_option("", "--indirect", action="store_true", default=False, help="set BROWSER_WRAPPER rather than " "running valgrind directly") self._parser.add_option("-v", "--verbose", action="store_true", default=False, help="verbose output - enable debug log messages") self._parser.add_option("", "--trace_children", action="store_true", default=False, help="also trace child processes") self._parser.add_option("", "--num-callers", dest="num_callers", default=30, help="number of callers to show in stack traces") self._parser.add_option("", "--nocleanup_on_exit", action="store_true", default=False, help="don't delete directory with logs on exit") self._parser.add_option("", "--ignore_exit_code", action="store_true", default=False, help="ignore exit code of the test " "(e.g. test failures)") self.ExtendOptionParser(self._parser) self._parser.description = __doc__ def ExtendOptionParser(self, parser): parser.add_option("", "--generate_dsym", action="store_true", default=False, help="Generate .dSYM file on Mac if needed. Slow!") def ParseArgv(self, args): self.CreateOptionParser() # self._tool_flags will store those tool flags which we don't parse # manually in this script. self._tool_flags = [] known_args = [] """ We assume that the first argument not starting with "-" is a program name and all the following flags should be passed to the program. TODO(timurrrr): customize optparse instead """ while len(args) > 0 and args[0][:1] == "-": arg = args[0] if (arg == "--"): break if self._parser.has_option(arg.split("=")[0]): known_args += [arg] else: self._tool_flags += [arg] args = args[1:] if len(args) > 0: known_args += args self._options, self._args = self._parser.parse_args(known_args) self._timeout = int(self._options.timeout) self._num_callers = int(self._options.num_callers) self._suppressions = self._options.suppressions self._source_dir = self._options.source_dir self._nocleanup_on_exit = self._options.nocleanup_on_exit self._ignore_exit_code = self._options.ignore_exit_code if self._options.gtest_filter != "": self._args.append("--gtest_filter=%s" % self._options.gtest_filter) if self._options.gtest_repeat: self._args.append("--gtest_repeat=%s" % self._options.gtest_repeat) if self._options.gtest_print_time: self._args.append("--gtest_print_time") return True def Setup(self, args): return self.ParseArgv(args) def PrepareForTest(self): if common.IsWine(): self.PrepareForTestWine() elif common.IsMac(): self.PrepareForTestMac() def PrepareForTestMac(self): """Runs dsymutil if needed. Valgrind for Mac OS X requires that debugging information be in a .dSYM bundle generated by dsymutil. It is not currently able to chase DWARF data into .o files like gdb does, so executables without .dSYM bundles or with the Chromium-specific "fake_dsym" bundles generated by build/mac/strip_save_dsym won't give source file and line number information in valgrind. This function will run dsymutil if the .dSYM bundle is missing or if it looks like a fake_dsym. A non-fake dsym that already exists is assumed to be up-to-date. """ test_command = self._args[0] dsym_bundle = self._args[0] + '.dSYM' dsym_file = os.path.join(dsym_bundle, 'Contents', 'Resources', 'DWARF', os.path.basename(test_command)) dsym_info_plist = os.path.join(dsym_bundle, 'Contents', 'Info.plist') needs_dsymutil = True saved_test_command = None if os.path.exists(dsym_file) and os.path.exists(dsym_info_plist): # Look for the special fake_dsym tag in dsym_info_plist. dsym_info_plist_contents = open(dsym_info_plist).read() if not re.search('^\s*fake_dsym$', dsym_info_plist_contents, re.MULTILINE): # fake_dsym is not set, this is a real .dSYM bundle produced by # dsymutil. dsymutil does not need to be run again. needs_dsymutil = False else: # fake_dsym is set. dsym_file is a copy of the original test_command # before it was stripped. Copy it back to test_command so that # dsymutil has unstripped input to work with. Move the stripped # test_command out of the way, it will be restored when this is # done. saved_test_command = test_command + '.stripped' os.rename(test_command, saved_test_command) shutil.copyfile(dsym_file, test_command) shutil.copymode(saved_test_command, test_command) if needs_dsymutil: if self._options.generate_dsym: # Remove the .dSYM bundle if it exists. shutil.rmtree(dsym_bundle, True) dsymutil_command = ['dsymutil', test_command] # dsymutil is crazy slow. Ideally we'd have a timeout here, # but common.RunSubprocess' timeout is only checked # after each line of output; dsymutil is silent # until the end, and is then killed, which is silly. common.RunSubprocess(dsymutil_command) if saved_test_command: os.rename(saved_test_command, test_command) else: logging.info("No real .dSYM for test_command. Line numbers will " "not be shown. Either tell xcode to generate .dSYM " "file, or use --generate_dsym option to this tool.") def PrepareForTestWine(self): """Set up the Wine environment. We need to run some sanity checks, set up a Wine prefix, and make sure wineserver is running by starting a dummy win32 program. """ if not os.path.exists('/usr/share/ca-certificates/root_ca_cert.crt'): logging.warning('WARNING: SSL certificate missing! SSL tests will fail.') logging.warning('You need to run:') logging.warning('sudo cp src/net/data/ssl/certificates/root_ca_cert.crt ' '/usr/share/ca-certificates/') logging.warning('sudo vi /etc/ca-certificates.conf') logging.warning(' (and add the line root_ca_cert.crt)') logging.warning('sudo update-ca-certificates') # Shutdown the Wine server in case the last run got interrupted. common.RunSubprocess([os.environ.get('WINESERVER'), '-k']) # Yes, this can be dangerous if $WINEPREFIX is set incorrectly. shutil.rmtree(os.environ.get('WINEPREFIX'), ignore_errors=True) winetricks = os.path.join(self._source_dir, 'tools', 'valgrind', 'wine_memcheck', 'winetricks') common.RunSubprocess(['sh', winetricks, 'nocrashdialog', 'corefonts', 'gecko']) time.sleep(1) # Start a dummy program like winemine so Valgrind won't run memcheck on # the wineserver startup routine when it launches the test binary, which # is slow and not interesting to us. common.RunSubprocessInBackground([os.environ.get('WINE'), 'winemine']) return def ValgrindCommand(self): """Get the valgrind command to run.""" # Note that self._args begins with the exe to be run. tool_name = self.ToolName() # Construct the valgrind command. if self.SelfContained(): proc = ["valgrind-%s.sh" % tool_name] else: proc = ["valgrind", "--tool=%s" % tool_name] proc += ["--num-callers=%i" % self._num_callers] if self._options.trace_children: proc += ["--trace-children=yes"] proc += self.ToolSpecificFlags() proc += self._tool_flags suppression_count = 0 for suppression_file in self._suppressions: if os.path.exists(suppression_file): suppression_count += 1 proc += ["--suppressions=%s" % suppression_file] if not suppression_count: logging.warning("WARNING: NOT USING SUPPRESSIONS!") logfilename = self.TMP_DIR + ("/%s." % tool_name) + "%p" if self.UseXML(): if os.system("valgrind --help | grep -q xml-file") == 0: proc += ["--xml=yes", "--xml-file=" + logfilename] else: # TODO(dank): remove once valgrind-3.5 is deployed everywhere proc += ["--xml=yes", "--log-file=" + logfilename] else: proc += ["--log-file=" + logfilename] # The Valgrind command is constructed. if self._options.indirect: self.CreateBrowserWrapper(" ".join(proc)) proc = [] proc += self._args return proc def ToolSpecificFlags(self): return [] def Execute(self): ''' Execute the app to be tested after successful instrumentation. Full execution command-line provided by subclassers via proc.''' logging.info("starting execution...") proc = self.ValgrindCommand() os.putenv("G_SLICE", "always-malloc") logging.info("export G_SLICE=always-malloc") os.putenv("NSS_DISABLE_ARENA_FREE_LIST", "1") logging.info("export NSS_DISABLE_ARENA_FREE_LIST=1") os.putenv("GTEST_DEATH_TEST_USE_FORK", "1") logging.info("export GTEST_DEATH_TEST_USE_FORK=1") if common.IsWine(): os.putenv("CHROME_ALLOCATOR", "winheap") logging.info("export CHROME_ALLOCATOR=winheap") return common.RunSubprocess(proc, self._timeout) def Analyze(self, check_sanity=False): raise RuntimeError, "This method should be implemented " \ "in the tool-specific subclass" def Cleanup(self): # Right now, we can cleanup by deleting our temporary directory. Other # cleanup is still a TODO? if not self._nocleanup_on_exit: shutil.rmtree(self.TMP_DIR, ignore_errors=True) if common.IsWine(): # Shutdown the Wine server. common.RunSubprocess([os.environ.get('WINESERVER'), '-k']) return True def RunTestsAndAnalyze(self, check_sanity): self.PrepareForTest() exec_retcode = self.Execute() analyze_retcode = self.Analyze(check_sanity) if analyze_retcode: logging.error("Analyze failed.") return analyze_retcode if exec_retcode: if self._ignore_exit_code: logging.info("Test execution failed, but the exit code is ignored.") else: logging.error("Test execution failed.") return exec_retcode else: logging.info("Test execution completed successfully.") if not analyze_retcode: logging.info("Analysis completed successfully.") return 0 def CreateBrowserWrapper(self, command): """The program being run invokes Python or something else that can't stand to be valgrinded, and also invokes the Chrome browser. Set an environment variable to tell the program to prefix the Chrome commandline with a magic wrapper. Build the magic wrapper here. """ (fd, indirect_fname) = tempfile.mkstemp(dir=self.TMP_DIR, prefix="browser_wrapper.", text=True) f = os.fdopen(fd, "w") f.write("#!/bin/sh\n") f.write(command) f.write(' "$@"\n') f.close() os.chmod(indirect_fname, stat.S_IRUSR|stat.S_IXUSR) os.putenv("BROWSER_WRAPPER", indirect_fname) logging.info('export BROWSER_WRAPPER=' + indirect_fname) def Main(self, args, check_sanity): '''Call this to run through the whole process: Setup, Execute, Analyze''' start = datetime.datetime.now() retcode = -1 if self.Setup(args): retcode = self.RunTestsAndAnalyze(check_sanity) self.Cleanup() else: logging.error("Setup failed") end = datetime.datetime.now() seconds = (end - start).seconds hours = seconds / 3600 seconds = seconds % 3600 minutes = seconds / 60 seconds = seconds % 60 logging.info("elapsed time: %02d:%02d:%02d" % (hours, minutes, seconds)) return retcode # TODO(timurrrr): Split into a separate file. class Memcheck(ValgrindTool): """Memcheck""" def __init__(self): ValgrindTool.__init__(self) def ToolName(self): return "memcheck" def ExtendOptionParser(self, parser): ValgrindTool.ExtendOptionParser(self, parser) parser.add_option("", "--suppressions", default=[], action="append", help="path to a valgrind suppression file") parser.add_option("", "--show_all_leaks", action="store_true", default=False, help="also show less blatant leaks") parser.add_option("", "--track_origins", action="store_true", default=False, help="Show whence uninitialized bytes came. 30% slower.") def ToolSpecificFlags(self): ret = ["--leak-check=full", "--gen-suppressions=all", "--demangle=no"] if self._options.show_all_leaks: ret += ["--show-reachable=yes"] else: ret += ["--show-possible=no"] if self._options.track_origins: ret += ["--track-origins=yes"] return ret def Analyze(self, check_sanity=False): # Glob all the files in the "valgrind.tmp" directory filenames = glob.glob(self.TMP_DIR + "/memcheck.*") use_gdb = common.IsMac() analyzer = memcheck_analyze.MemcheckAnalyze(self._source_dir, filenames, self._options.show_all_leaks, use_gdb=use_gdb) ret = analyzer.Report(check_sanity) if ret != 0: logging.info("Please see http://dev.chromium.org/developers/how-tos/" "using-valgrind for the info on Memcheck/Valgrind") return ret class ThreadSanitizer(ValgrindTool): """ThreadSanitizer""" def __init__(self): ValgrindTool.__init__(self) def ToolName(self): return "tsan" def UseXML(self): return False def SelfContained(self): return True def ExtendOptionParser(self, parser): ValgrindTool.ExtendOptionParser(self, parser) parser.add_option("", "--suppressions", default=[], action="append", help="path to a valgrind suppression file") parser.add_option("", "--pure-happens-before", default="yes", dest="pure_happens_before", help="Less false reports, more missed races") parser.add_option("", "--ignore-in-dtor", default="no", dest="ignore_in_dtor", help="Ignore data races inside destructors") parser.add_option("", "--announce-threads", default="yes", dest="announce_threads", help="Show the the stack traces of thread creation") def EvalBoolFlag(self, flag_value): if (flag_value in ["1", "true", "yes"]): return True elif (flag_value in ["0", "false", "no"]): return False raise RuntimeError, "Can't parse flag value (%s)" % flag_value def ToolSpecificFlags(self): ret = [] ignore_files = ["ignores.txt"] for platform_suffix in common.PlatformNames(): ignore_files.append("ignores_%s.txt" % platform_suffix) for ignore_file in ignore_files: fullname = os.path.join(self._source_dir, "tools", "valgrind", "tsan", ignore_file) if os.path.exists(fullname): ret += ["--ignore=%s" % fullname] # The -v flag is needed for printing the list of used suppressions. ret += ["-v"] # This should shorten filepaths for local builds. ret += ["--file-prefix-to-cut=%s/" % self._source_dir] # This should shorten filepaths on bots. ret += ["--file-prefix-to-cut=build/src/"] # This should shorten filepaths for functions intercepted in TSan. ret += ["--file-prefix-to-cut=scripts/tsan/tsan/"] if self.EvalBoolFlag(self._options.pure_happens_before): ret += ["--pure-happens-before=yes"] # "no" is the default value for TSAN if not self.EvalBoolFlag(self._options.ignore_in_dtor): ret += ["--ignore-in-dtor=no"] # "yes" is the default value for TSAN if self.EvalBoolFlag(self._options.announce_threads): ret += ["--announce-threads"] # --show-pc flag is needed for parsing the error logs on Darwin. if platform_suffix == 'mac': ret += ["--show-pc=yes"] # Don't show googletest frames in stacks. ret += ["--cut_stack_below=testing*Test*Run*"] return ret def Analyze(self, check_sanity=False): filenames = glob.glob(self.TMP_DIR + "/tsan.*") use_gdb = common.IsMac() analyzer = tsan_analyze.TsanAnalyze(self._source_dir, filenames, use_gdb=use_gdb) ret = analyzer.Report(check_sanity) if ret != 0: logging.info("Please see http://dev.chromium.org/developers/how-tos/" "using-valgrind/threadsanitizer for the info on " "ThreadSanitizer") return ret class ToolFactory: def Create(self, tool_name): if tool_name == "memcheck" and not common.IsWine(): return Memcheck() if tool_name == "wine_memcheck" and common.IsWine(): return Memcheck() if tool_name == "tsan": if common.IsWindows(): logging.info("WARNING: ThreadSanitizer Windows support is experimental.") return ThreadSanitizer() try: platform_name = common.PlatformNames()[0] except common.NotImplementedError: platform_name = sys.platform + "(Unknown)" raise RuntimeError, "Unknown tool (tool=%s, platform=%s)" % (tool_name, platform_name) def RunTool(argv, module): # TODO(timurrrr): customize optparse instead tool_name = "memcheck" args = argv[1:] for arg in args: if arg.startswith("--tool="): tool_name = arg[7:] args.remove(arg) break tool = ToolFactory().Create(tool_name) MODULES_TO_SANITY_CHECK = ["base"] check_sanity = module in MODULES_TO_SANITY_CHECK return tool.Main(args, check_sanity) if __name__ == "__main__": if sys.argv.count("-v") > 0 or sys.argv.count("--verbose") > 0: logging_utils.config_root(logging.DEBUG) else: logging_utils.config_root() # TODO(timurrrr): valgrind tools may use -v/--verbose as well ret = RunTool(sys.argv) sys.exit(ret)