Added sanity checks for base_unittests to Memcheck and ThreadSanitizer test runners.

Note that the sanity checks may fail if the tests are ran with the sanity tests excluded (e.g. local test runs). I suppose this shouldn't be a problem, because the exit code matters only on buildbots.
Review URL: http://codereview.chromium.org/598013

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@38612 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 44 insertions, 16 deletions

diff --git a/tools/valgrind/chrome_tests.py b/tools/valgrind/chrome_tests.py
index c94aa77..eb46cd1 100755
--- a/tools/valgrind/chrome_tests.py
+++ b/tools/valgrind/chrome_tests.py
@@ -221,7 +221,7 @@ class ChromeTests:
     # Sets LD_LIBRARY_PATH to the build folder so external libraries can be
     # loaded.
     os.putenv("LD_LIBRARY_PATH", self._options.build_dir)
-    return valgrind_test.RunTool(cmd)
+    return valgrind_test.RunTool(cmd, module)
 
   def TestBase(self):
     return self.SimpleTest("base", "base_unittests")
diff --git a/tools/valgrind/memcheck_analyze.py b/tools/valgrind/memcheck_analyze.py
index af5d09d..c70df99 100755
--- a/tools/valgrind/memcheck_analyze.py
+++ b/tools/valgrind/memcheck_analyze.py
@@ -311,6 +311,7 @@ class MemcheckAnalyze:
   ''' Given a set of Valgrind XML files, parse all the errors out of them,
   unique them and output the results.'''
 
+  SANITY_TEST_SUPPRESSION = "Memcheck sanity test"
   def __init__(self, source_dir, files, show_all_leaks=False, use_gdb=False):
     '''Reads in a set of files.
 
@@ -429,19 +430,23 @@ class MemcheckAnalyze:
         logging.warn("Last 20 lines of %s :" % file)
         os.system("tail -n 20 '%s' 1>&2" % file)
 
-  def Report(self):
+  def Report(self, check_sanity=False):
     if self._parse_failed:
       logging.error("FAIL! Couldn't parse Valgrind output file")
       return -2
 
+    is_sane = False
     print "-----------------------------------------------------"
     print "Suppressions used:"
     print "  count name"
     for item in sorted(self._suppcounts.items(), key=lambda (k,v): (v,k)):
       print "%7s %s" % (item[1], item[0])
+      if item[0].startswith(MemcheckAnalyze.SANITY_TEST_SUPPRESSION):
+        is_sane = True
     print "-----------------------------------------------------"
     sys.stdout.flush()
 
+    retcode = 0
     if self._errors:
       logging.error("FAIL! There were %s errors: " % len(self._errors))
 
@@ -452,7 +457,15 @@ class MemcheckAnalyze:
       for error in self._errors:
         logging.error(error)
 
-      return -1
+      retcode = -1
+
+    # Report tool's insanity even if there were errors.
+    if check_sanity and not is_sane:
+      logging.error("FAIL! Sanity check failed!")
+      retcode = -3
+
+    if retcode != 0:
+      return retcode
 
     logging.info("PASS! No errors found!")
     return 0
diff --git a/tools/valgrind/tsan_analyze.py b/tools/valgrind/tsan_analyze.py
index dcb5060..9ddcdb4 100644
--- a/tools/valgrind/tsan_analyze.py
+++ b/tools/valgrind/tsan_analyze.py
@@ -47,6 +47,7 @@ class TsanAnalyze:
   THREAD_CREATION_STR = ("INFO: T.* "
       "(has been created by T.* at this point|is program's main thread)")
 
+  SANITY_TEST_SUPPRESSION = "ThreadSanitizer sanity test"
   def __init__(self, source_dir, files, use_gdb=False):
     '''Reads in a set of files.
 
@@ -131,15 +132,19 @@ class TsanAnalyze:
           self.used_suppressions[supp_name] = count
     self.cur_fd_.close()
 
-  def Report(self):
+  def Report(self, check_sanity=False):
+    is_sane = False
     print "-----------------------------------------------------"
     print "Suppressions used:"
     print "  count name"
     for item in sorted(self.used_suppressions.items(), key=lambda (k,v): (v,k)):
       print "%7s %s" % (item[1], item[0])
+      if item[0].startswith(TsanAnalyze.SANITY_TEST_SUPPRESSION):
+        is_sane = True
     print "-----------------------------------------------------"
     sys.stdout.flush()
 
+    retcode = 0
     if len(self.races) > 0:
       logging.error("Found %i race reports" % len(self.races))
       for report_list in self.races:
@@ -147,7 +152,15 @@ class TsanAnalyze:
         for line in report_list:
           report += str(line)
         logging.error('\n' + report)
-      return -1
+      retcode = -1
+
+    # Report tool's insanity even if there were errors.
+    if check_sanity and not is_sane:
+      logging.error("FAIL! Sanity check failed!")
+      retcode = -3
+
+    if retcode != 0:
+      return retcode
     logging.info("PASS: No race reports found")
     return 0
 
diff --git a/tools/valgrind/valgrind_test.py b/tools/valgrind/valgrind_test.py
index 0a736a1..ef9caac 100755
--- a/tools/valgrind/valgrind_test.py
+++ b/tools/valgrind/valgrind_test.py
@@ -303,7 +303,7 @@ class ValgrindTool(object):
 
     return common.RunSubprocess(proc, self._timeout)
 
-  def Analyze(self):
+  def Analyze(self, check_sanity=False):
     raise RuntimeError, "This method should be implemented " \
                         "in the tool-specific subclass"
 
@@ -318,10 +318,10 @@ class ValgrindTool(object):
       common.RunSubprocess([os.environ.get('WINESERVER'), '-k'])
     return True
 
-  def RunTestsAndAnalyze(self):
+  def RunTestsAndAnalyze(self, check_sanity=False):
     self.PrepareForTest()
     exec_retcode = self.Execute()
-    analyze_retcode = self.Analyze()
+    analyze_retcode = self.Analyze(check_sanity)
 
     if analyze_retcode:
       logging.error("Analyze failed.")
@@ -351,12 +351,12 @@ class ValgrindTool(object):
     os.putenv("BROWSER_WRAPPER", indirect_fname)
     logging.info('export BROWSER_WRAPPER=' + indirect_fname)
 
-  def Main(self, args):
+  def Main(self, args, check_sanity=False):
     '''Call this to run through the whole process: Setup, Execute, Analyze'''
     start = datetime.datetime.now()
     retcode = -1
     if self.Setup(args):
-      retcode = self.RunTestsAndAnalyze()
+      retcode = self.RunTestsAndAnalyze(check_sanity)
       self.Cleanup()
     else:
       logging.error("Setup failed")
@@ -404,7 +404,7 @@ class Memcheck(ValgrindTool):
 
     return ret
 
-  def Analyze(self):
+  def Analyze(self, check_sanity=False):
     # Glob all the files in the "valgrind.tmp" directory
     filenames = glob.glob(self.TMP_DIR + "/memcheck.*")
 
@@ -412,7 +412,7 @@ class Memcheck(ValgrindTool):
     analyzer = memcheck_analyze.MemcheckAnalyze(self._source_dir, filenames,
                                                 self._options.show_all_leaks,
                                                 use_gdb=use_gdb)
-    ret = analyzer.Report()
+    ret = analyzer.Report(check_sanity)
     if ret != 0:
       logging.info("Please see http://dev.chromium.org/developers/how-tos/"
                    "using-valgrind for the info on Memcheck/Valgrind")
@@ -484,12 +484,12 @@ class ThreadSanitizer(ValgrindTool):
 
     return ret
 
-  def Analyze(self):
+  def Analyze(self, check_sanity=False):
     filenames = glob.glob(self.TMP_DIR + "/tsan.*")
     use_gdb = common.IsMac()
     analyzer = tsan_analyze.TsanAnalyze(self._source_dir, filenames,
                                         use_gdb=use_gdb)
-    ret = analyzer.Report()
+    ret = analyzer.Report(check_sanity)
     if ret != 0:
       logging.info("Please see http://dev.chromium.org/developers/how-tos/"
                    "using-valgrind/threadsanitizer for the info on "
@@ -516,7 +516,7 @@ class ToolFactory:
     raise RuntimeError, "Unknown tool (tool=%s, platform=%s)" % (tool_name,
                                                                  platform_name)
 
-def RunTool(argv):
+def RunTool(argv, module):
   # TODO(timurrrr): customize optparse instead
   tool_name = "memcheck"
   args = argv[1:]
@@ -527,7 +527,9 @@ def RunTool(argv):
       break
 
   tool = ToolFactory().Create(tool_name)
-  return tool.Main(args)
+  MODULES_TO_SANITY_CHECK = ["base"]
+  check_sanity = module in MODULES_TO_SANITY_CHECK
+  return tool.Main(args, check_sanity)
 
 if __name__ == "__main__":
   if sys.argv.count("-v") > 0 or sys.argv.count("--verbose") > 0: