Update shard-all-tests.sh to save the raw valgrind output files,

and make it take commandline arguments so people don't have to
edit it.  Have it build the tests, too.

Review URL: http://codereview.chromium.org/203036

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@26060 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 74 insertions, 30 deletions

diff --git a/tools/valgrind/shard-all-tests.sh b/tools/valgrind/shard-all-tests.sh
index 9c66294..c496fb9 100644
--- a/tools/valgrind/shard-all-tests.sh
+++ b/tools/valgrind/shard-all-tests.sh
@@ -4,7 +4,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# Script to run all tests under tools/valgrind/chrome_tests.sh
+# Script to run tests under tools/valgrind/chrome_tests.sh
 # in a loop looking for rare/flaky valgrind warnings, and
 # generate suppressions for them, to be later filed as bugs
 # and added to our suppressions file.
@@ -21,42 +21,83 @@ then
   exit 1
 fi
 
+if test "$1" = ""
+then
+  echo "Usage: shard-all-tests.sh [BUILDTYPE=Release] target [target ...]"
+  echo "Example: shard-all-tests.sh ui_tests"
+  exit 1
+fi
+
 set -x
 set -e
 
 # Regexp to match any valgrind error
 PATTERN="ERROR SUMMARY: [^0]|are definitely|uninitialised|Unhandled exception|\
 Invalid read|Invalid write|Invalid free|Source and desti|Mismatched free|\
-unaddressable byte|vex x86"
-
-# List of test executables to build and run
-# You might want to trim this list down to just one if you have an idea
-# which test has interesting problems (e.g. ui_tests)
-TESTS="base_unittests googleurl_unittests ipc_tests media_unittests \
-       net_unittests printing_unittests startup_tests \
-       test_shell_tests ui_tests unit_tests"
-
-TESTS_BUILDABLE=`echo $TESTS | tr ' ' '\012' | grep -v layout_tests`
-
-# Build the tests (if we know how)
-OS=`uname`
-case $OS in
-Linux)
-  # Lame way to autodetect whether 'make' or 'hammer' is in use
-  if test -d out
-  then
-    make -j4 chrome test_shell $TESTS_BUILDABLE
-  else
-    hammer chrome test_shell $TESTS_BUILDABLE
-  fi
-  ;;
-*) echo "don't know how to build on os $OS"
-  ;;
+unaddressable byte|vex x86|impossible|Assertion|INTERNAL ERROR|finish writing|OUCH"
+
+BUILDTYPE=Debug
+case "$1" in
+  BUILDTYPE=Debug) BUILDTYPE=Debug ; shift ;;
+  BUILDTYPE=Release) BUILDTYPE=Release ; shift ;;
+  BUILDTYPE=*) echo "unknown build type $1"; exit 1;;
+  *) ;;
 esac
+TESTS="$@"
+
+what_to_build() {
+  echo $TESTS | tr ' ' '\012' | grep -v layout_tests || true
+  echo $TESTS | grep -q layout_tests && echo test_shell || true
+  echo $TESTS | grep -q ui_tests && echo chrome || true
+}
 
-# Divide each test suite up into 50 shards, as first step
+# Wrap xcodebuild to take same arguments as our make, more or less
+xcodemake() {
+  for target in $*
+  do
+    case $target in
+    chrome)         xcodebuild -configuration $BUILDTYPE -project chrome/chrome.xcodeproj -target chrome ;;
+    ui_tests)       xcodebuild -configuration $BUILDTYPE -project chrome/chrome.xcodeproj -target ui_tests ;;
+    base_unittests) xcodebuild -configuration $BUILDTYPE -project base/base.xcodeproj     -target base_unittests ;;
+    net_unittests)  xcodebuild -configuration $BUILDTYPE -project net/net.xcodeproj       -target net_unittests ;;
+    *) echo "dunno how to build $target yet"; exit 1 ;;
+    esac
+  done
+}
+
+build_tests() {
+  buildtype=$1
+  shift
+
+  OS=`uname`
+  case $OS in
+  Linux)
+    # Lame way to autodetect whether 'make' or 'hammer' is in use
+    if test -d out
+    then
+      make -j4 BUILDTYPE=$1 $@
+    else
+      # fixme: obey buildtype
+      hammer $@
+    fi
+    ;;
+  Darwin)
+    xcodemake $@
+    ;;
+  *) echo "don't know how to build on os $OS"
+    ;;
+  esac
+}
+
+TESTS_BUILDABLE=`what_to_build`
+echo building $TESTS_BUILDABLE
+build_tests $BUILDTYPE $TESTS_BUILDABLE
+
+# Divide each test suite up into 100 shards, as first step
 # in tracking down exact source of errors.
-export GTEST_TOTAL_SHARDS=50
+export GTEST_TOTAL_SHARDS=100
+
+rm -rf *.vlog *.vtmp || true
 
 iter=0
 while test $iter -lt 1000
@@ -67,7 +108,8 @@ do
     while test $GTEST_SHARD_INDEX -lt $GTEST_TOTAL_SHARDS
     do
       i=$GTEST_SHARD_INDEX
-      sh tools/valgrind/chrome_tests.sh -t ${testname} > ${testname}_$i.vlog 2>&1
+      sh tools/valgrind/chrome_tests.sh -b xcodebuild/$BUILDTYPE -t ${testname} --tool_flags="--nocleanup_on_exit" > ${testname}_$i.vlog 2>&1 || true
+      mv valgrind.tmp ${testname}_$i.vtmp
       GTEST_SHARD_INDEX=`expr $GTEST_SHARD_INDEX + 1`
     done
   done
@@ -78,8 +120,10 @@ do
   then
     mkdir -p shard-results/$iter
     mv `egrep -l "$PATTERN" *.vlog` shard-results/$iter
+    # ideally we'd only save the .vtmp's corresponding to the .vlogs we saved
+    mv *.vtmp shard-results/$iter
   fi
 
-  rm *.vlog
+  rm -rf *.vlog *.vtmp || true
   iter=`expr $iter + 1`
 done