Fix some bugs in resolving expectations for tests.

Review URL: http://codereview.chromium.org/486046

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@34480 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 60 insertions, 53 deletions

diff --git a/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js b/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js
index e9a790d..e2f3ae0 100644
--- a/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js
+++ b/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js
@@ -17,8 +17,10 @@
  */
 function setupExpectationsTest() {
   allExpectations = null;
+  allTests = null;
   expectationsByTest = {};
   resultsByBuilder = {};
+  builders = {};
 }
 
 /**
@@ -27,84 +29,85 @@ function setupExpectationsTest() {
  *
  * @param {string} builder Builder the test is run on.
  * @param {string} test The test name.
- * @param {Array} expectationsArray Array of test expectations. This should be
- *    in the same format as in expectations.json as output by
- *    run_webkit_tests.py and on the buildbots.
- * @param {Object} results Object listing the results for this test on the bot.
- *    This should be in the same format as in expectations.json as output by
- *    run_webkit_tests.py and on the buildbots.
  * @param {string} expectations Sorted string of what the expectations for this
  *    test ought to be for this builder.
  * @param {string} modifiers Sorted string of what the modifiers for this
  *    test ought to be for this builder.
  */
-function runExpectationsTest(builder, test, expectationsArray, results,
-    expectations, modifiers) {
-  // Setup global dashboard state.
-  builders = {};
+function runExpectationsTest(builder, test, expectations, modifiers) {
   builders[builder] = true;
-  expectationsByTest[test] = expectationsArray;
-  resultsByBuilder[builder] = results;
+
+  // Put in some dummy results. processExpectations expects the test to be
+  // there.
+  var tests = {};
+  tests[test] = {'results': [[100, 'F']], 'times': [[100, 0]]};
+  resultsByBuilder[builder] = {'tests': tests};
 
   processExpectations();
   var resultsForTest = createResultsObjectForTest(test, builder);
   populateExpectationsData(resultsForTest);
 
-  assertEquals(resultsForTest.expectations, expectations);
-  assertEquals(resultsForTest.modifiers, modifiers);
+  assertEquals(resultsForTest, resultsForTest.expectations, expectations);
+  assertEquals(resultsForTest, resultsForTest.modifiers, modifiers);
 }
 
-function assertEquals(actual, expected) {
+function assertEquals(resultsForTest, actual, expected) {
   if (expected !== actual) {
-    throw Error('Got: ' + actual + ' expected: ' + expected);
+    throw Error('Builder: ' + resultsForTest.builder + ' test: ' +
+        resultsForTest.test + ' got: ' + actual + ' expected: ' + expected);
   }
 }
 
+function throwError(resultsForTests, actual, expected) {
+}
+
 function testReleaseFail() {
   var builder = 'Webkit';
   var test = 'foo/1.html';
-  var expectatiosnArray = [
+  var expectationsArray = [
     {'modifiers': 'RELEASE', 'expectations': 'FAIL'}
   ];
-  var results = {'tests': {
-    'foo/1.html': {
-      'results': [[100, 'F']],
-      'times': [[100, 0]]
-  }}};
-  runExpectationsTest(builder, test, expectatiosnArray, results, 'FAIL',
-      'RELEASE');
+  expectationsByTest[test] = expectationsArray;
+  runExpectationsTest(builder, test, 'FAIL', 'RELEASE');
 }
 
 function testReleaseFailDebugCrashReleaseBuilder() {
   var builder = 'Webkit';
   var test = 'foo/1.html';
-  var expectatiosnArray = [
+  var expectationsArray = [
     {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
     {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
   ];
-  var results = {'tests': {
-    'foo/1.html': {
-      'results': [[100, 'F']],
-      'times': [[100, 0]]
-  }}};
-  runExpectationsTest(builder, test, expectatiosnArray, results, 'FAIL',
-      'RELEASE');
+  expectationsByTest[test] = expectationsArray;
+  runExpectationsTest(builder, test, 'FAIL', 'RELEASE');
 }
 
 function testReleaseFailDebugCrashDebugBuilder() {
   var builder = 'Webkit(dbg)';
   var test = 'foo/1.html';
-  var expectatiosnArray = [
+  var expectationsArray = [
     {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
     {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
   ];
-  var results = {'tests': {
-    'foo/1.html': {
-      'results': [[100, 'F']],
-      'times': [[100, 0]]
-  }}};
-  runExpectationsTest(builder, test, expectatiosnArray, results,  'CRASH',
-      'DEBUG');
+  expectationsByTest[test] = expectationsArray;
+  runExpectationsTest(builder, test, 'CRASH', 'DEBUG');
+}
+
+function testOverrideJustBuildType() {
+  var test = 'bar/1.html';
+  expectationsByTest['bar'] = [
+    {'modifiers': 'WONTFIX', 'expectations': 'FAIL PASS TIMEOUT'}
+  ];
+  expectationsByTest[test] = [
+    {'modifiers': 'WONTFIX MAC', 'expectations': 'FAIL'},
+    {'modifiers': 'LINUX DEBUG', 'expectations': 'CRASH'},
+  ];
+  runExpectationsTest('Webkit', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
+  runExpectationsTest('Webkit (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
+  runExpectationsTest('Webkit Linux', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
+  runExpectationsTest('Webkit Linux (dbg)(3)', test, 'CRASH', 'LINUX DEBUG');
+  runExpectationsTest('Webkit Mac10.5', test, 'FAIL', 'WONTFIX MAC');
+  runExpectationsTest('Webkit Mac10.5 (dbg)(3)', test, 'FAIL', 'WONTFIX MAC');
 }
 
 function runTests() {
diff --git a/webkit/tools/layout_tests/flakiness_dashboard.html b/webkit/tools/layout_tests/flakiness_dashboard.html
index a385808..0dc65d5 100644
--- a/webkit/tools/layout_tests/flakiness_dashboard.html
+++ b/webkit/tools/layout_tests/flakiness_dashboard.html
@@ -292,13 +292,6 @@
     if (currentState.useTestData)
       return;
 
-    if (!currentState.tests && !('builder' in currentState)) {
-      for (var builder in builders) {
-        currentState.builder = builder;
-        break;
-      }
-    }
-
     document.body.innerHTML = '<div id="loading-ui">LOADING...</div>';
 
     if (currentState.tests) {
@@ -399,7 +392,8 @@
     showSlow: false,
     showSkipped: false,
     expectationsUpdate: false,
-    updateIndex: 0
+    updateIndex: 0,
+    builder: 'Webkit'
   };
 
   //////////////////////////////////////////////////////////////////////////////
@@ -705,10 +699,17 @@
       if (platformObject[buildType])
         return platformObject[buildType]
 
-      if (testObject[ALL] && testObject[ALL][buildType])
-        return testObject[ALL][buildType]
+      if (platformObject[ALL])
+        return platformObject[ALL];
+
+      if (testObject[ALL]) {
+        var allPlatformObject = testObject[ALL];
+        if (allPlatformObject[buildType])
+          return allPlatformObject[buildType]
 
-      return platformObject[ALL];
+        if (allPlatformObject[ALL])
+          return allPlatformObject[ALL]
+      }
     }
     return null;
   }
@@ -1528,7 +1529,9 @@
    */
   function showUpdateInfoForTest(testsNeedingUpdate, keys) {
     var test = keys[currentState.updateIndex];
-    document.body.innerHTML = '<b>' + test + '</b><br><br>';
+    document.body.innerHTML = '<b>' + test + '</b>' +
+        '<div style="float:right">' + (currentState.updateIndex + 1) + ' of ' +
+        + keys.length + ' tests</div><br><br>';
 
     var buttonRegion = document.createElement('div');
     var includeBtn = document.createElement('input');
@@ -1558,6 +1561,7 @@
     var confirmStr = JSON.stringify(testsNeedingUpdate[test], null, 4);
 
     var pre = document.createElement('pre');
+    pre.style.marginTop = 0;
     pre.innerHTML = confirmStr;
     document.body.appendChild(pre);
     var div = document.createElement('div');
@@ -1610,7 +1614,7 @@
       var tracURL = TEST_URL_BASE_PATH + test
       html += getLinkHTMLToOpenWindow(tracURL, tracURL) +
           '<div><b>If a builder is not listed, that means the builder does ' +
-          'run that test or all runs of the test passed.</b></div>';
+          'not run that test or all runs of the test passed.</b></div>';
 
       for (var j = 0; j < testResults.length; j++) {
         html += getHTMLForSingleTestRow(testResults[j], true);