Add tests to the flakiness dashboard. This are really poor man's tests,

but they get the job done for testing some of the trickier bits to the
dashboard that keep regressing.
Review URL: http://codereview.chromium.org/437082

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@33418 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 159 insertions, 8 deletions

diff --git a/webkit/tools/layout_tests/dashboards/dashboard_base.js b/webkit/tools/layout_tests/dashboards/dashboard_base.js
index a8618de..7d2cf31 100644
--- a/webkit/tools/layout_tests/dashboards/dashboard_base.js
+++ b/webkit/tools/layout_tests/dashboards/dashboard_base.js
@@ -85,8 +85,9 @@ function handleValidHashParameterWrapper(key, value) {
 
       return true;
 
-    case 'useWebKitCanary':
+    case 'useTestData':
     case 'useV8Canary':
+    case 'useWebKitCanary':
       currentState[key] = value == 'true';
       return true;
 
@@ -271,6 +272,11 @@ function ADD_EXPECTATIONS(expectations) {
 function appendJSONScriptElements() {
   parseParameters();
 
+  if (currentState.useTestData) {
+    appendScript('dashboards/flakiness_dashboard_tests.js');
+    return;
+  }
+
   for (var builderName in builders) {
     appendScript(getPathToBuilderResultsFile(builderName) + 'results.json');
   }
diff --git a/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js b/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js
new file mode 100644
index 0000000..e9a790d
--- /dev/null
+++ b/webkit/tools/layout_tests/dashboards/flakiness_dashboard_tests.js
@@ -0,0 +1,136 @@
+/**
+ * @fileoverview Poor-man's unittests for some of the trickier logic bits of
+ * the flakiness dashboard.
+ *
+ * Currently only tests processExpectations and populateExpectationsData.
+ * A test just consists of calling runExpectationsTest with the appropriate
+ * arguments.
+ */
+
+// TODO(ojan): Add more test cases.
+// TODO(ojan): Add tests for processMissingAndExtraExpectations
+
+/**
+ * Clears out the global objects modified or used by processExpectations and
+ * populateExpectationsData. A bit gross since it's digging into implementation
+ * details, but it's good enough for now.
+ */
+function setupExpectationsTest() {
+  allExpectations = null;
+  expectationsByTest = {};
+  resultsByBuilder = {};
+}
+
+/**
+ * Processes the expectations for a test and asserts that the final expectations
+ * and modifiers we apply to a test match what we expect.
+ *
+ * @param {string} builder Builder the test is run on.
+ * @param {string} test The test name.
+ * @param {Array} expectationsArray Array of test expectations. This should be
+ *    in the same format as in expectations.json as output by
+ *    run_webkit_tests.py and on the buildbots.
+ * @param {Object} results Object listing the results for this test on the bot.
+ *    This should be in the same format as in expectations.json as output by
+ *    run_webkit_tests.py and on the buildbots.
+ * @param {string} expectations Sorted string of what the expectations for this
+ *    test ought to be for this builder.
+ * @param {string} modifiers Sorted string of what the modifiers for this
+ *    test ought to be for this builder.
+ */
+function runExpectationsTest(builder, test, expectationsArray, results,
+    expectations, modifiers) {
+  // Setup global dashboard state.
+  builders = {};
+  builders[builder] = true;
+  expectationsByTest[test] = expectationsArray;
+  resultsByBuilder[builder] = results;
+
+  processExpectations();
+  var resultsForTest = createResultsObjectForTest(test, builder);
+  populateExpectationsData(resultsForTest);
+
+  assertEquals(resultsForTest.expectations, expectations);
+  assertEquals(resultsForTest.modifiers, modifiers);
+}
+
+function assertEquals(actual, expected) {
+  if (expected !== actual) {
+    throw Error('Got: ' + actual + ' expected: ' + expected);
+  }
+}
+
+function testReleaseFail() {
+  var builder = 'Webkit';
+  var test = 'foo/1.html';
+  var expectatiosnArray = [
+    {'modifiers': 'RELEASE', 'expectations': 'FAIL'}
+  ];
+  var results = {'tests': {
+    'foo/1.html': {
+      'results': [[100, 'F']],
+      'times': [[100, 0]]
+  }}};
+  runExpectationsTest(builder, test, expectatiosnArray, results, 'FAIL',
+      'RELEASE');
+}
+
+function testReleaseFailDebugCrashReleaseBuilder() {
+  var builder = 'Webkit';
+  var test = 'foo/1.html';
+  var expectatiosnArray = [
+    {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
+    {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
+  ];
+  var results = {'tests': {
+    'foo/1.html': {
+      'results': [[100, 'F']],
+      'times': [[100, 0]]
+  }}};
+  runExpectationsTest(builder, test, expectatiosnArray, results, 'FAIL',
+      'RELEASE');
+}
+
+function testReleaseFailDebugCrashDebugBuilder() {
+  var builder = 'Webkit(dbg)';
+  var test = 'foo/1.html';
+  var expectatiosnArray = [
+    {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
+    {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
+  ];
+  var results = {'tests': {
+    'foo/1.html': {
+      'results': [[100, 'F']],
+      'times': [[100, 0]]
+  }}};
+  runExpectationsTest(builder, test, expectatiosnArray, results,  'CRASH',
+      'DEBUG');
+}
+
+function runTests() {
+  document.body.innerHTML = '<pre id=unittest-results></pre>';
+  for (var name in window) {
+    if (typeof window[name] == 'function' && /^test/.test(name)) {
+      setupExpectationsTest();
+
+      var test = window[name];
+      var error = null;
+
+      try {
+        test();
+      } catch (err) {
+        error = err;
+      }
+
+      var result = error ? error.toString() : 'PASSED';
+      $('unittest-results').insertAdjacentHTML("beforeEnd",
+          name + ': ' + result + '\n');
+    }
+  }
+}
+
+if (document.readyState == 'complete') {
+  runTests();
+} else {
+  window.addEventListener('load', runTests, false);
+}
diff --git a/webkit/tools/layout_tests/flakiness_dashboard.html b/webkit/tools/layout_tests/flakiness_dashboard.html
index 5535fe8..8b9a07e 100644
--- a/webkit/tools/layout_tests/flakiness_dashboard.html
+++ b/webkit/tools/layout_tests/flakiness_dashboard.html
@@ -289,6 +289,9 @@
   // Methods and objects from dashboard_base.js to override.
   //////////////////////////////////////////////////////////////////////////////
   function generatePage() {
+    if (currentState.useTestData)
+      return;
+
     if (!currentState.tests && !('builder' in currentState)) {
       for (var builder in builders) {
         currentState.builder = builder;
@@ -416,9 +419,10 @@
   var testToResultsMap = {};
   var numFlakyTestsPerBuilder = {};
 
-  function createResultsObjectForTest(test) {
+  function createResultsObjectForTest(test, builder) {
     return {
       test: test,
+      builder: builder,
       // HTML for display of the results in the flakiness column
       html: '',
       flips: 0,
@@ -705,8 +709,13 @@
     return null;
   }
 
-  function populateExpectationsData(resultsObj, platform, buildType, builder) {
+  function populateExpectationsData(resultsObj) {
     var test = resultsObj.test;
+
+    var buildInfo = getPlatformAndBuildType(resultsObj.builder);
+    var platform = buildInfo.platform;
+    var buildType = buildInfo.buildType;
+
     var thisPlatformExpectations = getExpectations(test, platform, buildType);
 
     var htmlArrays = {};
@@ -714,7 +723,7 @@
     htmlArrays.modifiers = [];
     htmlArrays.bugs = [];
 
-    var tests = resultsByBuilder[builder].tests;
+    var tests = resultsByBuilder[resultsObj.builder].tests;
 
     var testObject = allExpectations[test];
     var usedExpectations = {};
@@ -930,10 +939,8 @@
     var allTestsForThisBuilder = resultsByBuilder[builderName].tests;
 
     for (var test in allTestsForThisBuilder) {
-      var resultsForTest = createResultsObjectForTest(test);
-      resultsForTest.builder = builderName;
-      populateExpectationsData(resultsForTest, platform, buildType,
-          builderName);
+      var resultsForTest = createResultsObjectForTest(test, builderName);
+      populateExpectationsData(resultsForTest);
 
       var rawTest = resultsByBuilder[builderName].tests[test];
       resultsForTest.rawTimes = rawTest.times;
@@ -2153,6 +2160,8 @@
     }
   }, false);
 
+
+
   </script>
 </head>