summaryrefslogtreecommitdiffstats
path: root/webkit/tools/layout_tests
diff options
context:
space:
mode:
authorojan@chromium.org <ojan@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2009-08-05 19:24:47 +0000
committerojan@chromium.org <ojan@chromium.org@0039d316-1c4b-4281-b951-d872f2087c98>2009-08-05 19:24:47 +0000
commit7fa319410e5dbc1739e476f317a33e0ab276a94a (patch)
treec7f4e77ea5cb7dc6e67699f2df63463daf23a741 /webkit/tools/layout_tests
parent4a83c269850847be740568f72640bd57b0c2c86d (diff)
downloadchromium_src-7fa319410e5dbc1739e476f317a33e0ab276a94a.zip
chromium_src-7fa319410e5dbc1739e476f317a33e0ab276a94a.tar.gz
chromium_src-7fa319410e5dbc1739e476f317a33e0ab276a94a.tar.bz2
First stab at a layout tests flakiness/speed dashboard.
This isn't functional yet, but I want to get this reviewed and in the tree so I can do the rest incrementally. This works by having the bots generate JSON that is then red into a static HTML file that generates the dashboard from the JSON. I've tried to make this generic, so we should be able to use the same HTML file for our other test types (e.g. UI tests) as well once this is functional by just having the bots that run those tests generate the JSON files and copy them to the right place. All the work that needs doing to get this 100% functional is listed as a TODO at the top of flakiness_dashboard.html. Most of what's left is buildbot integration (i.e. copy files to the right place on the bot). Review URL: http://codereview.chromium.org/149656 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@22505 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'webkit/tools/layout_tests')
-rw-r--r--webkit/tools/layout_tests/flakiness_dashboard.html625
-rw-r--r--webkit/tools/layout_tests/layout_package/json_results_generator.py182
-rw-r--r--webkit/tools/layout_tests/layout_package/test_expectations.py57
-rwxr-xr-xwebkit/tools/layout_tests/run_webkit_tests.py32
4 files changed, 886 insertions, 10 deletions
diff --git a/webkit/tools/layout_tests/flakiness_dashboard.html b/webkit/tools/layout_tests/flakiness_dashboard.html
new file mode 100644
index 0000000..f73fe0e
--- /dev/null
+++ b/webkit/tools/layout_tests/flakiness_dashboard.html
@@ -0,0 +1,625 @@
+<!DOCTYPE HTML>
+<html>
+
+<head>
+ <title>Webkit Layout Test History</title>
+ <style>
+ .test-link {
+ white-space: normal;
+ }
+ .test-table {
+ white-space: nowrap;
+ }
+ .test-table {
+ width: 100%;
+ }
+ .test-table td {
+ font-size: 13px;
+ padding: 0 2px;
+ }
+ .test-table tr {
+ border: 1px solid red;
+ background-color: #E8E8E8;
+ }
+ .test-table tbody tr:hover {
+ opacity: .7;
+ }
+ .test-table th {
+ -webkit-user-select: none;
+ -moz-user-select: none;
+ }
+ .link {
+ color: blue;
+ text-decoration: underline;
+ }
+ .header-container,
+ .header-container *,
+ .table-header-content,
+ .table-header-content * {
+ display: -webkit-box;
+ }
+ .table-header-content * {
+ -webkit-box-flex: 1;
+ }
+ .header-container * {
+ -webkit-box-flex: 0;
+ }
+ .header-title {
+ -webkit-box-flex: 1;
+ -webkit-box-pack: center;
+ font-weight: bold;
+ white-space: nowrap;
+ }
+ .results {
+ cursor: pointer;
+ }
+ .legend * {
+ padding: 0 3px;
+ }
+ .builders * {
+ margin: 0 5px;
+ }
+ .test-table .expectation-mismatch,
+ .expectation-mismatch {
+ background-color: #afdaff;
+ }
+ .P {
+ background-color: #8fdf5f;
+ }
+ .N {
+ background-color: #e0b0ff;
+ }
+ .C {
+ background-color: #ffc343;
+ }
+ .T {
+ background-color: #fffc6c;
+ }
+ .I {
+ background-color: yellow;
+ }
+ .S {
+ background-color: red;
+ }
+ .F {
+ background-color: #e98080;
+ }
+ .O {
+ background-color: blue;
+ }
+ .separator {
+ border: 1px solid lightgray;
+ height: 0px;
+ }
+ .different-platform {
+ color: gray;
+ font-size: 10px;
+ }
+ .current-builder {
+ text-decoration: none;
+ color: black;
+ }
+ </style>
+
+ <script>
+ /**
+ * @fileoverview Creates a dashboard for multiple runs of a given set of tests
+ * on the buildbots. Pulls in JSONP-ish files with the results for running
+ * tests on a given builder (i.e. ADD_RESULTS(json_here)) and the expectations
+ * for all tests on all builders (i.e. ADD_EXPECTATIONS(json_here)).
+ *
+ * This shows flakiness of the tests as well as runtimes for slow tests.
+ *
+ * Also, each column in the dashboard is sortable.
+ *
+ * Currently, only webkit tests are supported, but adding other test types
+ * should just require the following steps:
+ * -generate results.json and expectations.json for these tests
+ * -copy them to the appropriate location
+ * -add the builder name to the list of builders below.
+ */
+ var builders = ['WebKit', 'WebKitDebug'];
+
+ /* TODO
+ PYTHON
+ -have bots copy JSON files for build results + expectations to the right
+ places. Include the test type in the path so we can reuse this HTML for
+ other tests (e.g. UI tests)
+ -get the build_number and builder_name from the bot
+ -make file paths local
+
+ JS
+ -make getPathToBuilderResultsFile point to the actual bots
+ -UNEXPECTED RESULTS PRINTS WRONG THING
+ (e.g. undefined for "N" and SIMPLIFIED for "S")
+
+ JS - Nice-to-haves
+ -highlight expected results that never happen
+ -show range of build numbers that are included
+ -checkboxes to hide defer/wontfix tests
+ */
+ var resultsByBuilder = {};
+ var expectationsByTest = {};
+ var testType = window.location.search.substring(1);
+ function ADD_RESULTS(builds) {
+ for (var builderName in builds)
+ resultsByBuilder[builderName] = builds[builderName];
+ }
+ var BUILDER_BASE = 'LayoutTestDashBoard';
+ function getPathToBuilderResultsFile(builderName) {
+ // TODO(ojan): Make this match the actual path to the bots.
+ return BUILDER_BASE + testType + builders[i]
+ }
+ for (var i = 0; i < builders.length; i++) {
+ var script = document.createElement('script');
+ script.src = getPathToBuilderResultsFile(builders[i]) + 'results.js';
+ document.getElementsByTagName('head')[0].appendChild(script);
+ }
+
+ var script = document.createElement('script');
+ // Grab expectations file from any builder.
+ script.src = getPathToBuilderResultsFile(builders[0]) + 'expectations.json';
+ document.getElementsByTagName('head')[0].appendChild(script);
+
+ function ADD_EXPECTATIONS(expectations) {
+ for (var test in expectations)
+ expectationsByTest[test] = expectations[test];
+ }
+
+ // CONSTANTS
+ var FORWARD = 'forward';
+ var BACKWARD = 'backward';
+ var TEST_URL_BASE_PATH =
+ 'http://trac.webkit.org/projects/webkit/browser/trunk/';
+ var BUILDERS_BASE_PATH =
+ 'http://build.chromium.org/buildbot/waterfall/builders/';
+ var EXPECTATIONS_MAP = {
+ 'T': 'TIMEOUT',
+ 'C': 'CRASH',
+ 'P': 'PASS',
+ 'F': 'TEXT FAIL',
+ 'S': 'SIMPLIFIED',
+ 'I': 'IMAGE',
+ 'O': 'OTHER'
+ };
+ var TABLE_HEADERS = ['test', 'modifiers', 'expectations',
+ 'unexpected results', 'slowest run',
+ 'flakiness (numbers are runtimes in seconds)'];
+ var PLATFORMS = {'MAC': 'MAC', 'LINUX': 'LINUX', 'WIN': 'WIN'};
+ var BUILD_TYPES = {'DEBUG': 'DBG', 'RELEASE': 'RELEASE'};
+
+ // GLOBALS
+ var currentState = {builder: null, sortFailures: 'test', sortPasses: 'test',
+ sortFailuresOrder: FORWARD, sortPassesOrder: FORWARD};
+ var perBuilderPlatformAndBuildType = {};
+ var oldLocation;
+ var perBuilderFailures = {};
+ var allTestsCache;
+
+ function createResultsObjectForTest(test) {
+ return {
+ test: test,
+ // HTML for display of the results in the flakiness column
+ html: '',
+ flips: 0,
+ slowestTime: 0,
+ meetsExpectations: true,
+ // Map of test results that don't match their expectations
+ unexpectedExpectations: {},
+ // Sorted string of unexpected expectations
+ unexpected: '',
+ // HTML for expectations for this test for all platforms
+ expectationsHTML: '',
+ // HTML for modifiers for this test for all platforms
+ modifiersHTML: ''
+ };
+ }
+
+ function getMatchingElement(stringToMatch, elementsMap) {
+ for (var element in elementsMap) {
+ if (stringToMatch.indexOf(elementsMap[element]) != -1)
+ return element;
+ }
+ }
+
+ function stringContains(a, b) {
+ return a.indexOf(b) != -1;
+ }
+
+ function anyKeyInString(object, string) {
+ for (var key in object) {
+ if (stringContains(string, key))
+ return true;
+ }
+ return false;
+ }
+
+ function hasPlatformAndBuildTypeForBuilder(builderName, modifiers) {
+ var currentPlatformAndBuildType;
+ if (perBuilderPlatformAndBuildType[builderName]) {
+ currentPlatformAndBuildType = perBuilderPlatformAndBuildType[builderName];
+ } else {
+ // If the build name does not contain a platform
+ // or build type, assume Windows Release.
+ var currentBuildUppercase = builderName.toUpperCase();
+ var platform = getMatchingElement(currentBuildUppercase, PLATFORMS) ||
+ 'WIN';
+ var buildType = getMatchingElement(currentBuildUppercase, BUILD_TYPES) ||
+ 'RELEASE';
+ currentPlatformAndBuildType = {platform: platform, buildType: buildType};
+ }
+
+ var hasThisPlatform = stringContains(modifiers,
+ currentPlatformAndBuildType.platform);
+ var hasThisBuildType = stringContains(modifiers,
+ currentPlatformAndBuildType.buildType);
+
+ var hasAnyPlatform = anyKeyInString(PLATFORMS, modifiers);
+ var hasAnyBuildType = anyKeyInString(BUILD_TYPES, modifiers);
+
+ return (!hasAnyBuildType || hasThisBuildType) &&
+ (!hasAnyPlatform || hasThisPlatform);
+ }
+
+ function processTestRunsForBuilder(builderName) {
+ if (perBuilderFailures[builderName])
+ return;
+
+ var failures = [];
+ var passes = [];
+
+ var allTests = resultsByBuilder[builderName].tests;
+
+ var expectationsMap = {};
+ var testPrefixes = {};
+ var testsWithoutDirectExpectations = [];
+ for (var test in allTests) {
+ if (expectationsByTest[test])
+ expectationsMap[test] = expectationsByTest[test];
+ else
+ testsWithoutDirectExpectations.push(test);
+ }
+ for (var path in expectationsByTest) {
+ if (allTests[path]) {
+ // Test path is an exact match (i.e. not a prefix match)
+ expectationsMap[path] = expectationsByTest[path];
+ testPrefixes[path] = path;
+ } else {
+ // Test path doesn't match a specific test, see if it prefix matches
+ // any test.
+ for (var test in allTests) {
+ if (stringContains(test, path) &&
+ (!testPrefixes[test] || testPrefixes[test].indexOf(path) == -1)) {
+ expectationsMap[test] = expectationsByTest[path];
+ testPrefixes[test] = path;
+ }
+ }
+ }
+ }
+
+ for (var test in allTests) {
+ var resultsForTest = createResultsObjectForTest(test);
+
+ if (expectationsMap[test] && expectationsMap[test].length) {
+ var expectationsArray = expectationsMap[test];
+ var expectationsHTMLArray = [];
+ var modifiersHTMLArray = [];
+
+ var thisBuilderExpectations;
+ var thisBuilderModifiers;
+ for (var i = 0; i < expectationsArray.length; i++) {
+ var modifiers = expectationsArray[i].modifiers;
+ var expectations = expectationsArray[i].expectations;
+ if (hasPlatformAndBuildTypeForBuilder(builderName, modifiers)) {
+ resultsForTest.expectations = expectations;
+ resultsForTest.modifiers = modifiers;
+ } else {
+ // TODO: add classname to the ones that don't apply for this
+ // platform in getHTMLForOptionsList so they can be greyed out
+ expectationsHTMLArray.push(
+ '<div class="option different-platform">' + expectations +
+ '</div>');
+ modifiersHTMLArray.push(
+ '<div class="option different-platform">' + modifiers +
+ '</div>');
+ }
+ }
+
+ if (resultsForTest.expectations) {
+ expectationsHTMLArray.push('<div class="option">' +
+ resultsForTest.expectations + '</div>');
+ modifiersHTMLArray.push('<div class="option">' +
+ resultsForTest.modifiers + '</div>');
+ }
+
+ resultsForTest.expectationsHTML +=
+ expectationsHTMLArray.join('<div class=separator></div>');
+ resultsForTest.modifiersHTML +=
+ modifiersHTMLArray.join('<div class=separator></div>');
+ }
+
+ var results = resultsByBuilder[builderName].tests[test].results.split('');
+
+ var unexpectedExpectations = [];
+ for (var i = 0; i < results.length - 1; i++) {
+ if (results[i] != results[i + 1])
+ resultsForTest.flips++;
+
+ var expectation = EXPECTATIONS_MAP[results[i]];
+ if (hasExpectations(resultsForTest.expectations, results[i])) {
+ unexpectedExpectations[expectation] = true;
+ resultsForTest.meetsExpectations = false;
+ }
+ }
+
+ var times = resultsByBuilder[builderName].tests[test].times;
+ resultsForTest.slowestTime = Math.max.apply(null, times)
+
+ resultsForTest.html = getHtmlForIndividualTest(builderName, test);
+
+ var expectations = [];
+ for (var expectation in unexpectedExpectations) {
+ expectations.push(expectation);
+ }
+ if (expectations.length) {
+ resultsForTest.unexpected = expectations.join(' ');
+ }
+
+ if (didTestPassAllRuns(builderName, test)) {
+ resultsForTest.meetsExpectations =
+ expectations || expectations == 'PASS';
+ }
+
+ failures.push(resultsForTest);
+ }
+
+ perBuilderFailures[currentState.builder] = failures;
+ }
+
+ function hasExpectations(expectations, result) {
+ // For the purposes of comparing against the expecations of a test,
+ // consider image/simplified diff failures as just failures since
+ // the test_expectations file doesn't treat them specially.
+ if (result == 'S' || result == 'I')
+ result = 'F'
+
+ var thisExpectation = EXPECTATIONS_MAP[result];
+ if (!thisExpectation)
+ return true;
+
+ if (!expectations)
+ return false;
+
+ return expectations.indexOf(thisExpectation) != -1;
+ }
+
+ function didTestPassAllRuns(builderName, testPath) {
+ var numBuilds = resultsByBuilder[builderName].buildNumbers.length;
+ var passingResults = Array(numBuilds + 1).join('P');
+ var results = resultsByBuilder[builderName].tests[testPath].results;
+ return results == passingResults;
+ }
+
+ function loadBuilderPageForBuildNumber(builderName, buildNumber) {
+ window.open(BUILDERS_BASE_PATH + builderName + '/builds/' + buildNumber);
+ }
+
+ function getHtmlForIndividualTest(builderName, testPath) {
+ var html = '';
+ var test = resultsByBuilder[builderName].tests[testPath];
+ var results = test.results.split('');
+ var times = test.times;
+ var buildNumbers = resultsByBuilder[builderName].buildNumbers;
+ for (var i = 0; i < results.length; i++) {
+ var buildNumber = buildNumbers[i];
+ // To avoid noise, only print times that are larger than 1 second.
+ // TODO: See if this is necessary. If it is, look into tweaking the
+ // threshold lower/higher.
+ var innerHTML = times[i] > 1 ? times[i] : '&nbsp;';
+ html += '<td title="Build:' + buildNumber + '" class="results ' +
+ results[i] + '" onclick=\'loadBuilderPageForBuildNumber("' +
+ builderName + '","' + buildNumber + '")\'>' + innerHTML + '</td>';
+ }
+ return html;
+ }
+
+ function getHTMLForTestTable(results, id, sort, order) {
+ var html = '<table id=' + id + ' class=test-table>' +
+ getTableHeaders(sort, order) + '<tbody>';
+
+ sortTests(results, sort, order);
+ for (var i = 0; i < results.length; i++) {
+ var test = results[i];
+ html += '<tr class=' +
+ (test.meetsExpectations ? '' : 'expectation-mismatch') +
+ // TODO: If a test is a chrome/ or a pending/ test, point to
+ // src.chromium.org instead of trac.webkit.org.
+ '><td class=test-link><a href="' + TEST_URL_BASE_PATH + test.test +
+ '">' + test.test + '</a>' +
+ '</td><td class=options-container>' + test.modifiersHTML +
+ '</td><td class=options-container>' + test.expectationsHTML +
+ '</td><td>' + test.unexpected +
+ '</td><td>' + (test.slowestTime ? test.slowestTime + 's' : '') +
+ '</td>' +
+ test.html +
+ '</tr>';
+ }
+
+ html += "</tbody></table>"
+ return html;
+ }
+
+ function getTableHeaders(sort, order) {
+ var html = '<thead><tr>';
+ for (var i = 0; i < TABLE_HEADERS.length; i++) {
+ // Use the first word of the header title as the sortkey
+ var thisSortValue = TABLE_HEADERS[i].split(' ')[0];
+ var arrowHTML = thisSortValue == sort ?
+ '<span class=' + order + '>' +
+ (order == FORWARD ? '&uarr;' : '&darr;' ) + '</span>' :
+ '';
+ html += '<th sortValue=' + thisSortValue +
+ // Extend last th through all the rest of the columns.
+ (i == TABLE_HEADERS.length - 1 ? ' colspan=10000' : '') +
+ // Extra span here is so flex boxing actually centers.
+ // There's probably a better way to do this with CSS only though.
+ '><div class=table-header-content><span></span>' + arrowHTML +
+ '<span class=link>' + TABLE_HEADERS[i] + '</span>' +
+ arrowHTML + '</div></th>';
+ }
+ html += '</tr></thead>';
+ return html;
+ }
+
+ function getHTMLForPageHeader() {
+ var html = '<div class=header-container>';
+
+ html += '<span class=builders>';
+ for (var builder in resultsByBuilder) {
+ var className = builder == currentState.builder ? 'current-builder' : '';
+ html += '<a' + (className ? ' class="' + className : '') + '" link"' +
+ ' onclick=\'setState("builder", "' + builder +
+ '");return false;\'>' + builder + '</a>';
+ }
+ html += '</span>';
+
+ html += '<span class=header-title></span>';
+
+ html += '<span class=legend>';
+ for (var expectation in EXPECTATIONS_MAP) {
+ html += '<span class=' + expectation + '>' +
+ EXPECTATIONS_MAP[expectation] + '</span>';
+ }
+ html += '<span class=N>NO DATA</span>'
+ html += '<span class=expectation-mismatch>WRONG EXPECTATIONS</span>'
+ html += '</span>';
+
+ html += '</div>';
+ return html;
+ }
+
+ function setFullPageHTML() {
+ var html = getHTMLForPageHeader();
+ html += getHTMLForTestTable(perBuilderFailures[currentState.builder],
+ 'failures', currentState.sortFailures, currentState.sortFailuresOrder);
+ document.body.innerHTML = html;
+ }
+
+ function getAlphanumericCompare(column, reverse) {
+ return getReversibleCompareFunction(function(a, b) {
+ // Put null entries at the bottom
+ var a = a[column] ? String(a[column]) : 'z';
+ var b = b[column] ? String(b[column]) : 'z';
+
+ if (a < b)
+ return -1;
+ else if (a == b)
+ return 0;
+ else
+ return 1;
+ }, reverse);
+ }
+
+ function getNumericSort(column, reverse) {
+ return getReversibleCompareFunction(function(a, b) {
+ a = parseFloat(a[column]);
+ b = parseFloat(b[column]);
+ return a - b;
+ }, reverse);
+ }
+
+ function getReversibleCompareFunction(compare, reverse) {
+ return function(a, b) {
+ return compare(reverse ? b : a, reverse ? a : b);
+ }
+ }
+
+ function changeSort(e) {
+ var target = e.currentTarget;
+ e.preventDefault();
+
+ var sortValue = target.getAttribute('sortValue');
+ while (target && target.tagName != 'TABLE') {
+ target = target.parentNode;
+ }
+
+ var sort = target.id == 'passes' ? 'sortPasses' : 'sortFailures';
+
+ var orderKey = sort + 'Order';
+ if (sortValue == currentState[sort] && currentState[orderKey] == FORWARD)
+ order = BACKWARD;
+ else
+ order = FORWARD;
+
+ setState(sort, sortValue);
+ setState(orderKey, order);
+ }
+
+ function sortTests(tests, column, order) {
+ var resultsProperty, sortFunctionGetter;
+ if (column == 'flakiness') {
+ sortFunctionGetter = getNumericSort;
+ resultsProperty = 'flips';
+ } else if (column == 'slowest') {
+ sortFunctionGetter = getNumericSort;
+ resultsProperty = 'slowestTime';
+ } else {
+ sortFunctionGetter = getAlphanumericCompare;
+ resultsProperty = column;
+ }
+
+ tests.sort(sortFunctionGetter(resultsProperty, order == BACKWARD));
+ }
+
+ function generatePage() {
+ currentPlatformAndBuildType = null;
+ oldLocation = window.location.toString();
+ var hash = window.location.hash;
+ var builderName;
+ if (hash) {
+ hashParts = hash.slice(1).split('&');
+ for (var i = 0; i < hashParts.length; i++) {
+ var itemParts = hashParts[i].split('=');
+ // TODO: Validate itemParts[0]
+ currentState[itemParts[0]] = itemParts[1];
+ }
+ }
+ if (!(currentState.builder in resultsByBuilder)) {
+ for (builder in resultsByBuilder) {
+ currentState.builder = builder;
+ break;
+ }
+ }
+
+ processTestRunsForBuilder(currentState.builder);
+ setFullPageHTML();
+
+ var ths = document.getElementsByTagName('th');
+ for (var i = 0; i < ths.length; i++) {
+ ths[i].addEventListener('click', changeSort, false);
+ }
+ }
+
+ function setState(key, value) {
+ currentState[key] = value;
+ window.location.replace(window.location.pathname + '#' +
+ 'builder=' + currentState.builder + '&' +
+ 'sortFailures=' + currentState.sortFailures + '&' +
+ 'sortFailuresOrder=' + currentState.sortFailuresOrder + '&' +
+ 'sortPasses=' + currentState.sortPasses + '&' +
+ 'sortPassesOrder=' + currentState.sortPassesOrder);
+ }
+
+ window.onload = function() {
+ // Poll for hash changes.
+ // TODO: Use hashchange event when it is supported.
+ setInterval(function() {
+ if (oldLocation != window.location)
+ generatePage();
+ }, 100);
+ }
+ </script>
+</head>
+
+<body></body>
+</html>
diff --git a/webkit/tools/layout_tests/layout_package/json_results_generator.py b/webkit/tools/layout_tests/layout_package/json_results_generator.py
new file mode 100644
index 0000000..86c1da6
--- /dev/null
+++ b/webkit/tools/layout_tests/layout_package/json_results_generator.py
@@ -0,0 +1,182 @@
+# Copyright (c) 2009 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import re
+
+from layout_package import path_utils
+from layout_package import test_failures
+
+class ResultAndTime:
+ """A holder for a single result and runtime for a test."""
+ time = 0
+ result = "N"
+
+class JSONResultsGenerator:
+
+ MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 100
+ # Min time (seconds) that will be added to the JSON.
+ MIN_TIME = 1
+ JSON_PREFIX = "ADD_RESULTS("
+ JSON_SUFFIX = ");"
+
+ def __init__(self, failures, individual_test_timings, builder_name,
+ build_number, results_file_path):
+ """
+ failures: Map of test name to list of failures.
+ individual_test_times: Map of test name to a tuple containing the
+ test_run-time.
+ builder_name: The name of the builder the tests are being run on.
+ build_number: The build number for this run.
+ results_file_path: Absolute path to the results json file.
+ """
+ self._failures = failures
+ self._test_timings = individual_test_timings
+ self._builder_name = builder_name
+ self._build_number = build_number
+ self._results_file_path = results_file_path
+
+ def GetJSON(self):
+ """Gets the results for the results.json file."""
+ failures_for_json = {}
+ for test in self._failures:
+ failures_for_json[test] = ResultAndTime()
+ # TODO(ojan): get relative path here
+ failures_for_json[test].result = self._GetResultsCharForFailure(test)
+
+ for test_tuple in self._test_timings:
+ test = test_tuple.filename
+ if not test in failures_for_json:
+ failures_for_json[test] = ResultAndTime()
+ # Floor for now to get time in seconds.
+ # TODO(ojan): As we make tests faster, reduce to tenth of a second
+ # granularity.
+ failures_for_json[test].time = int(test_tuple.test_run_time)
+
+ # If results file exists, read it out, put new info in it.
+ if os.path.exists(self._results_file_path):
+ old_results_file = open(self._results_file_path, "r")
+ old_results = old_results_file.read()
+ # Strip the prefix and suffix so we can get the actual JSON object.
+ old_results = old_results[
+ len(self.JSON_PREFIX) : len(old_results) - len(self.JSON_SUFFIX)]
+ results_json = eval(old_results)
+
+ if self._builder_name not in results_json:
+ logging.error("Builder name (%s) is not in the results.json file." %
+ self._builder_name);
+ else:
+ # TODO(ojan): If the build output directory gets clobbered, we should
+ # grab this file off wherever it's archived to. Maybe we should always
+ # just grab it from wherever it's archived to.
+ results_json = {}
+
+ if self._builder_name not in results_json:
+ results_json[self._builder_name] = self._CreateResultsForBuilderJSON()
+
+ tests = results_json[self._builder_name]["tests"]
+ all_failing_tests = set(self._failures.iterkeys())
+ all_failing_tests.update(tests.iterkeys())
+
+ for test in all_failing_tests:
+ if test in failures_for_json:
+ result_and_time = failures_for_json[test]
+ else:
+ result_and_time = ResultAndTime()
+
+ if test not in tests:
+ tests[test] = self._CreateResultsAndTimesJSON()
+
+ thisTest = tests[test]
+ thisTest["results"] = result_and_time.result + thisTest["results"]
+ thisTest["times"].insert(0, result_and_time.time)
+
+ self._NormalizeResultsJSON(thisTest, test, tests)
+
+ results_json[self._builder_name]["buildNumbers"].insert(0,
+ self._build_number)
+
+ # Generate the JSON and strip whitespace to keep filesize down.
+ # TODO(ojan): Generate the JSON using a JSON library should someone ever
+ # add a non-primitive type to results_json.
+ results_str = self.JSON_PREFIX + repr(results_json) + self.JSON_SUFFIX
+ return re.sub(r'\s+', '', results_str)
+
+ def _CreateResultsAndTimesJSON(self):
+ results_and_times = {}
+ results_and_times["results"] = ""
+ results_and_times["times"] = []
+ return results_and_times
+
+ def _CreateResultsForBuilderJSON(self):
+ results_for_builder = {}
+ results_for_builder['buildNumbers'] = []
+ results_for_builder['tests'] = {}
+ return results_for_builder
+
+ def _GetResultsCharForFailure(self, test):
+ """Returns the worst failure from the list of failures for this test
+ since we can only show one failure per run for each test on the dashboard.
+ """
+ failures = [failure.__class__ for failure in self._failures[test]]
+
+ if test_failures.FailureCrash in failures:
+ return "C"
+ elif test_failures.FailureTimeout in failures:
+ return "T"
+ elif test_failures.FailureImageHashMismatch in failures:
+ return "I"
+ elif test_failures.FailureSimplifiedTextMismatch in failures:
+ return "S"
+ elif test_failures.FailureTextMismatch in failures:
+ return "F"
+ else:
+ return "O"
+
+ def _NormalizeResultsJSON(self, test, test_path, tests):
+ """ Prune tests where all runs pass or tests that no longer exist and
+ truncate all results to maxNumberOfBuilds and pad results that don't
+ have encough runs for maxNumberOfBuilds.
+
+ Args:
+ test: ResultsAndTimes object for this test.
+ test_path: Path to the test.
+ tests: The JSON object with all the test results for this builder.
+ """
+ results = test["results"]
+ num_results = len(results)
+ times = test["times"]
+
+ if num_results != len(times):
+ logging.error("Test has different number of build times versus results")
+ times = []
+ results = ""
+ num_results = 0
+
+ # Truncate or right-pad so there are exactly maxNumberOfBuilds results.
+ if num_results > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
+ results = results[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG]
+ times = times[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG]
+ elif num_results < self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
+ num_to_pad = self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG - num_results
+ results = results + num_to_pad * 'N'
+ times.extend(num_to_pad * [0])
+
+ test["results"] = results
+ test["times"] = times
+
+ # If the test has all passes or all no-data (that wasn't just padded in) and
+ # times that take less than a second, remove it from the results to reduce
+ # noise and filesize.
+ if (max(times) >= self.MIN_TIME and num_results and
+ (results == self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG * 'P' or
+ results == self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG * 'N')):
+ del tests[test_path]
+
+ # Remove tests that don't exist anymore.
+ full_path = os.path.join(path_utils.LayoutTestsDir(test_path), test_path)
+ full_path = os.path.normpath(full_path)
+ if not os.path.exists(full_path):
+ del tests[test_path]
diff --git a/webkit/tools/layout_tests/layout_package/test_expectations.py b/webkit/tools/layout_tests/layout_package/test_expectations.py
index 51945a6..089b896 100644
--- a/webkit/tools/layout_tests/layout_package/test_expectations.py
+++ b/webkit/tools/layout_tests/layout_package/test_expectations.py
@@ -22,7 +22,6 @@ import compare_failures
# Test expectation file update action constants
(NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
-
class TestExpectations:
TEST_LIST = "test_expectations.txt"
@@ -37,6 +36,9 @@ class TestExpectations:
# TODO(ojan): Replace the Get* calls here with the more sane API exposed
# by TestExpectationsFile below. Maybe merge the two classes entirely?
+ def GetExpectationsForAllPlatforms(self):
+ return self._expected_failures.GetExpectationsForAllPlatforms()
+
def GetFixable(self):
return self._expected_failures.GetTestSet(NONE)
@@ -145,6 +147,17 @@ def StripComments(line):
if line == '': return None
else: return line
+class ModifiersAndExpectations:
+ """A holder for modifiers and expectations on a test that serializes to JSON.
+ """
+ def __init__(self, modifiers, expectations):
+ self.modifiers = modifiers
+ self.expectations = expectations
+
+ def __repr__(self):
+ return ("{modifiers:'" + self.modifiers + "', expectations:'" +
+ self.expectations + "'}")
+
class TestExpectationsFile:
"""Test expectation files consist of lines with specifications of what
to expect from layout test cases. The test cases can be directories
@@ -213,6 +226,11 @@ class TestExpectationsFile:
self._platform = platform
self._is_debug_mode = is_debug_mode
+ # Maps relative test paths as listed in the expectations file to a list of
+ # maps containing modifiers and expectations for each time the test is
+ # listed in the expectations file.
+ self._all_expectations = {}
+
# Maps a test to its list of expectations.
self._test_to_expectations = {}
@@ -249,6 +267,9 @@ class TestExpectationsFile:
def GetExpectations(self, test):
return self._test_to_expectations[test]
+ def GetExpectationsForAllPlatforms(self):
+ return self._all_expectations
+
def Contains(self, test):
return test in self._test_to_expectations
@@ -421,6 +442,12 @@ class TestExpectationsFile:
return True
+ def _AddToAllExpectations(self, test, options, expectations):
+ if not test in self._all_expectations:
+ self._all_expectations[test] = []
+ self._all_expectations[test].append(
+ ModifiersAndExpectations(options, expectations))
+
def _Read(self, path):
"""For each test in an expectations file, generate the expectations for it.
"""
@@ -430,16 +457,15 @@ class TestExpectationsFile:
line = StripComments(line)
if not line: continue
- modifiers = set()
+ options_string = None
+ expectations_string = None
+
if line.find(':') is -1:
test_and_expectations = line
else:
parts = line.split(':')
test_and_expectations = parts[1]
- options = self._GetOptionsList(parts[0])
- if not self._HasValidModifiersForCurrentPlatform(options, lineno,
- test_and_expectations, modifiers):
- continue
+ options_string = parts[0]
tests_and_expecation_parts = test_and_expectations.split('=')
if (len(tests_and_expecation_parts) is not 2):
@@ -447,12 +473,23 @@ class TestExpectationsFile:
continue
test_list_path = tests_and_expecation_parts[0].strip()
- expectations = self._ParseExpectations(tests_and_expecation_parts[1],
- lineno, test_list_path)
+ expectations_string = tests_and_expecation_parts[1]
+
+ self._AddToAllExpectations(test_list_path, options_string,
+ expectations_string)
+
+ modifiers = set()
+ options = self._GetOptionsList(options_string)
+ if options and not self._HasValidModifiersForCurrentPlatform(options,
+ lineno, test_and_expectations, modifiers):
+ continue
+
+ expectations = self._ParseExpectations(expectations_string, lineno,
+ test_list_path)
if 'slow' in options and TIMEOUT in expectations:
- self._AddError(lineno, 'A test cannot be both slow and timeout. If the '
- 'test times out indefinitely, the it should be listed as timeout.',
+ self._AddError(lineno, 'A test should not be both slow and timeout. '
+ 'If it times out indefinitely, then it should be just timeout.',
test_and_expectations)
full_path = os.path.join(path_utils.LayoutTestsDir(test_list_path),
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index 1b09bd2..89af345 100755
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -28,6 +28,7 @@ import optparse
import os
import Queue
import random
+import re
import shutil
import subprocess
import sys
@@ -39,6 +40,7 @@ import google.path_utils
from layout_package import compare_failures
from layout_package import test_expectations
from layout_package import http_server
+from layout_package import json_results_generator
from layout_package import path_utils
from layout_package import platform_utils
from layout_package import test_failures
@@ -575,6 +577,9 @@ class TestRunner:
# Write summaries to stdout.
self._PrintResults(failures, sys.stdout)
+ if self._options.verbose:
+ self._WriteJSONFiles(failures, individual_test_timings);
+
# Write the same data to a log file.
out_filename = os.path.join(self._options.results_directory, "score.txt")
output_file = open(out_filename, "w")
@@ -591,6 +596,33 @@ class TestRunner:
sys.stderr.flush()
return len(regressions)
+ def _WriteJSONFiles(self, failures, individual_test_timings):
+ # Write a json file of the test_expectations.txt file for the layout tests
+ # dashboard.
+ expectations_file = open(os.path.join(self._options.results_directory,
+ "expectations.json"), "w")
+ # TODO(ojan): Generate JSON using a JSON library instead of relying on
+ # GetExpectationsForAllPlatforms returning an object that only uses
+ # primitive types.
+ # Strip whitespace to reduce filesize.
+ expectations_json = re.sub(r'\s+', '',
+ repr(self._expectations.GetExpectationsForAllPlatforms()))
+ expectations_file.write(("ADD_EXPECTATIONS(" + expectations_json + ");"))
+ expectations_file.close()
+
+ results_file_path = os.path.join(self._options.results_directory,
+ "results.json")
+ # TODO(ojan): get these from the bot
+ builder_name = "WebKitBuilder"
+ build_number = "12346"
+ json_generator = json_results_generator.JSONResultsGenerator(failures,
+ individual_test_timings, builder_name, build_number, results_file_path)
+ results_json = json_generator.GetJSON()
+
+ results_file = open(results_file_path, "w")
+ results_file.write(results_json)
+ results_file.close()
+
def _PrintTimingStatistics(self, directory_test_timings,
individual_test_timings, failures):
self._PrintAggregateTestStatistics(individual_test_timings)
generated by cgit v1.1 at 2025-01-31 17:15:46 +0000