diff options
| author | ojan@google.com <ojan@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2010-10-06 22:37:03 +0000 |
|---|---|---|
| committer | ojan@google.com <ojan@google.com@0039d316-1c4b-4281-b951-d872f2087c98> | 2010-10-06 22:37:03 +0000 |
| commit | 5407bb17521ef6100bd5079cdcca2bd3e59fca42 (patch) | |
| tree | c76d1ef7a38d6acde20ddc6a2ce03a37d924a36e /webkit/tools | |
| parent | bcc7d0c90d0eabd0f84bb776e3fc48abca46c743 (diff) | |
| download | chromium_src-5407bb17521ef6100bd5079cdcca2bd3e59fca42.zip chromium_src-5407bb17521ef6100bd5079cdcca2bd3e59fca42.tar.gz chromium_src-5407bb17521ef6100bd5079cdcca2bd3e59fca42.tar.bz2 | |
Extends update_expectations_from_dashboard.py to handle adding missing expectations to test_expectations.txt.
Submitted on behalf of koz@chromium.org.
BUG=None
TEST=Run expectations_line_unittest.py.
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@61720 0039d316-1c4b-4281-b951-d872f2087c98
Diffstat (limited to 'webkit/tools')
4 files changed, 358 insertions, 79 deletions
diff --git a/webkit/tools/layout_tests/webkitpy/layout_tests/expectations_line.py b/webkit/tools/layout_tests/webkitpy/layout_tests/expectations_line.py new file mode 100644 index 0000000..0bce734 --- /dev/null +++ b/webkit/tools/layout_tests/webkitpy/layout_tests/expectations_line.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# Copyright (c) 2010 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import copy +import os +import re +import sys + +# +# Find the WebKit python directories and add them to the PYTHONPATH +# +try: + f = __file__ +except NameError: + f = sys.argv[0] + +this_file = os.path.abspath(f) +base_dir = this_file[0:this_file.find('webkit'+ os.sep + 'tools')] +webkitpy_dir = os.path.join(base_dir, 'third_party', 'WebKit', 'WebKitTools', + 'Scripts') +sys.path.append(webkitpy_dir) + +from webkitpy.layout_tests.layout_package import test_expectations + + +class ExpectationsLine(test_expectations.TestExpectationsFile): + """Represents a single line from test_expectations.txt. It is instantiated + like so: + + ExpectationsLine("BUG1 LINUX : fast/dom/test.html = TEXT") + """ + builds = set(['release', 'debug']) + platforms = set(['linux', 'win', 'mac']) + attributes = set(['^bug', 'slow', 'wontfix', 'skip', 'defer']) + + def __init__(self, line): + self._line = line + self._test, self._options, self._expectations = ( + self.parse_expectations_line(line, 0)) + + self._options = set(self._options) + + self._attributes = set(self._remove_from_options(ExpectationsLine.attributes)) + self._platforms = set(self._remove_from_options(ExpectationsLine.platforms)) + self._builds = set(self._remove_from_options(ExpectationsLine.builds)) + + if len(self._platforms) == 0: + self._platforms = ExpectationsLine.platforms + + if len(self._builds) == 0: + self._builds = ExpectationsLine.builds + + def _remove_from_options(self, regexes): + result = [] + for option in self._options: + for regex in regexes: + if re.match(regex, option) and not (option in result): + result.append(option) + + for removed_option in result: + self._options.remove(removed_option) + return result + + def can_merge(self, other): + if self._test != other._test: + return False + if self._expectations != other._expectations: + return False + if self._attributes != other._attributes: + return False + + self2 = copy.deepcopy(self) + + expected_len = self2.target_count() + other.target_count() + + self2.merge(other) + + return self2.target_count() == expected_len + + def merge(self, other): + for build in other._builds: + if build not in self._builds: + self._builds.add(build) + for platform in other._platforms: + if platform not in self._platforms: + self._platforms.add(platform) + + def does_target(self, platform, build): + return (self._empty_or_contains(platform, self._platforms) and + self._empty_or_contains(build, self._builds)) + + def _empty_or_contains(self, elem, list): + return len(list) == 0 or elem in list + + def target_count(self): + return len(self._platforms) * len(self._builds) + + def targets(self): + result = [] + for platform in ExpectationsLine.platforms: + for build in ExpectationsLine.builds: + if self.does_target(platform, build): + result += [(platform, build)] + return result + + def can_add_target(self, platform, build): + platform_count = len(self._platforms) + if not platform in self._platforms: + platform_count += 1 + + build_count = len(self._builds) + if not build in self._builds: + build_count += 1 + + targets_after_add = platform_count * build_count + return targets_after_add == self.target_count() + 1 + + def add_target(self, platform, build): + if not platform in self._platforms: + self._platforms.append(platform) + if not build in self._builds: + self._builds.append(build) + + def _omit_if_all_present(self, values, possible_values): + if possible_values == values: + return set() + return values + + def _sorted_list(self, values): + result = list(values) + result.sort() + return result + + def __str__(self): + builds_and_platforms = [] + builds_and_platforms += self._sorted_list(self._omit_if_all_present(self._platforms, ExpectationsLine.platforms)) + builds_and_platforms += self._sorted_list(self._omit_if_all_present(self._builds, ExpectationsLine.builds)) + + result = '%s : %s = %s' % ( + " ".join(self._sorted_list(self._attributes) + builds_and_platforms + self._sorted_list(self._options)).upper(), + self._test, + " ".join(self._expectations).upper()) + return result + diff --git a/webkit/tools/layout_tests/webkitpy/layout_tests/expectations_line_unittest.py b/webkit/tools/layout_tests/webkitpy/layout_tests/expectations_line_unittest.py new file mode 100644 index 0000000..15456de --- /dev/null +++ b/webkit/tools/layout_tests/webkitpy/layout_tests/expectations_line_unittest.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2010 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unittests for ExpectationsLine.""" + +import os +import sys +import unittest + +from update_expectations_from_dashboard import ExpectationsLine + + +class ExpectationsLineUnittest(unittest.TestCase): + ########################################################################### + # Tests + def setUp(self): + pass + + def test___str___works(self): + input = "BUG1 WIN RELEASE : test.html = FAIL" + line = ExpectationsLine(input) + self.assertEquals(input, str(line)) + + def test_mismatched_attributes_cant_merge(self): + line1 = ExpectationsLine("BUG1 SLOW WIN RELEASE : test.html = FAIL") + line2 = ExpectationsLine("BUG1 WIN DEBUG : test.html = FAIL") + + self.assertFalse(line1.can_merge(line2)) + + def test_mismatched_expectations_cant_merge(self): + line1 = ExpectationsLine("BUG1 WIN RELEASE : test.html = TEXT") + line2 = ExpectationsLine("BUG1 WIN DEBUG : test.html = IMAGE") + + self.assertFalse(line1.can_merge(line2)) + + def test_mismatched_test_names_cant_merge(self): + line1 = ExpectationsLine("BUG1 WIN RELEASE : test1.html = TEXT") + line2 = ExpectationsLine("BUG1 WIN DEBUG : test2.html = TEXT") + + self.assertFalse(line1.can_merge(line2)) + + + def test_targets(self): + input = ExpectationsLine("BUG13907 LINUX MAC : media/video-zoom.html = IMAGE PASS") + + self.assertTrue(('linux', 'release') in input.targets()) + + def test_merges_if_build_is_covered(self): + line1 = ExpectationsLine("BUG1 WIN RELEASE : test.html = FAIL") + line2 = ExpectationsLine("BUG1 WIN DEBUG : test.html = FAIL") + + self.assertTrue(line1.can_merge(line2)) + line1.merge(line2) + self.assertEquals("BUG1 WIN : test.html = FAIL", str(line1)) + +if '__main__' == __name__: + unittest.main() diff --git a/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard.py b/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard.py index 64e1161..45bb1f3 100644 --- a/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard.py +++ b/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard.py @@ -13,9 +13,12 @@ Usage: 3. python update_expectations_from_dashboard.py path/to/local/file """ +import copy import logging import os import sys +import re +from expectations_line import ExpectationsLine # # Find the WebKit python directories and add them to the PYTHONPATH @@ -309,6 +312,18 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile): updates = [candidate] return updates + def _merge(self, expectations): + result = [] + for expectation in expectations: + if not len(result): + result.append(expectation) + elif result[-1].can_merge(expectation): + result[-1].merge(expectation) + else: + result.append(expectation) + + return result + def update_based_on_json(self, update_json): """Updates the expectations based on the update_json, which is of the following form: @@ -317,6 +332,7 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile): "WIN RELEASE": {"extra": "FAIL"} }} """ + unused = copy.deepcopy(update_json) output = [] comment_lines = [] @@ -348,6 +364,7 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile): platform, build_type = build_info.lower().split(' ') if platform in platforms and build_type in build_types: has_updates_for_this_line = True + del(unused[test][build_info]) # If the updates for this test don't apply for the platforms / # build-types listed in this line, then output the line unmodified. @@ -373,6 +390,19 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile): comment_lines, test, deduped_updates) # Append any comment/whitespace lines at the end of test_expectations. output.extend(comment_lines) + + new_expectations = [] + for test in unused: + for build_info in unused[test]: + if 'missing' in unused[test][build_info]: + new_expectations.append(ExpectationsLine("BUG_AUTO %s : %s = %s\n" % (build_info, test, unused[test][build_info]['missing']))) + + new_expectations = self._merge(self._merge(new_expectations)) + + if len(new_expectations): + output += ["\n"] + output += [str(x) + "\n" for x in new_expectations] + return "".join(output) def _write_updates(self, output, comment_lines, test, updates): @@ -480,7 +510,6 @@ class ExpectationsUpdater(test_expectations.TestExpectationsFile): self._write_completed_lines(output, comment_lines, " ".join(line) + "\n") - def main(): logging.basicConfig(level=logging.INFO, format='%(message)s') diff --git a/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard_unittest.py b/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard_unittest.py index a5b7e7a..5e1c76f 100644 --- a/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard_unittest.py +++ b/webkit/tools/layout_tests/webkitpy/layout_tests/update_expectations_from_dashboard_unittest.py @@ -20,7 +20,7 @@ class UpdateExpectationsUnittest(unittest.TestCase): self._port = update_expectations_from_dashboard.get_port() def test_keeps_unmodified_lines(self): - expectations = """// Ensure comments and newlines don't get stripped. + input = """// Ensure comments and newlines don't get stripped. BUG1 SLOW : 1.html = PASS BUG2 : 2.html = FAIL TIMEOUT @@ -33,10 +33,10 @@ class UpdateExpectationsUnittest(unittest.TestCase): """ updates = [] - self.update_expectations(expectations, updates, exp_results) + self.update_expectations(input, updates, exp_results) def test_remove_flaky_expectation(self): - expectations = "BUG1 : 1.html = TIMEOUT FAIL\n" + input = "BUG1 : 1.html = TIMEOUT FAIL\n" expected_results = "BUG1 : 1.html = TIMEOUT\n" updates = {"1.html": { "WIN RELEASE": {"extra": "FAIL"}, @@ -45,10 +45,10 @@ class UpdateExpectationsUnittest(unittest.TestCase): "LINUX DEBUG": {"extra": "FAIL"}, "MAC RELEASE": {"extra": "FAIL"}, "MAC DEBUG": {"extra": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_remove_expectation_slow_test(self): - expectations = "BUG1 SLOW : 1.html = FAIL\n" + input = "BUG1 SLOW : 1.html = FAIL\n" expected_results = "BUG1 SLOW : 1.html = PASS\n" updates = {"1.html": { "WIN RELEASE": {"extra": "FAIL"}, @@ -57,10 +57,10 @@ class UpdateExpectationsUnittest(unittest.TestCase): "LINUX DEBUG": {"extra": "FAIL"}, "MAC RELEASE": {"extra": "FAIL"}, "MAC DEBUG": {"extra": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_remove_expectation(self): - expectations = "BUG1 : 1.html = FAIL\n" + input = "BUG1 : 1.html = FAIL\n" expected_results = "" updates = {"1.html": { "WIN RELEASE": {"extra": "FAIL"}, @@ -69,18 +69,18 @@ class UpdateExpectationsUnittest(unittest.TestCase): "LINUX DEBUG": {"extra": "FAIL"}, "MAC RELEASE": {"extra": "FAIL"}, "MAC DEBUG": {"extra": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_remove_expectation_from_one_platform(self): - expectations = "BUG1 : 1.html = FAIL\n" + input = "BUG1 : 1.html = FAIL\n" expected_results = """BUG1 MAC WIN DEBUG : 1.html = FAIL BUG1 RELEASE : 1.html = FAIL """ updates = {"1.html": {"LINUX DEBUG": {"extra": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_remove_slow(self): - expectations = "BUG1 SLOW : 1.html = PASS\n" + input = "BUG1 SLOW : 1.html = PASS\n" expected_results = "" updates = {"1.html": { "WIN RELEASE": {"extra": "SLOW"}, @@ -89,10 +89,10 @@ BUG1 RELEASE : 1.html = FAIL "LINUX DEBUG": {"extra": "SLOW"}, "MAC RELEASE": {"extra": "SLOW"}, "MAC DEBUG": {"extra": "SLOW"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_flaky_expectation(self): - expectations = "BUG1 : 1.html = TIMEOUT\n" + input = "BUG1 : 1.html = TIMEOUT\n" expected_results = "BUG1 : 1.html = TIMEOUT FAIL\n" updates = {"1.html": { "WIN RELEASE": {"missing": "FAIL"}, @@ -101,10 +101,10 @@ BUG1 RELEASE : 1.html = FAIL "LINUX DEBUG": {"missing": "FAIL"}, "MAC RELEASE": {"missing": "FAIL"}, "MAC DEBUG": {"missing": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_expectation_slow_test(self): - expectations = "BUG1 SLOW : 1.html = PASS\n" + input = "BUG1 SLOW : 1.html = PASS\n" expected_results = "BUG1 SLOW : 1.html = PASS FAIL\n" updates = {"1.html": { "WIN RELEASE": {"missing": "FAIL"}, @@ -113,13 +113,13 @@ BUG1 RELEASE : 1.html = FAIL "LINUX DEBUG": {"missing": "FAIL"}, "MAC RELEASE": {"missing": "FAIL"}, "MAC DEBUG": {"missing": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_expectation(self): # not yet implemented return - expectations = "" + input = "" expected_results = "BUG1 : 1.html = FAIL\n" updates = {"1.html": { "WIN RELEASE": {"missing": "FAIL"}, @@ -128,21 +128,19 @@ BUG1 RELEASE : 1.html = FAIL "LINUX DEBUG": {"missing": "FAIL"}, "MAC RELEASE": {"missing": "FAIL"}, "MAC DEBUG": {"missing": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_expectation_for_one_platform(self): - expectations = "BUG1 WIN : 1.html = TIMEOUT\n" - expected_results = "BUG1 WIN : 1.html = TIMEOUT\n" - # TODO(ojan): Once we add currently unlisted tests, this expect results - # for this test should be: - #expected_results = """BUG1 WIN : 1.html = TIMEOUT - #BUG_AUTO LINUX DEBUG : 1.html = TIMEOUT - #""" + input = "BUG1 WIN : 1.html = TIMEOUT\n" + expected_results = """BUG1 WIN : 1.html = TIMEOUT + +BUG_AUTO LINUX DEBUG : 1.html = TIMEOUT +""" updates = {"1.html": {"LINUX DEBUG": {"missing": "TIMEOUT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_slow(self): - expectations = "BUG1 : 1.html = FAIL\n" + input = "BUG1 : 1.html = FAIL\n" expected_results = "BUG1 SLOW : 1.html = FAIL\n" updates = {"1.html": { "WIN RELEASE": {"missing": "SLOW"}, @@ -151,20 +149,17 @@ BUG1 RELEASE : 1.html = FAIL "LINUX DEBUG": {"missing": "SLOW"}, "MAC RELEASE": {"missing": "SLOW"}, "MAC DEBUG": {"missing": "SLOW"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_remove_multiple_expectations(self): - expectations = """BUG1 WIN : 1.html = FAIL + input = """BUG1 WIN : 1.html = FAIL BUG2 MAC : 1.html = FAIL""" expected_results = """BUG1 SLOW WIN : 1.html = FAIL -BUG2 MAC : 1.html = TIMEOUT\n""" - # TODO(ojan): Once we add currently unlisted tests, this expect results - # for this test should be: - #expected_results = """BUG1 SLOW WIN : 1.html = FAIL - #BUG_AUTO LINUX SLOW : 1.html = PASS - #BUG2 MAC : 1.html = TIMEOUT - #""" +BUG2 MAC : 1.html = TIMEOUT +BUG_AUTO LINUX : 1.html = SLOW +""" + # TODO this test is wrong - SLOW should go in the 'attributes' section of the generated expectation. updates = {"1.html": { "WIN RELEASE": {"missing": "SLOW"}, "WIN DEBUG": {"missing": "SLOW"}, @@ -172,16 +167,16 @@ BUG2 MAC : 1.html = TIMEOUT\n""" "LINUX DEBUG": {"missing": "SLOW"}, "MAC RELEASE": {"missing": "TIMEOUT", "extra": "FAIL"}, "MAC DEBUG": {"missing": "TIMEOUT", "extra": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_existing_expectation(self): - expectations = "BUG1 : 1.html = FAIL\n" + input = "BUG1 : 1.html = FAIL\n" expected_results = "BUG1 : 1.html = FAIL\n" updates = {"1.html": {"WIN RELEASE": {"missing": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_image_or_text_to_fail_expectation(self): - expectations = """BUG1 WIN RELEASE : 1.html = FAIL + input = """BUG1 WIN RELEASE : 1.html = FAIL BUG1 MAC RELEASE : 1.html = FAIL BUG1 LINUX RELEASE : 1.html = FAIL BUG1 LINUX DEBUG : 1.html = TIMEOUT @@ -197,54 +192,55 @@ BUG1 LINUX DEBUG : 1.html = TIMEOUT IMAGE+TEXT "MAC RELEASE": {"missing": "IMAGE"}, "LINUX RELEASE": {"missing": "TEXT"}, "LINUX DEBUG": {"missing": "IMAGE+TEXT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_other(self): - # Other is a catchall for more obscure expectations results. + # Other is a catchall for more obscure input results. # We should never add it to test_expectations. - expectations = "BUG1 WIN RELEASE : 1.html = FAIL\n" + input = "BUG1 WIN RELEASE : 1.html = FAIL\n" expected_results = "BUG1 WIN RELEASE : 1.html = FAIL\n" updates = {"1.html": {"WIN RELEASE": {"missing": "OTHER"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_remove_non_existant_expectation(self): - expectations = "BUG1 : 1.html = FAIL\n" + input = "BUG1 : 1.html = FAIL\n" expected_results = "BUG1 : 1.html = FAIL\n" updates = {"1.html": {"WIN RELEASE": {"extra": "TIMEOUT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_update_some_platforms(self): - expectations = "BUG1 DEBUG : 1.html = TEXT PASS\n" - # TODO(ojan): Once we add currently unlisted tests, the expect results - # for this test should include the missing bits for RELEASE. - expected_results = "BUG1 LINUX DEBUG : 1.html = TEXT PASS\n" + input = "BUG1 DEBUG : 1.html = TEXT PASS\n" + expected_results = """BUG1 LINUX DEBUG : 1.html = TEXT PASS + +BUG_AUTO MAC WIN RELEASE : 1.html = PASS TEXT +""" updates = {"1.html": { "WIN RELEASE": {"missing": "PASS TEXT"}, "WIN DEBUG": {"extra": "MISSING TEXT"}, "MAC RELEASE": {"missing": "PASS TEXT"}, "MAC DEBUG": {"extra": "MISSING TEXT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_timeout_to_slow_test(self): # SLOW tests needing TIMEOUT need manual updating. Should just print # a log and not modify the test. - expectations = "BUG1 SLOW : 1.html = TEXT\n" + input = "BUG1 SLOW : 1.html = TEXT\n" expected_results = "BUG1 SLOW : 1.html = TEXT\n" updates = {"1.html": {"WIN RELEASE": {"missing": "TIMEOUT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_add_slow_to_timeout_test(self): # SLOW tests needing TIMEOUT need manual updating. Should just print # a log and not modify the test. - expectations = "BUG1 : 1.html = TIMEOUT\n" + input = "BUG1 : 1.html = TIMEOUT\n" expected_results = "BUG1 : 1.html = TIMEOUT\n" updates = {"1.html": {"WIN RELEASE": {"missing": "SLOW"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_include_last_platform_in_flakiness(self): - # If a test is flaky on 5/6 platforms and the 6th's expectations are a - # subset of the other 5/6, then give them all the same expectations. - expectations = "BUG2 : 1.html = FAIL\n" + # If a test is flaky on 5/6 platforms and the 6th's input are a + # subset of the other 5/6, then give them all the same input. + input = "BUG2 : 1.html = FAIL\n" expected_results = "BUG2 : 1.html = FAIL TIMEOUT\n" updates = {"1.html": { "WIN RELEASE": {"missing": "TIMEOUT", "extra": "FAIL"}, @@ -253,24 +249,24 @@ BUG1 LINUX DEBUG : 1.html = TIMEOUT IMAGE+TEXT "LINUX DEBUG": {"missing": "TIMEOUT"}, "MAC RELEASE": {"missing": "TIMEOUT"}, "MAC DEBUG": {"missing": "TIMEOUT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_include_last_platform_in_flakiness_three_out_of_four(self): - # If a test is flaky on 5/6 platforms and the 6th's expectations are a - # subset of the other 5/6, then give them all the same expectations. - expectations = "BUG2 MAC LINUX : 1.html = FAIL\n" + # If a test is flaky on 5/6 platforms and the 6th's input are a + # subset of the other 5/6, then give them all the same input. + input = "BUG2 MAC LINUX : 1.html = FAIL\n" expected_results = "BUG2 LINUX MAC : 1.html = FAIL TIMEOUT\n" updates = {"1.html": { "LINUX RELEASE": {"missing": "TIMEOUT"}, "MAC RELEASE": {"missing": "TIMEOUT"}, "MAC DEBUG": {"missing": "TIMEOUT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_exclude_last_platform_from_flakiness(self): - # If a test is flaky on 5/6 platforms and the 6th's expectations + # If a test is flaky on 5/6 platforms and the 6th's input # are not a subset of the other 5/6, then don't give them - # all the same expectations. - expectations = "BUG1 : 1.html = FAIL\n" + # all the same input. + input = "BUG1 : 1.html = FAIL\n" expected_results = """BUG1 DEBUG : 1.html = FAIL TIMEOUT BUG1 LINUX MAC RELEASE : 1.html = FAIL TIMEOUT BUG1 WIN RELEASE : 1.html = FAIL CRASH @@ -282,10 +278,10 @@ BUG1 WIN RELEASE : 1.html = FAIL CRASH "LINUX DEBUG": {"missing": "TIMEOUT"}, "MAC RELEASE": {"missing": "TIMEOUT"}, "MAC DEBUG": {"missing": "TIMEOUT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_strip_comments(self): - expectations = """BUG1 : 1.html = TIMEOUT + input = """BUG1 : 1.html = TIMEOUT // Comment/whitespace should be removed when the test is. BUG2 WIN RELEASE : 2.html = TEXT @@ -308,10 +304,10 @@ BUG2 MAC DEBUG : 2.html = TEXT updates = {"2.html": { "WIN RELEASE": {"extra": "TEXT"}, "MAC RELEASE": {"extra": "TEXT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_leave_comments(self): - expectations = """BUG1 : 1.html = TIMEOUT + input = """BUG1 : 1.html = TIMEOUT // Comment/whitespace should remain. BUG2 : 2.html = FAIL PASS @@ -327,10 +323,10 @@ BUG2 LINUX MAC RELEASE : 2.html = FAIL PASS "WIN RELEASE": {"extra": "FAIL"}, "WIN DEBUG": {"extra": "FAIL"}, "LINUX DEBUG": {"extra": "FAIL"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_leave_comments_if_no_whitespace_after_test(self): - expectations = """// Comment/whitespace should remain. + input = """// Comment/whitespace should remain. BUG2 WIN RELEASE : 2.html = TEXT BUG2 : 1.html = IMAGE """ @@ -339,21 +335,70 @@ BUG2 LINUX MAC RELEASE : 2.html = FAIL PASS """ updates = {"2.html": {"WIN RELEASE": {"extra": "TEXT"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) def test_leaves_unmodified_expectations_untouched(self): # Ensures tests that would just change sort order of a line are noops. - expectations = "BUG1 WIN LINUX : 1.html = TIMEOUT\n" - expected_results = "BUG1 WIN LINUX : 1.html = TIMEOUT\n" + input = "BUG1 WIN LINUX : 1.html = TIMEOUT\n" + expected_results = """BUG1 WIN LINUX : 1.html = TIMEOUT + +BUG_AUTO MAC RELEASE : 1.html = SLOW +""" updates = {"1.html": {"MAC RELEASE": {"missing": "SLOW"}}} - self.update_expectations(expectations, updates, expected_results) + self.update_expectations(input, updates, expected_results) + + def test_adding_an_expectation_to_an_empty_expectations_file(self): + input = "" + expected_results = "\nBUG_AUTO MAC DEBUG : 1.html = TIMEOUT\n" + updates = {"1.html": { + "MAC DEBUG": {"missing": "TIMEOUT"}}} + self.update_expectations(input, updates, expected_results) + + def test_adding_an_expectation_to_a_non_empty_expectations_file(self): + input = "BUG1 : 1.html = FAIL\n" + expected_results = (input + "\n" + + "BUG_AUTO MAC DEBUG : 2.html = TIMEOUT\n") + updates = {"2.html": { + "MAC DEBUG": {"missing": "TIMEOUT"}}} + self.update_expectations(input, updates, expected_results) + + def test_adding_a_new_expectation_and_modifying_existing_expectations(self): + input = "BUG1 : 1.html = FAIL\n" + expected_results = """BUG1 : 1.html = FAIL TIMEOUT + +BUG_AUTO MAC DEBUG : 2.html = TIMEOUT +""" + updates = { + "2.html": { + "MAC DEBUG": {"missing": "TIMEOUT"} + }, + "1.html": { + "WIN RELEASE": {"missing": "TIMEOUT"}, + "WIN DEBUG": {"missing": "TIMEOUT"}, + "LINUX RELEASE": {"missing": "TIMEOUT"}, + "LINUX DEBUG": {"missing": "TIMEOUT"}, + "MAC RELEASE": {"missing": "TIMEOUT"}, + "MAC DEBUG": {"missing": "TIMEOUT"} + }, + } + self.update_expectations(input, updates, expected_results) + + def test_new_expectations_get_merged(self): + input = "BUG1 : 1.html = FAIL\n" + expected_results = (input + "\n" + + "BUG_AUTO LINUX WIN DEBUG : test.html = TEXT\n") + updates = { "test.html": + {"WIN DEBUG": {"missing":"TEXT"}, + "LINUX DEBUG":{"missing":"TEXT"}}} + + self.update_expectations(input, updates, expected_results) ########################################################################### # Helper functions - def update_expectations(self, expectations, updates, expected_results): + def update_expectations(self, input, updates, expected_results): results = update_expectations_from_dashboard.update_expectations( - self._port, expectations, updates) + self._port, input, updates) self.assertEqual(expected_results, results) if '__main__' == __name__: |