diff options
4 files changed, 120 insertions, 19 deletions
diff --git a/media/tools/layout_tests/layouttest_analyzer.py b/media/tools/layout_tests/layouttest_analyzer.py index 7d6ae2a..5df6c161 100644 --- a/media/tools/layout_tests/layouttest_analyzer.py +++ b/media/tools/layout_tests/layouttest_analyzer.py @@ -21,6 +21,8 @@ from trend_graph import TrendGraph DEFAULT_RESULT_DIR = 'result' DEFAULT_ANNO_FILE = os.path.join('anno', 'anno.csv') DEFAULT_GRAPH_FILE = os.path.join('graph', 'graph.html') +DEFAULT_STATS_CSV_FILENAME = 'stats.csv' +DEFAULT_ISSUES_CSV_FILENAME = 'issues.csv' # Predefined result files for debug. CUR_TIME_FOR_DEBUG = '2011-09-11-19' CURRENT_RESULT_FILE_FOR_DEBUG = os.path.join(DEFAULT_RESULT_DIR, @@ -196,7 +198,8 @@ def ReadEmailInformation(bug_annotation_file_location, else: data = csv.reader(file_object) for row in data: - anno_map[row[0]] = row[1] + if len(row) > 1: + anno_map[row[0]] = row[1] file_object.close() appended_text_to_email = '' @@ -273,7 +276,8 @@ def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map, layouttest_analyzer_helpers.SendStatusEmail( prev_time, analyzer_result_map, diff_map, anno_map, receiver_email_address, test_group_name, - appended_text_to_email, email_content, rev_str) + appended_text_to_email, email_content, rev_str, + email_only_change_mode) if simple_rev_str: simple_rev_str = '\'' + simple_rev_str + '\'' else: @@ -284,6 +288,8 @@ def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map, result_change = True diff_map = None simple_rev_str = 'undefined' + email_content = analyzer_result_map.ConvertToString(None, diff_map, + anno_map) return (result_change, diff_map, simple_rev_str, rev, rev_date, email_content) @@ -454,6 +460,19 @@ def main(): anno_map, appended_text_to_email, options.email_only_change_mode, options.debug, options.receiver_email_address, options.test_group_name)) + + # Create CSV texts and save them for bug spreadsheet. + (stats, issues_txt) = analyzer_result_map.ConvertToCSVText( + start_time.strftime('%Y-%m-%d-%H')) + file_object = open(os.path.join(options.result_directory_location, + DEFAULT_STATS_CSV_FILENAME), 'wb') + file_object.write(stats) + file_object.close() + file_object = open(os.path.join(options.result_directory_location, + DEFAULT_ISSUES_CSV_FILENAME), 'wb') + file_object.write(issues_txt) + file_object.close() + if not options.debug and (result_change or not prev_analyzer_result_map): # Save the current result when result is changed or the script is # executed for the first time. diff --git a/media/tools/layout_tests/layouttest_analyzer_helpers.py b/media/tools/layout_tests/layouttest_analyzer_helpers.py index 44d4496..18d844e 100644 --- a/media/tools/layout_tests/layouttest_analyzer_helpers.py +++ b/media/tools/layout_tests/layouttest_analyzer_helpers.py @@ -11,6 +11,7 @@ from email.mime.text import MIMEText import fileinput import os import pickle +import re import smtplib import socket import sys @@ -132,6 +133,48 @@ class AnalyzerResultMap: 'less than that of "skip"') return 100 - len(self.result_map['nonskip'].keys()) * 100 / delta + def ConvertToCSVText(self, current_time): + """Convert |self.result_map| into stats and issues text in CSV format. + + Both are used as inputs for Google spreadsheet. + + Args: + current_time: a string depicting a time in year-month-day-hour + format (e.g., 2011-11-08-16). + + Returns: + a tuple of stats and issues_txt + stats: analyzer result in CSV format that shows: + (current_time, the number of tests, the number of skipped tests, + the number of failing tests) + For example, + "2011-11-10-15,204,22,12" + issues_txt: issues listed in CSV format that shows: + (BUGWK or BUGCR, bug number, the test expectation entry, + the name of the test) + For example, + "BUGWK,71543,TIMEOUT PASS,media/media-element-play-after-eos.html, + BUGCR,97657,IMAGE CPU MAC TIMEOUT PASS,media/audio-repaint.html," + """ + stats = ','.join([current_time, str(len(self.result_map['whole'].keys())), + str(len(self.result_map['skip'].keys())), + str(len(self.result_map['nonskip'].keys()))]) + issues_txt = '' + for bug_txt, test_info_list in ( + self.GetListOfBugsForNonSkippedTests().iteritems()): + matches = re.match(r'(BUG(CR|WK))(\d+)', bug_txt) + bug_suffix = '' + bug_no = '' + if matches: + bug_suffix = matches.group(1) + bug_no = matches.group(3) + issues_txt += bug_suffix + ',' + bug_no + ',' + for test_info in test_info_list: + test_name, te_info = test_info + issues_txt += ' '.join(te_info.keys()) + ',' + test_name + ',' + issues_txt += '\n' + return stats, issues_txt + def ConvertToString(self, prev_time, diff_map, bug_anno_map): """Convert this result to HTML display for email. @@ -144,19 +187,21 @@ class AnalyzerResultMap: Returns: a analyzer result string in HTML format. """ - return_str = ('<b>Statistics (Diff Compared to %s):</b><ul>' - '<li>The number of tests: %d (%s)</li>' - '<li>The number of failing skipped tests: %d (%s)</li>' - '<li>The number of failing non-skipped tests: %d (%s)</li>' - '<li>Passing rate: %d %%</li></ul>') % ( - prev_time, len(self.result_map['whole'].keys()), - AnalyzerResultMap.GetDiffString(diff_map['whole'], 'whole'), - len(self.result_map['skip'].keys()), - AnalyzerResultMap.GetDiffString(diff_map['skip'], 'skip'), - len(self.result_map['nonskip'].keys()), - AnalyzerResultMap.GetDiffString(diff_map['nonskip'], - 'nonskip'), - self.GetPassingRate()) + return_str = '' + if diff_map: + return_str += ('<b>Statistics (Diff Compared to %s):</b><ul>' + '<li>The number of tests: %d (%s)</li>' + '<li>The number of failing skipped tests: %d (%s)</li>' + '<li>The number of failing non-skipped tests: %d (%s)</li>' + '<li>Passing rate: %d %%</li></ul>') % ( + prev_time, len(self.result_map['whole'].keys()), + AnalyzerResultMap.GetDiffString(diff_map['whole'], 'whole'), + len(self.result_map['skip'].keys()), + AnalyzerResultMap.GetDiffString(diff_map['skip'], 'skip'), + len(self.result_map['nonskip'].keys()), + AnalyzerResultMap.GetDiffString(diff_map['nonskip'], + 'nonskip'), + self.GetPassingRate()) return_str += '<b>Current issues about failing non-skipped tests:</b>' for (bug_txt, test_info_list) in ( self.GetListOfBugsForNonSkippedTests().iteritems()): @@ -255,7 +300,8 @@ class AnalyzerResultMap: def SendStatusEmail(prev_time, analyzer_result_map, diff_map, bug_anno_map, receiver_email_address, test_group_name, - appended_text_to_email, email_content, rev_str): + appended_text_to_email, email_content, rev_str, + email_only_change_mode): """Send status email. Args: @@ -285,14 +331,19 @@ def SendStatusEmail(prev_time, analyzer_result_map, diff_map, rev_str: a revision string that contains revision information that is sent out in the status email. It is obtained by calling |GetRevisionString()|. + email_only_change_mode: please refer to |options|. """ if rev_str: email_content += '<br><b>Revision Information:</b>' email_content += rev_str localtime = time.asctime(time.localtime(time.time())) + change_str = '' + if email_only_change_mode: + change_str = 'Status Change ' + subject = 'Layout Test Analyzer Result %s(%s): %s' % (change_str, + test_group_name, + localtime) # TODO(imasaki): remove my name from here. - subject = 'Layout Test Analyzer Result (%s): %s' % (test_group_name, - localtime) SendEmail('imasaki@chromium.org', [receiver_email_address], subject, email_content + appended_text_to_email) diff --git a/media/tools/layout_tests/layouttest_analyzer_helpers_unittest.py b/media/tools/layout_tests/layouttest_analyzer_helpers_unittest.py index 2c399a1..2301d52 100644 --- a/media/tools/layout_tests/layouttest_analyzer_helpers_unittest.py +++ b/media/tools/layout_tests/layouttest_analyzer_helpers_unittest.py @@ -183,6 +183,28 @@ class TestLayoutTestAnalyzerHelpers(unittest.TestCase): self.assertFalse( layouttest_analyzer_helpers.FindLatestResult('test_data')) + def testConvertToCSVText(self): + file_path = os.path.join('test_data', 'base') + analyzerResultMapBase = ( + layouttest_analyzer_helpers.AnalyzerResultMap.Load(file_path)) + data, issues_txt = analyzerResultMapBase.ConvertToCSVText('11-10-10-2011') + self.assertEquals(data, '11-10-10-2011,204,36,10') + expected_issues_txt = """\ +BUGWK,66310,TEXT PASS,media/media-blocked-by-beforeload.html,DEBUG TEXT PASS,\ +media/video-source-error.html, +BUGCR,86714,GPU IMAGE CRASH MAC,media/video-zoom.html,GPU IMAGE CRASH MAC,\ +media/video-controls-rendering.html, +BUGCR,74102,GPU IMAGE PASS LINUX,media/video-controls-rendering.html, +BUGWK,55718,TEXT IMAGE IMAGE+TEXT,media/media-document-audio-repaint.html, +BUGCR,78376,TIMEOUT,http/tests/media/video-play-stall-seek.html, +BUGCR,59415,WIN TEXT TIMEOUT PASS,media/video-loop.html, +BUGCR,72223,IMAGE PASS,media/video-frame-accurate-seek.html, +BUGCR,75354,TEXT IMAGE IMAGE+TEXT,media/media-document-audio-repaint.html, +BUGCR,73609,TEXT,http/tests/media/video-play-stall.html, +BUGWK,64003,DEBUG TEXT MAC PASS,media/video-delay-load-event.html, +""" + self.assertEquals(issues_txt, expected_issues_txt) + if __name__ == '__main__': unittest.main() diff --git a/media/tools/layout_tests/layouttest_analyzer_runner.py b/media/tools/layout_tests/layouttest_analyzer_runner.py index bb0966b..ae545d1 100644 --- a/media/tools/layout_tests/layouttest_analyzer_runner.py +++ b/media/tools/layout_tests/layouttest_analyzer_runner.py @@ -68,6 +68,13 @@ def ParseOption(): '(default to %default and no text is ' 'appended in that case.)'), default=None) + option_parser.add_option('-e', '--email-only-change-mode', + dest='email_only_change_mode', + help=('With this mode, email is sent out ' + 'only when there is a change in the ' + 'analyzer result compared to the previous ' + 'result (off by default)'), + action='store_true', default=False) return option_parser.parse_args()[0] @@ -176,7 +183,7 @@ def main(): anno_file = os.path.join(options.annotation_directory_location, test_group_name_for_data + '.csv') cmd = ('python layouttest_analyzer.py -x %s -d %s -t %s' - ' -q %s -a %s -c ') % ( + ' -q %s -a %s ') % ( test_group, result_dir, graph_file, dashboard_file_location, anno_file) if run_config_map[test_group][0]: @@ -185,6 +192,8 @@ def main(): cmd += '-r ' + run_config_map[test_group][1] + ' ' if options.email_appended_text_file_location: cmd += ' -b ' + options.email_appended_text_file_location + if options.email_only_change_mode: + cmd += ' -c ' print 'Running ' + cmd proc = Popen(cmd, shell=True) proc.communicate() |