# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import unittest from auto_bisect import source_control as source_control_module # Special import necessary because filename contains dash characters. bisect_perf_module = __import__('bisect-perf-regression') class BisectPerfRegressionTest(unittest.TestCase): """Test case for other functions and classes in bisect-perf-regression.py.""" def _AssertConfidence(self, score, bad_values, good_values): """Checks whether the given sets of values have a given confidence score. The score represents our confidence that the two sets of values wouldn't be as different as they are just by chance; that is, that some real change occurred between the two sets of values. Args: score: Expected confidence score. bad_values: First list of numbers. good_values: Second list of numbers. """ # ConfidenceScore takes a list of lists but these lists are flattened # inside the function. confidence = bisect_perf_module.ConfidenceScore( [[v] for v in bad_values], [[v] for v in good_values]) self.assertEqual(score, confidence) def testConfidenceScore_ZeroConfidence(self): # The good and bad sets contain the same values, so the confidence that # they're different should be zero. self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) def testConfidenceScore_MediumConfidence(self): self._AssertConfidence(80.0, [0, 1, 1, 1, 2, 2], [1, 1, 1, 3, 3, 4]) def testConfidenceScore_HighConfidence(self): self._AssertConfidence(95.0, [0, 1, 1, 1, 2, 2], [1, 2, 2, 3, 3, 4]) def testConfidenceScore_VeryHighConfidence(self): # Confidence is high if the two sets of values have no internal variance. self._AssertConfidence(99.9, [1, 1, 1, 1], [1.2, 1.2, 1.2, 1.2]) self._AssertConfidence(99.9, [1, 1, 1, 1], [1.01, 1.01, 1.01, 1.01]) def testConfidenceScore_UnbalancedSampleSize(self): # The second set of numbers only contains one number, so confidence is 0. self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2], [1.4]) def testConfidenceScore_EmptySample(self): # Confidence is zero if either or both samples are empty. self._AssertConfidence(0.0, [], []) self._AssertConfidence(0.0, [], [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3]) self._AssertConfidence(0.0, [1.1, 1.2, 1.1, 1.2, 1.0, 1.3, 1.2, 1.3], []) def testConfidenceScore_FunctionalTestResults(self): self._AssertConfidence(80.0, [1, 1, 0, 1, 1, 1, 0, 1], [0, 0, 1, 0, 1, 0]) self._AssertConfidence(99.9, [1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0]) def testParseDEPSStringManually(self): """Tests DEPS parsing.""" deps_file_contents = """ vars = { 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', 'webkit_url': 'https://chromium.googlesource.com/chromium/blink.git', 'git_url': 'https://chromium.googlesource.com', 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' }""" # Should only expect SVN/git revisions to come through, and URLs should be # filtered out. expected_vars_dict = { 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' } # Testing private function. # pylint: disable=W0212 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( deps_file_contents) self.assertEqual(vars_dict, expected_vars_dict) def _AssertParseResult(self, expected_values, result_string): """Asserts some values are parsed from a RESULT line.""" results_template = ('RESULT other_chart: other_trace= 123 count\n' 'RESULT my_chart: my_trace= %(value)s\n') results = results_template % {'value': result_string} metric = ['my_chart', 'my_trace'] # Testing private function. # pylint: disable=W0212 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) self.assertEqual(expected_values, values) def testTryParseResultValuesFromOutput_WithSingleValue(self): """Tests result pattern <*>RESULT : = """ self._AssertParseResult([66.88], '66.88 kb') self._AssertParseResult([66.88], '66.88 ') self._AssertParseResult([-66.88], '-66.88 kb') self._AssertParseResult([66], '66 kb') self._AssertParseResult([0.66], '.66 kb') self._AssertParseResult([], '. kb') self._AssertParseResult([], 'aaa kb') def testTryParseResultValuesFromOutput_WithMultiValue(self): """Tests result pattern <*>RESULT : = [,, ..]""" self._AssertParseResult([66.88], '[66.88] kb') self._AssertParseResult([66.88, 99.44], '[66.88, 99.44]kb') self._AssertParseResult([66.88, 99.44], '[ 66.88, 99.44 ]') self._AssertParseResult([-66.88, 99.44], '[-66.88, 99.44] kb') self._AssertParseResult([-66, 99], '[-66,99] kb') self._AssertParseResult([-66, 99], '[-66,99,] kb') self._AssertParseResult([-66, 0.99], '[-66,.99] kb') self._AssertParseResult([], '[] kb') self._AssertParseResult([], '[-66,abc] kb') def testTryParseResultValuesFromOutputWithMeanStd(self): """Tests result pattern <*>RESULT : = {= r276728, only android-chrome-shell can be used. self._AssertCompatibleCommand( 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', 276628, 'android') def testGetCompatibleCommand_NoChange(self): # For revisions < r276728, android-chromium-testshell can be used. self._AssertCompatibleCommand( 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', 274858, 'android') # For revisions > r274857, android-chrome-shell can be used. self._AssertCompatibleCommand( 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', 274858, 'android') def testGetCompatibleCommand_NonAndroidPlatform(self): # In most cases, there's no need to change Telemetry command. # For revisions >= r276728, only android-chrome-shell can be used. self._AssertCompatibleCommand( 'tools/perf/run_benchmark -v --browser=release foo', 'tools/perf/run_benchmark -v --browser=release foo', 276628, 'chromium') # This method doesn't reference self; it fails if an error is thrown. # pylint: disable=R0201 def testDryRun(self): """Does a dry run of the bisect script. This serves as a smoke test to catch errors in the basic execution of the script. """ options_dict = { 'debug_ignore_build': True, 'debug_ignore_sync': True, 'debug_ignore_perf_test': True, 'command': 'fake_command', 'metric': 'fake/metric', 'good_revision': 280000, 'bad_revision': 280005, } bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) source_control = source_control_module.DetermineAndCreateSourceControl( bisect_options) bisect_instance = bisect_perf_module.BisectPerformanceMetrics( source_control, bisect_options) results = bisect_instance.Run(bisect_options.command, bisect_options.bad_revision, bisect_options.good_revision, bisect_options.metric) bisect_instance.FormatAndPrintResults(results) if __name__ == '__main__': unittest.main()