diff options
| -rw-r--r-- | tools/perf/page_sets/top_25.json | 8 | ||||
| -rw-r--r-- | tools/perf/perf_tools/memory_benchmark.py | 26 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/all_page_interactions.py | 20 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/browser_options.py | 4 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/click_to_navigate_interaction.py | 4 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/compound_interaction.py | 15 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/compound_interaction_unittest.py | 5 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/discover.py | 40 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/inspector_page_unittest.py | 7 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/multi_page_benchmark.py | 6 | ||||
| -rwxr-xr-x | tools/telemetry/telemetry/multi_page_benchmark_runner.py | 47 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/multi_page_benchmark_unittest.py | 24 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/multi_page_benchmark_unittest_base.py | 2 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/page_interaction.py | 11 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/page_runner.py | 7 | ||||
| -rw-r--r-- | tools/telemetry/telemetry/page_test.py | 45 |
16 files changed, 197 insertions, 74 deletions
diff --git a/tools/perf/page_sets/top_25.json b/tools/perf/page_sets/top_25.json index 7ffa9f9..8ae04c3 100644 --- a/tools/perf/page_sets/top_25.json +++ b/tools/perf/page_sets/top_25.json @@ -61,18 +61,18 @@ "credentials": "facebook", "scroll_is_infinite": true, "stress_memory": { - "action": "compound", + "action": "compound_interaction", "actions": [ { - "action": "click_to_navigate", + "action": "click_to_navigate_interaction", "selector": "a[href=\"http://www.facebook.com/WomenforObama\"]" }, { - "action": "click_to_navigate", + "action": "click_to_navigate_interaction", "selector": "a[href=\"http://www.facebook.com/WomenforObama/info\"]" }, { - "action": "click_to_navigate", + "action": "click_to_navigate_interaction", "selector": "a[href=\"http://www.facebook.com/?ref=tn_tnmn\"]" } ] diff --git a/tools/perf/perf_tools/memory_benchmark.py b/tools/perf/perf_tools/memory_benchmark.py new file mode 100644 index 0000000..c3e974b --- /dev/null +++ b/tools/perf/perf_tools/memory_benchmark.py @@ -0,0 +1,26 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +from telemetry import multi_page_benchmark + +MEMORY_HISTOGRAMS = [ + {'name': 'V8.MemoryExternalFragmentationTotal', 'units': 'percent'}, + {'name': 'V8.MemoryHeapSampleTotalCommitted', 'units': 'kb'}, + {'name': 'V8.MemoryHeapSampleTotalUsed', 'units': 'kb'}] + +class MemoryBenchmark(multi_page_benchmark.MultiPageBenchmark): + def __init__(self): + super(MemoryBenchmark, self).__init__('stress_memory') + + def CustomizeBrowserOptions(self, options): + options.AppendExtraBrowserArg('--dom-automation') + + def CanRunForPage(self, page): + return hasattr(page, 'stress_memory') + + def MeasurePage(self, page, tab, results): + for histogram in MEMORY_HISTOGRAMS: + name = histogram['name'] + data = tab.runtime.Evaluate( + 'window.domAutomationController.getHistogram("%s")' % name) + results.Add(name, histogram['units'], data, data_type='histogram') diff --git a/tools/telemetry/telemetry/all_page_interactions.py b/tools/telemetry/telemetry/all_page_interactions.py new file mode 100644 index 0000000..1e7d29f --- /dev/null +++ b/tools/telemetry/telemetry/all_page_interactions.py @@ -0,0 +1,20 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +import os + +from telemetry import discover +from telemetry import page_interaction + +_page_interaction_classes = discover.Discover(os.path.dirname(__file__), + 'interaction', + page_interaction.PageInteraction) + +def GetAllClasses(): + return list(_page_interaction_classes.values()) + +def FindClassWithName(name): + return _page_interaction_classes.get(name) + +def RegisterClassForTest(name, clazz): + _page_interaction_classes[name] = clazz diff --git a/tools/telemetry/telemetry/browser_options.py b/tools/telemetry/telemetry/browser_options.py index 25eff1f..01601c4 100644 --- a/tools/telemetry/telemetry/browser_options.py +++ b/tools/telemetry/telemetry/browser_options.py @@ -146,3 +146,7 @@ class BrowserOptions(optparse.Values): return ret parser.parse_args = ParseArgs return parser + + def AppendExtraBrowserArg(self, arg): + if arg not in self.extra_browser_args: + self.extra_browser_args.append(arg) diff --git a/tools/telemetry/telemetry/click_to_navigate_interaction.py b/tools/telemetry/telemetry/click_to_navigate_interaction.py index 0360fd8..3eafcf8 100644 --- a/tools/telemetry/telemetry/click_to_navigate_interaction.py +++ b/tools/telemetry/telemetry/click_to_navigate_interaction.py @@ -13,6 +13,4 @@ class ClickToNavigateInteraction(page_interaction.PageInteraction): def DoClick(): tab.runtime.Execute(code) tab.page.PerformActionAndWaitForNavigate(DoClick) - - -page_interaction.RegisterClass('click_to_navigate', ClickToNavigateInteraction) + tab.WaitForDocumentReadyStateToBeComplete() diff --git a/tools/telemetry/telemetry/compound_interaction.py b/tools/telemetry/telemetry/compound_interaction.py index 2ff48b0..5e93cf8 100644 --- a/tools/telemetry/telemetry/compound_interaction.py +++ b/tools/telemetry/telemetry/compound_interaction.py @@ -6,12 +6,17 @@ from telemetry import page_interaction class CompoundInteraction(page_interaction.PageInteraction): def __init__(self, attributes=None): super(CompoundInteraction, self).__init__(attributes) - - def PerformInteraction(self, page, tab): + self._interaction_list = [] + from telemetry import all_page_interactions for interaction_data in self.actions: - interaction = page_interaction.FindClassWithName( + interaction = all_page_interactions.FindClassWithName( interaction_data['action'])(interaction_data) - interaction.PerformInteraction(page, tab) + self._interaction_list.append(interaction) + def CustomizeBrowserOptions(self, options): + for interaction in self._interaction_list: + interaction.CustomizeBrowserOptions(options) -page_interaction.RegisterClass('compound', CompoundInteraction) + def PerformInteraction(self, page, tab): + for interaction in self._interaction_list: + interaction.PerformInteraction(page, tab) diff --git a/tools/telemetry/telemetry/compound_interaction_unittest.py b/tools/telemetry/telemetry/compound_interaction_unittest.py index 9b0c9115..23ab392 100644 --- a/tools/telemetry/telemetry/compound_interaction_unittest.py +++ b/tools/telemetry/telemetry/compound_interaction_unittest.py @@ -2,6 +2,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from telemetry import all_page_interactions from telemetry import compound_interaction from telemetry import page_interaction from telemetry import tab_test_case @@ -23,8 +24,8 @@ class CompoundInteractionTest(tab_test_case.TabTestCase): def PerformInteraction(self, page, tab): CompoundInteractionTest.interaction2_called = True - page_interaction.RegisterClass('mock1', MockInteraction1) - page_interaction.RegisterClass('mock2', MockInteraction2) + all_page_interactions.RegisterClassForTest('mock1', MockInteraction1) + all_page_interactions.RegisterClassForTest('mock2', MockInteraction2) i = compound_interaction.CompoundInteraction({ 'action': 'compound', diff --git a/tools/telemetry/telemetry/discover.py b/tools/telemetry/telemetry/discover.py new file mode 100644 index 0000000..5ddc52b --- /dev/null +++ b/tools/telemetry/telemetry/discover.py @@ -0,0 +1,40 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +import inspect +import logging +import os +import traceback + +def Discover(start_dir, suffix, clazz): + """Discover all classes in |start_dir| which subclass |clazz|. + + Args: + start_dir: The directory to recursively search. + suffix: file name suffix for files to import, without the '.py' ending. + clazz: The base class to search for. + + Returns: + dict of {module_name: class}. + """ + top_level_dir = os.path.join(start_dir, '..') + classes = {} + for dirpath, _, filenames in os.walk(start_dir): + for filename in filenames: + if not filename.endswith(suffix + '.py'): + continue + name, _ = os.path.splitext(filename) + relpath = os.path.relpath(dirpath, top_level_dir) + fqn = relpath.replace('/', '.') + '.' + name + try: + module = __import__(fqn, fromlist=[True]) + except Exception: + logging.error('While importing [%s]\n' % fqn) + traceback.print_exc() + continue + for name, obj in inspect.getmembers(module): + if inspect.isclass(obj): + if clazz in inspect.getmro(obj): + name = module.__name__.split('.')[-1] + classes[name] = obj + return classes diff --git a/tools/telemetry/telemetry/inspector_page_unittest.py b/tools/telemetry/telemetry/inspector_page_unittest.py index 33c476e..8bb9b99 100644 --- a/tools/telemetry/telemetry/inspector_page_unittest.py +++ b/tools/telemetry/telemetry/inspector_page_unittest.py @@ -8,7 +8,6 @@ from telemetry import tab_test_case class InspectorPageTest(tab_test_case.TabTestCase): def __init__(self, *args): super(InspectorPageTest, self).__init__(*args) - self._custom_action_called = False def testPageNavigateToNormalUrl(self): self._tab.page.Navigate('http://www.google.com') @@ -34,13 +33,13 @@ class InspectorPageTest(tab_test_case.TabTestCase): self.assertEquals(self._tab.runtime.Evaluate('document.location.pathname;'), '/page_with_link.html') - self._custom_action_called = False + custom_action_called = [False] def CustomAction(): - self._custom_action_called = True + custom_action_called[0] = True self._tab.runtime.Execute('document.getElementById("clickme").click();') self._tab.page.PerformActionAndWaitForNavigate(CustomAction) - self.assertTrue(self._custom_action_called) + self.assertTrue(custom_action_called[0]) self.assertEquals(self._tab.runtime.Evaluate('document.location.pathname;'), '/blank.html') diff --git a/tools/telemetry/telemetry/multi_page_benchmark.py b/tools/telemetry/telemetry/multi_page_benchmark.py index a17a031..b24374c 100644 --- a/tools/telemetry/telemetry/multi_page_benchmark.py +++ b/tools/telemetry/telemetry/multi_page_benchmark.py @@ -138,7 +138,7 @@ class MultiPageBenchmark(page_test.PageTest): To add test-specific options: class BodyChildElementBenchmark(MultiPageBenchmark): - def AddOptions(parser): + def AddCommandLineOptions(parser): parser.add_option('--element', action='store', default='body') def MeasurePage(self, page, tab, results): @@ -146,8 +146,8 @@ class MultiPageBenchmark(page_test.PageTest): 'document.querySelector('%s').children.length') results.Add('children', 'count', child_count) """ - def __init__(self): - super(MultiPageBenchmark, self).__init__('_RunTest') + def __init__(self, interaction_name=''): + super(MultiPageBenchmark, self).__init__('_RunTest', interaction_name) def _RunTest(self, page, tab, results): results.WillMeasurePage(page) diff --git a/tools/telemetry/telemetry/multi_page_benchmark_runner.py b/tools/telemetry/telemetry/multi_page_benchmark_runner.py index 48837ee..89fc5e5 100755 --- a/tools/telemetry/telemetry/multi_page_benchmark_runner.py +++ b/tools/telemetry/telemetry/multi_page_benchmark_runner.py @@ -3,59 +3,26 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import csv -import inspect import logging import os import sys -import traceback +from telemetry import all_page_interactions # pylint: disable=W0611 from telemetry import browser_finder from telemetry import browser_options +from telemetry import discover from telemetry import multi_page_benchmark from telemetry import page_runner from telemetry import page_set - -def _Discover(start_dir, clazz): - """Discover all classes in |start_dir| which subclass |clazz|. - - Args: - start_dir: The directory to recursively search. - clazz: The base class to search for. - - Returns: - dict of {module_name: class}. - """ - top_level_dir = os.path.join(start_dir, '..') - classes = {} - for dirpath, _, filenames in os.walk(start_dir): - for filename in filenames: - if not filename.endswith('.py'): - continue - name, _ = os.path.splitext(filename) - relpath = os.path.relpath(dirpath, top_level_dir) - fqn = relpath.replace('/', '.') + '.' + name - try: - module = __import__(fqn, fromlist=[True]) - except Exception: - logging.error('While importing [%s]\n' % fqn) - traceback.print_exc() - continue - for name, obj in inspect.getmembers(module): - if inspect.isclass(obj): - if clazz in inspect.getmro(obj): - name = module.__name__.split('.')[-1] - classes[name] = obj - return classes - - def Main(benchmark_dir): """Turns a MultiPageBenchmark into a command-line program. Args: benchmark_dir: Path to directory containing MultiPageBenchmarks. """ - benchmarks = _Discover(benchmark_dir, multi_page_benchmark.MultiPageBenchmark) + benchmarks = discover.Discover(benchmark_dir, '', + multi_page_benchmark.MultiPageBenchmark) # Naively find the benchmark. If we use the browser options parser, we run # the risk of failing to parse if we use a benchmark-specific parameter. @@ -70,7 +37,7 @@ def Main(benchmark_dir): benchmark = None if benchmark_name is not None: benchmark = benchmarks[benchmark_name]() - benchmark.AddOptions(parser) + benchmark.AddCommandLineOptions(parser) _, args = parser.parse_args() @@ -103,4 +70,8 @@ Use --browser=list to figure out which are available.\n""" if len(results.page_failures): logging.warning('Failed pages: %s', '\n'.join( [failure['page'].url for failure in results.page_failures])) + + if len(results.skipped_pages): + logging.warning('Skipped pages: %s', '\n'.join( + [skipped['page'].url for skipped in results.skipped_pages])) return min(255, len(results.page_failures)) diff --git a/tools/telemetry/telemetry/multi_page_benchmark_unittest.py b/tools/telemetry/telemetry/multi_page_benchmark_unittest.py index 675f355..45a387c 100644 --- a/tools/telemetry/telemetry/multi_page_benchmark_unittest.py +++ b/tools/telemetry/telemetry/multi_page_benchmark_unittest.py @@ -6,6 +6,7 @@ import os from telemetry import multi_page_benchmark from telemetry import multi_page_benchmark_unittest_base from telemetry import page as page_module +from telemetry import page_interaction from telemetry import page_set from telemetry import wpr_modes @@ -14,7 +15,7 @@ class BenchThatFails(multi_page_benchmark.MultiPageBenchmark): raise multi_page_benchmark.MeasurementFailure('Intentional failure.') class BenchThatHasDefaults(multi_page_benchmark.MultiPageBenchmark): - def AddOptions(self, parser): + def AddCommandLineOptions(self, parser): parser.add_option('-x', dest='x', default=3) def MeasurePage(self, page, tab, results): @@ -38,6 +39,13 @@ class BenchQueryParams(multi_page_benchmark.MultiPageBenchmark): query = tab.runtime.Evaluate('window.location.search') assert query.strip() == '?foo=1' +class BenchWithInteraction(multi_page_benchmark.MultiPageBenchmark): + def __init__(self): + super(BenchWithInteraction, self).__init__('test_interaction') + + def MeasurePage(self, page, tab, results): + pass + class MultiPageBenchmarkUnitTest( multi_page_benchmark_unittest_base.MultiPageBenchmarkUnitTestBase): @@ -102,3 +110,17 @@ class MultiPageBenchmarkUnitTest( finally: if os.path.isfile(test_archive): os.remove(test_archive) + + def testInteractions(self): + interaction_called = [False] + class MockInteraction(page_interaction.PageInteraction): + def PerformInteraction(self, page, tab): + interaction_called[0] = True + from telemetry import all_page_interactions + all_page_interactions.RegisterClassForTest('mock', MockInteraction) + + ps = self.CreatePageSetFromFileInUnittestDataDir('blank.html') + setattr(ps.pages[0], 'test_interaction', {'action': 'mock'}) + benchmark = BenchWithInteraction() + self.RunBenchmark(benchmark, ps) + self.assertTrue(interaction_called[0]) diff --git a/tools/telemetry/telemetry/multi_page_benchmark_unittest_base.py b/tools/telemetry/telemetry/multi_page_benchmark_unittest_base.py index 617d77f..128ff3f 100644 --- a/tools/telemetry/telemetry/multi_page_benchmark_unittest_base.py +++ b/tools/telemetry/telemetry/multi_page_benchmark_unittest_base.py @@ -35,7 +35,7 @@ class MultiPageBenchmarkUnitTestBase(unittest.TestCase): options = options_for_unittests.Get() assert options temp_parser = options.CreateParser() - benchmark.AddOptions(temp_parser) + benchmark.AddCommandLineOptions(temp_parser) defaults = temp_parser.get_default_values() for k, v in defaults.__dict__.items(): if hasattr(options, k): diff --git a/tools/telemetry/telemetry/page_interaction.py b/tools/telemetry/telemetry/page_interaction.py index 108db76..57b2d7b 100644 --- a/tools/telemetry/telemetry/page_interaction.py +++ b/tools/telemetry/telemetry/page_interaction.py @@ -24,14 +24,3 @@ class PageInteraction(object): def CleanUp(self, page, tab): pass - -_page_interaction_classes = {} -def GetAllClasses(): - return list(_page_interaction_classes.values()) - -def FindClassWithName(name): - return _page_interaction_classes.get(name) - -def RegisterClass(name, interaction_class): - assert name not in _page_interaction_classes - _page_interaction_classes[name] = interaction_class diff --git a/tools/telemetry/telemetry/page_runner.py b/tools/telemetry/telemetry/page_runner.py index 827d266..9ad5351 100644 --- a/tools/telemetry/telemetry/page_runner.py +++ b/tools/telemetry/telemetry/page_runner.py @@ -73,6 +73,9 @@ http://goto/read-src-internal, or create a new archive using --record. if not os.path.exists(credentials_path): credentials_path = None + for page in self.page_set: + test.CustomizeBrowserOptionsForPage(page, possible_browser.options) + with possible_browser.Create() as b: b.credentials.credentials_path = credentials_path test.SetUpBrowser(b) @@ -101,6 +104,10 @@ http://goto/read-src-internal, or create a new archive using --record. self._RunPage(options, page, tab, test, results) def _RunPage(self, options, page, tab, test, results): + if not test.CanRunForPage(page): + results.AddSkippedPage(page, 'Test cannot run', '') + return + logging.info('Running %s' % page.url) page_state = PageState() diff --git a/tools/telemetry/telemetry/page_test.py b/tools/telemetry/telemetry/page_test.py index 2d4d24f..a55ee9e 100644 --- a/tools/telemetry/telemetry/page_test.py +++ b/tools/telemetry/telemetry/page_test.py @@ -9,24 +9,31 @@ class Failure(Exception): class PageTestResults(object): def __init__(self): self.page_failures = [] + self.skipped_pages = [] def AddFailure(self, page, message, details): self.page_failures.append({'page': page, 'message': message, 'details': details}) + def AddSkippedPage(self, page, message, details): + self.skipped_pages.append({'page': page, + 'message': message, + 'details': details}) + class PageTest(object): """A class styled on unittest.TestCase for creating page-specific tests.""" - def __init__(self, test_method_name): + def __init__(self, test_method_name, interaction_name_to_run=''): self.options = None try: self._test_method = getattr(self, test_method_name) except AttributeError: raise ValueError, 'No such method %s.%s' % ( self.__class_, test_method_name) # pylint: disable=E1101 + self._interaction_name_to_run = interaction_name_to_run - def AddOptions(self, parser): + def AddCommandLineOptions(self, parser): """Override to expose command-line options for this benchmark. The provided parser is an optparse.OptionParser instance and accepts all @@ -38,13 +45,47 @@ class PageTest(object): """Override to add test-specific options to the BrowserOptions object""" pass + def CustomizeBrowserOptionsForPage(self, page, options): + """Add options specific to the test and the given page.""" + if not self.CanRunForPage(page): + return + interaction = self.GetInteraction(page) + if interaction: + interaction.CustomizeBrowserOptions(options) + def SetUpBrowser(self, browser): """Override to customize the browser right after it has launched.""" pass + def CanRunForPage(self, page): #pylint: disable=W0613 + """Override to customize if the test can be ran for the given page.""" + return True + + def WillRunInteraction(self, page, tab): + """Override to do operations before running the interaction on the page.""" + pass + + def DidRunInteraction(self, page, tab): + """Override to do operations after running the interaction on the page.""" + pass + def Run(self, options, page, tab, results): self.options = options + interaction = self.GetInteraction(page) + if interaction: + tab.WaitForDocumentReadyStateToBeComplete() + self.WillRunInteraction(page, tab) + interaction.PerformInteraction(page, tab) + self.DidRunInteraction(page, tab) try: self._test_method(page, tab, results) finally: self.options = None + + def GetInteraction(self, page): + if not self._interaction_name_to_run: + return None + interaction_data = getattr(page, self._interaction_name_to_run) + from telemetry import all_page_interactions + return all_page_interactions.FindClassWithName( + interaction_data['action'])(interaction_data) |