# Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # # This file lists the pyauto tests that run as a part of the functional test # suite. # # Tests can be enabled on a per-platform basis. Tests not listed here will # not be run. # # Within each platform, tests are specified in up to 3 groups: (1) enabled # tests; (2) tests that are permanently disabled because they do not apply # to the given platform and hence should not be run; and (3) disabled tests # that need to be investigated/fixed. Tests are listed alphabetically within # each group. # # Test names can be specified in any of the following ways: # 1. as a module, in which case all tests in that module will be run # example: 'test_basic' # 2. or as a test case, in which case all tests in that test case will be run # example: 'test_basic.SimpleTest' # 3. or as an individual test # example: 'test_basic.SimpleTest.testCanOpenGoogle' # # Tests beginning with '-' will be excluded. This can be used to enforce # exclusions for a particular platform. # Names beginning with '@' will be treated as suite names and will be # recursively expanded. { # The CONTINUOUS suite includes tests that we expect to pass on a # continuous builder. # This is the suite that gets run on 'Chromium' builds. 'CONTINUOUS': { 'all': [ 'about_plugins_ui.AboutPluginsUITest', 'apptest', 'autofill', 'chromoting.it2me_basic', 'codesign', 'cookies', 'crash_reporter', 'downloads', 'execute_javascript', 'extensions', 'flash', 'fullscreen_mouselock', 'history', 'https', 'gtalk.test_basic', 'imports', 'infobars', 'instant', 'multiprofile', 'navigation', 'notifications', 'ntp', 'omnibox', 'passwords', 'plugins', 'plugins_check', 'policy', 'policy_prefs_ui', 'popups', 'prefs', 'prefs_ui', 'pyauto_webdriver', 'search_engines', 'shortcuts', 'special_tabs', 'sync.SyncTest', 'test_pyauto', 'themes', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/123396 '-fullscreen_mouselock.FullscreenMouselockTest.testPatternsForFSAndML', # crbug.com/132665 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavBackToPrevPage', '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavToNewPage', '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavBackToPrevPage', '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavToNewPage', # crbug.com/140460 '-fullscreen_mouselock.FullscreenMouselockTest.testNoTabFSExitWhenJSExitMouseLock', # crosbug.com/136875 '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSDoesNotExitForAnchorLinks', # crbug.com/142677 '-gtalk.test_basic.BasicTest.testCurrentVersion', '-gtalk.test_basic.BasicTest.testRCVersion', # crbug.com/131874 '-infobars.OneClickInfobarTest.testNoOneClickInfobarAfterCancel', # crbug.com/133315 '-infobars.OneClickInfobarTest.testDisplayOneClickInfobarAfterDismiss', # crbug.com/105560 '-notifications', # crbug.com/99506 '-notifications.NotificationsTest.testSpecialURLNotification', # crbug.com/143308 '-omnibox.OmniboxLiveTest.testGoogleSearch', # crbug.com/71715 '-omnibox.OmniboxTest.testHistoryResult', # crbug.com/123019 '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', # crbug.com/137041 '-omnibox.OmniboxTest.testDifferentTypesOfResults', # crbug.com/86524 '-plugins.PluginsTest.testAllowPluginException', # crbug.com/97140 '-plugins.PluginsTest.testDisableEnableAllPlugins', # crbug.com/85307 '-plugins.PluginsTest.testKillAndReloadAllPlugins', # crbug.com/115427 '-policy.PolicyTest.testAlwaysAuthorizePlugins', # crbug.com/118054 '-policy.PolicyTest.testApplicationLocaleValue', # crbug.com/131258 '-policy.PolicyTest.testDefaultSearchProviderOptions', # crbug.com/96896 '-policy.PolicyTest.testEnabledPlugins', # crbug.com/118123 '-policy.PolicyTest.testExtensionInstallFromForceList', # crbug.com/47935 '-popups.PopupsTest.testBlockedPopupNotShowInHistory', # crbug.com/83827 '-popups.PopupsTest.testUnblockedPopupAddedToOmnibox', # crbug.com/85600 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', # crbug.com/112051 '-prefs_ui.PrefsUITest.testChangeExceptionBehaviorUI', # crbug.com/88593 '-sync.SyncTest.testPersonalStuffSyncSection', # crbug.com/95140 '-translate.TranslateTest.testGoBackAndForwardToTranslatePage', # crbug.com/124913 '-infobars.OneClickInfobarTest.testNoSameIDSigninForTwoProfiles', # crbug.com/124280 '-downloads.DownloadsTest.testPauseAndResume', # Mysteriously broken? # crbug.com/138857 '-multiprofile.MultiprofileTest.testMakeSearchEngineDefaultInMultiprofile', ], 'win': [ # Enable sync.SyncIntegrationTest only for windows CONTINUIOUS, # which contains the sync protobufs. #'sync.SyncIntegrationTest', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # Disabled on vista in the actual test. crbug.com/89767 # -imports.ImportsTest.testImportFirefoxDataTwice # Disabled on vista in the actual test. crbug.com/89767 # -imports.ImportsTest.testFirefoxImportFromPrefs # Disabled on vista in the actual test. crbug.com/89767 # -imports.ImportsTest.testImportFirefoxPasswordsFromPrefs # crbug.com/121731 '-omnibox.OmniboxTest.testIncognitoAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testIncognitoAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testIncognitoAppSearchWithVeryLongAppName', # crbug.com/117569 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', # crbug.com/103379 '-policy.PolicyTest.testDisable3DAPIs', # crbug.com/104227 '-sync.SyncIntegrationTest', # crbug.com/88679 '-sync.SyncTest', # crbug.com/86949 '-sync.SyncTest.testRestartBrowser', # crbug.com/87642 '-sync.SyncTest.testDisableAndEnableDatatypes', # crbug.com/98526 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', # crbug.com/95031 '-notifications.NotificationsTest.testOriginPrefsNotSavedInIncognito', ], 'mac': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # codesign tests should run *after* signing. crbug.com/50481 '-codesign', # crbug.com/124922 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenBrowserLoseFocus', # crbug.com/125989 '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenAlertDialogShow', # Fails on chrome-mac-10_7-qa only: crbug.com/124886 '-fullscreen_mouselock.FullscreenMouselockTest.testPrefsForFullscreenExit', '-fullscreen_mouselock.FullscreenMouselockTest.testNoMouseLockWhenCancelFS', # crbug.com/114244 '-imports.ImportsTest.testImportSafariDataTwice', # crbug.com/121484 '-multiprofile.MultiprofileTest.test20NewProfiles', '-ntp.NTPTest.testLaunchAppNewWindow', # crbug.com/79812 # Causes browser crash. crbug.com/93548 '-notifications.NotificationsTest.testIncognitoNotification', '-notifications.NotificationsTest.testOriginPrefsNotSavedInIncognito', '-notifications.NotificationsTest.testNotificationOrderAfterClosingOne', # crbug.com/70437 '-omnibox.OmniboxTest.testHistoryResult', # crbug.com/91617 '-omnibox.OmniboxTest.testContentHistory', # Keychain popups make autofill/password tests difficult: crbug.com/49378 '-passwords', # crbug.com/97140 '-plugins.PluginsTest.testDisableEnableAllPlugins', # crbug.com/130094 '-policy', '-policy_prefs_ui.PolicyPrefsUITest.testToggleUserPolicyTogglesBanner', '-policy_prefs_ui.PolicyPrefsUITest.testUserPoliciesShowBanner', # crbug.com/69619 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', # Keychain popups make sync tests difficult on Mac. crbug.com/49378 '-sync', # crbug.com/98526 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', ], 'linux': [ # linux != chromeos 'test_clean_exit', # =========================== # Permanently-disabled tests. # =========================== # crbug.com/104206 '-notifications.NotificationsTest.testKillNotificationProcess', # Several omnibox tests do not show omnibox results when run # under virtual X on linux. The CONTINUOUS suite runs under virtual X # on the chromium bots, so exlcude these from here. They're included # back in the FULL suite below. # crbug.com/85310 '-omnibox.OmniboxTest.testAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testAppNameWithNumberSearch', '-omnibox.OmniboxTest.testAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testAppSearchWithVeryLongAppName', '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', '-omnibox.OmniboxTest.testAutoCompleteForSearch', '-omnibox.OmniboxTest.testBasicAppSearch', '-omnibox.OmniboxTest.testBookmarkResultInNewTabAndWindow', '-omnibox.OmniboxTest.testCrazyFilenames', '-omnibox.OmniboxTest.testDifferentTypesOfResults', '-omnibox.OmniboxTest.testFetchHistoryResultItems', '-omnibox.OmniboxLiveTest.testGoogleSearch', '-omnibox.OmniboxTest.testInlineAutoComplete', '-omnibox.OmniboxTest.testSelect', '-omnibox.OmniboxTest.testSuggest', '-omnibox.OmniboxTest.testSuggestPref', # System password manager obstructs password automation. '-passwords', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/91033 '-omnibox.OmniboxTest.testOmniboxSearchHistory', # crbug.com/103379 '-policy.PolicyTest.testDisable3DAPIs', # Gnome / kwallet popups on linux break sync tests. crbug.com/80329 '-sync', # crbug.com/66072 '-notifications.NotificationsTest.testNotificationOrderAfterClosingOne', ], # ChromeOS is linux, but note that this section does not include the # entries in the linux section above. # These tests use the named automation interface to control the # primary chrome on ChromeOS. 'chromeos': [ 'chromeos_basic', 'chromeos_browser', 'chromeos_crosh', 'chromeos_file_browser', 'chromeos_offline', 'chromeos_power', 'chromeos_prefs', 'chromeos_security', 'chromeos_time', 'chromeos_update', 'chromeos_wifi_sanity', 'doc_viewing', 'secure_shell', # =========================== # Permanently-disabled tests. # =========================== # ChromeOS doesn't yet support the chromoting host. '-chromoting.it2me_basic', # No codesign verification on ChromeOS. '-codesign', # Import tests are invalid on ChromeOS since Chrome is the only browser. '-imports', # Sync is already signed in with the login account. # So one-click infobar tests do tno apply to chromeos. '-infobars.OneClickInfobarTest', # Multi-profile doesn't apply to chromeos yet. '-multiprofile', '-ntp.NTPTest.testDifferentProfileNotAppearInMostVisited', # These require a custom setup on ChromeOS and have their own suite for # this platform. '-policy', '-policy_prefs_ui', # You cannot adjust the downloads directory on ChromeOS. '-prefs.PrefsTest.testDownloadDirPref', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/132337 '-autofill.AutofillTest.testTabOrderForEditAddress', # crbug.com/129469 '-chromeos_prefs.ChromeosPrefsTest.testCaptureUserPhoto', # Downloads panel stays even after declining a download. # crosbug.com/14728 '-downloads.DownloadsTest.testDeclineDangerousDownload', # crosbug.com/19556 '-extensions.ExtensionsTest.testAllowAccessFileURLs', '-extensions.ExtensionsTest.testAllowIncognitoExtension', '-extensions.ExtensionsTest.testDisableEnableExtension', # crbug.com/134593 '-gtalk.test_basic.BasicTest.testCurrentVersion', '-gtalk.test_basic.BasicTest.testRCVersion', # crbug.com/129498 '-https.SSLTest.testSSLCertAuthorityOK', '-https.SSLTest.testSSLCertOK', # crosbug.com/24496 '-infobars.InfobarTest.testPluginCrashForMultiTabs', # crbug.com/109035 '-infobars.InfobarTest.testPluginCrashInfobar', # crosbug.com/14256 '-ntp.NTPTest.testLaunchAppFullScreen', # Content history broken in omnibox. crosbug.com/14416 '-omnibox.OmniboxTest.testContentHistory', # crbug.com/91033 '-omnibox.OmniboxTest.testOmniboxSearchHistory', # crosbug.com/19760 '-passwords.PasswordTest.testClearFetchedCredForNewUserName', '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', # onunload popups get created in the same window on chromeos # crosbug.com/14924 '-popups.PopupsTest.testPopupsLaunchWhenTabIsClosed', # Session restore not working with PyAuto. crosbug.com/12648 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', '-prefs.PrefsTest.testSessionRestoreURLs', '-prefs.PrefsTest.testSessionRestore', # sync tests were running into a secondary passphrase. crosbug.com/15016. '-sync', # crbug.com/109035 '-plugins.PluginsTest.testBlockPluginException', # Deal with i18n chars. crosbug.com/12639 '-downloads.DownloadsTest.testCrazyFilenames', '-omnibox.OmniboxTest.testCrazyFilenames', # crosbug.com/16977 '-chromeos_wifi_sanity.ChromeosWifiSanity.testConnectToHiddenWiFiNonExistent', # crosbug.com/20025 '-chromeos_browser.ChromeosBrowserTest.testFullScreen', # crbug.com/102549 '-notifications.NotificationsTest.testOriginPrefsNotSavedInIncognito', # Chrome driver does not work in Chrome OS. # crosbug.com/19556 '-prefs_ui', '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied', '-prefs.PrefsTest.testGeolocationPref', '-pyauto_webdriver', ], }, # CONTINUOUS # The CONTINUOUS_PUBLIC is a subset of CONTINUOUS suite # and excludes tests that depend on internal data 'CONTINUOUS_PUBLIC': { 'all': [ '@CONTINUOUS', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # These currently fail even with internal data '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', # =========================== # Permanently-disabled tests. # =========================== '-infobars.InfobarTest.testPluginCrashForMultiTabs', '-flash.FlashTest.testCanLoadFlash', '-flash.FlashTest.testFlashLoadsAfterKill', '-flash.FlashTest.testFlashIncognitoMode', '-flash.FlashTest.testFlashWithMultipleTabs', '-flash.FlashTest.testSingleFlashPluginProcess', '-ntp.NTPTest.testGetAppsWhenInstallApp', '-ntp.NTPTest.testLaunchAppNewWindow', '-ntp.NTPTest.testLaunchAppPinnedTab', '-ntp.NTPTest.testLaunchAppRegularTab', '-ntp.NTPTest.testLaunchAppWithDefaultSettings', '-ntp.NTPTest.testUninstallApp', '-omnibox.OmniboxTest.testAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testAppNameWithNumberSearch', '-omnibox.OmniboxTest.testAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testAppSearchWithVeryLongAppName', '-omnibox.OmniboxTest.testBasicAppSearch', '-omnibox.OmniboxTest.testBeginningPartAppNameSearchInNewTab', '-omnibox.OmniboxTest.testEndPartAppNameSearchInNewTab', '-omnibox.OmniboxTest.testIncognitoAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testIncognitoAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testIncognitoAppSearchWithVeryLongAppName', '-omnibox.OmniboxTest.testRepeatedlyAppLaunchInTabs', '-passwords.PasswordTest.testDisplayAndSavePasswordInfobar', '-passwords.PasswordTest.testInfoBarDisappearByNavigatingPage', '-passwords.PasswordTest.testInfoBarDisappearByReload', '-passwords.PasswordTest.testLoginCredsNotShownInIncognito', '-passwords.PasswordTest.testNeverSavePasswords', '-passwords.PasswordTest.testPasswordInfobarShowsForBlockedDomain', '-plugins.PluginsTest.testBlockAllPlugins', '-popups.PopupsTest.testMultiplePopups', '-popups.PopupsTest.testPopupBlockedEverySec', ], 'win': [ ], 'mac': [ ], 'linux': [ ], 'chromeos': [ # These tests fail due to crosbug.com/19556 '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', # Media files test uses internal media files '-chromeos_file_browser.ChromeosFileBrowserTest.testOpenMediaFiles', # No flash plugin in non-branded OS images. crosbug.com/25905 '-youtube', '-flash', ], }, # The FULL suite includes CONTINUOUS, and more. # This the suite that gets run on 'Google Chrome' builds. 'FULL': { 'all': [ '@CONTINUOUS', 'about_plugins_ui.ChromeAboutPluginsUITest', 'media.media_basic_playback', 'nacl_sdk', 'translate', 'youtube', # =========================== # Permanently-disabled tests. # =========================== # The chromoting webapp isn't currently available when FULL is run. '-chromoting.it2me_basic', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # The source is behind. Waiting for dev to automate the update. # crbug.com/109160 '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtension', # crbug.com/132272 '-shortcuts.ShortcutsTest.testHelpShortcut', # crbug.com/132285 '-special_tabs.SpecialTabsTest.testSpecialURLTabs', # crbug.com/129802 '-translate.TranslateTest.testAlwaysTranslateLanguageButton', '-translate.TranslateTest.testChangeTargetLanguageAlwaysTranslate', '-translate.TranslateTest.testToggleTranslateOption', ], 'win': [ 'gpu', # =========================== # Permanently-disabled tests. # =========================== # Disable sync.SyncIntegrationTest on the QA bots in FULL, # until sync protobufs are copied over. '-sync.SyncIntegrationTest', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/105948 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', # crbug.com/111289 '-extensions.ExtensionsTest.testAllowAccessFileURLs', # crbug.com/113090 '-extensions.ExtensionsTest.testAllowIncognitoExtension', # crbug.com/94682 '-gpu.GpuTest.test2dCanvas', # crbug.com/125424 '-media.media_basic_playback.MediaConstrainedNetworkPerfTest.testBasicPlaybackMatrix', # crbug.com/115646 '-translate.TranslateTest.testSeveralLanguages', # crbug.com/95621 '-youtube.YoutubeTest.testPlayerStatus', ], 'mac': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/109035 '-youtube.YoutubeTest.testPlayerBytes', '-youtube.YoutubeTest.testPlayerResolution', # =========================== # Permanently-disabled tests. # =========================== ], 'linux': [ # Add back the omnibox tests excluded from the CONTINUOUS suite above # because they do not produce omnibox results under virtual X on linux. # crbug.com/85310 'omnibox.OmniboxTest.testAppComboNameWithSpecialCharSearch', 'omnibox.OmniboxTest.testAppNameWithNumberSearch', 'omnibox.OmniboxTest.testAppNameWithSpaceSearch', 'omnibox.OmniboxTest.testAppSearchWithVeryLongAppName', 'omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', 'omnibox.OmniboxTest.testAutoCompleteForSearch', 'omnibox.OmniboxTest.testBasicAppSearch', 'omnibox.OmniboxTest.testBookmarkResultInNewTabAndWindow', 'omnibox.OmniboxTest.testCrazyFilenames', 'omnibox.OmniboxTest.testDifferentTypesOfResults', 'omnibox.OmniboxTest.testFetchHistoryResultItems', 'omnibox.OmniboxLiveTest.testGoogleSearch', 'omnibox.OmniboxTest.testInlineAutoComplete', 'omnibox.OmniboxTest.testSelect', 'omnibox.OmniboxTest.testSuggest', 'omnibox.OmniboxTest.testSuggestPref', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/111289 '-extensions.ExtensionsTest.testAllowAccessFileURLs', # crbug.com/100365 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', # crbug.com/109035 '-youtube.YoutubeTest.testPlayerBytes', '-youtube.YoutubeTest.testPlayerResolution', ], 'chromeos': [ # =========================== # Permanently-disabled tests. # =========================== # No NaCl support on ChromeOS. '-nacl_sdk', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # Session restore not working with PyAuto. crosbug.com/12648 '-translate.TranslateTest.testSessionRestore', # Deal with i18n chars. crosbug.com/12639 '-translate.TranslateTest.testDownloadsNotTranslated', ], }, # Performance tests. 'PERFORMANCE': { 'all': [ 'perf', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== '-perf.HTML5BenchmarkTest', # crbug.com/134476 '-perf.WebGLTest.testWebGLField', # crbug.com/132797 # =========================== # Permanently-disabled tests. # =========================== # Invoked outside of the pyauto_tests framework. '-perf.BenchmarkPerfTest.testSpaceport', '-perf.PopularSitesScrollTest.test2012Q3', ], 'win': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== '-perf.GPUPerfTest', # Fails. Discuss with prachij@. '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. # =========================== # Permanently-disabled tests. # =========================== '-perf.MemoryTest', # Designed only for ChromeOS. '-perf.NetflixPerfTest', # Designed only for ChromeOS. ], 'mac': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== '-perf.GPUPerfTest', # Fails. Discuss with prachij@. '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 '-perf.LiveGamePerfTest', # Requires linux /proc/stat. '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. # =========================== # Permanently-disabled tests. # =========================== '-perf.MemoryTest', # Designed only for ChromeOS. '-perf.NetflixPerfTest', # Designed only for ChromeOS. ], 'linux': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== '-perf.GPUPerfTest', # Fails. Discuss with prachij@. '-perf.PageCyclerNetSimTest', # Dependence missing: crbug.com/132559 '-perf.WebGLTest', # May not render WebGL; need to verify on QA bot. '-perf.YoutubePerfTest', # AttributeError: AssertPlayingState. # =========================== # Permanently-disabled tests. # =========================== '-perf.MemoryTest', # Designed only for ChromeOS. '-perf.NetflixPerfTest', # Designed only for ChromeOS. ], 'chromeos': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== '-perf.NetflixPerfTest', # crosbug.com/32320. '-perf.PageCyclerNetSimTest', # Investigate Web Page Replay on ChromeOS. # =========================== # Permanently-disabled tests. # =========================== '-perf.GPUPerfTest', # Designed only for Chrome desktop. ], }, 'PERFORMANCE_LAB': { 'all': [ 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', 'perf.FlashTest.testFlashGaming', 'perf.ScrollTest.testGooglePlusScroll', 'perf.TabPerfTest.test20Tabs', 'perf.WebGLTest.testWebGLSpaceRocks', ], }, 'EMPTY': { }, 'CHROMEOS_CONNECTIVITY': { 'chromeos': [ 'chromeos_wifi_functional', 'chromeos_wifi_compliance', 'wifi_downloads', 'wifi_notification', ], }, # Subset of CONTINUOUS_PUBLIC to be run in a ChromeOS VM. 'CHROMEOS_VM_PUBLIC': { 'chromeos': [ '@CONTINUOUS_PUBLIC', '-chromeos_power', '-chromeos_prefs.ChromeosPrefsTest.testCaptureUserPhoto', '-chromeos_wifi_sanity', ], }, # ChromeOS flash tests. 'CHROMEOS_FLASH': { 'chromeos': [ 'flash', ], }, # ChromeOS login tests. 'CHROMEOS_LOGIN': { 'chromeos': [ 'chromeos_login', 'chromeos_oobe', # crosbug.com/32583 '-chromeos_login.ChromeosLogin.testCachedCredentials', ], }, # ChromeOS policy tests. Some of these are shared with Desktop Chrome tests # but require custom setup on ChromeOS. Others are specific to ChromeOS only. 'CHROMEOS_POLICY': { 'chromeos': [ 'chromeos_ephemeral', 'chromeos_device_policy', 'chromeos_onc', 'chromeos_retail_mode', 'policy', 'policy_prefs_ui', # crbug.com/115427 '-policy.PolicyTest.testAlwaysAuthorizePlugins', # crbug.com/118054 '-policy.PolicyTest.testApplicationLocaleValue', # crbug.com/118123 '-policy.PolicyTest.testExtensionInstallFromForceList', '-policy.PolicyTest.testSetDownloadDirectory', # This tests the value "0" for the RestoreOnStartup policy, which is not # supported on ChromeOS. '-policy.PolicyTest.testStartupOptionsHomepage', # crbug.com/131258 crosbug.com/32068 '-policy.PolicyTest.testDefaultSearchProviderOptions', # crosbug.com/32069 '-policy.PolicyTest.testTranslateEnabled', ], }, # ChromeOS volume tests. 'CHROMEOS_VOLUME': { 'chromeos': [ 'chromeos_volume', ], }, # ChromeOS Accessibility tests. 'CHROMEOS_ACCESSIBILITY': { 'chromeos': [ 'chromeos_accessibility', ], }, # PGO performance suite. 'PGO': { 'chromeos': [ 'perf.BenchmarkPerfTest.testV8BenchmarkSuite', 'perf.LiveWebappLoadTest.testNewTabGmail', 'perf.PageCyclerTest', 'perf.ScrollTest.testGooglePlusScroll', ], }, # Subset of ChromeOS performance tests for seaboard. 'CHROMEOS_PERF_SEABOARD': { 'chromeos': [ 'perf.ScrollTest.testBlankPageScroll', 'perf.ScrollTest.testGooglePlusScroll', 'perf.ScrollTest.testTextScroll', ], }, # HTML5 media performance tests. 'AV_PERF': { 'linux': [ 'media.audio_latency_perf', 'media.media_constrained_network_perf', 'media.media_scrub_perf', 'media.media_seek_perf', 'media.media_stat_perf', 'media.mixed_audio_latency_perf', ], 'win': [ 'media.audio_latency_perf', 'media.media_constrained_network_perf', 'media.media_scrub_perf', 'media.media_seek_perf', 'media.media_stat_perf', 'media.mixed_audio_latency_perf', ], }, # WebRTC MediaStream tests. Requires webcam and audio device to be present on # the test machine. 'WEBRTC': { 'all': [ 'media_stream_infobar', 'webrtc_brutality_test', 'webrtc_call', # crbug.com/140465 '-webrtc_brutality_test.WebrtcBrutalityTest.testRepeatedGetUserMediaRequests' ], }, # Trace event tests. 'TRACING': { 'all': [ 'tracing.tracing_smoke_test', ], }, # Pyauto functional tests running on coverage bots. 'CODE_COVERAGE': { 'all': [ '@FULL', ], 'linux': [ # These tests fail on coverage bots. Disabling for now. '-test_clean_exit', '-about_plugins_ui', '-autofill', '-pyauto_webdriver', '-chromoting_basic', '-extensions', '-flash', '-fullscreen_mouselock', '-gtalk.test_basic', '-infobars', '-multiprofile', '-policy', '-prefs', '-prefs_ui', '-apptest', '-plugins', '-omnibox', '-special_tabs', '-youtube.YoutubeTest.testPlayerStatus', '-translate.TranslateTest.testSeveralLanguages', '-translate.TranslateTest.testBarNotVisibleOnSSLErrorPage', '-ntp.NTPTest.testLaunchAppFullScreen', '-https.SSLTest.testSSLCertOK', '-https.SSLTest.testSSLCertAuthorityOK', '-media.media_basic_playback.MediaConstrainedNetworkPerfTest.testBasicPlaybackMatrix', '-policy_prefs_ui.PolicyPrefsUITest.testNoUserPoliciesNoBanner', '-policy_prefs_ui.PolicyPrefsUITest.testToggleUserPolicyTogglesBanner', '-policy_prefs_ui.PolicyPrefsUITest.testUserPoliciesShowBanner', '-popups.PopupsTest.testMultiplePopups', '-popups.PopupsTest.testPopupBlockedEverySec', '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', '-ntp.NTPTest.testUninstallApp', '-ntp.NTPTest.testLaunchAppWithDefaultSettings', '-ntp.NTPTest.testLaunchAppRegularTab', '-ntp.NTPTest.testLaunchAppPinnedTab', '-ntp.NTPTest.testGetAppsWhenInstallApp', '-ntp.NTPTest.testLaunchAppNewWindow', ], }, }