# Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # # This file lists the pyauto tests that run as a part of the functional test # suite. # # Tests can be enabled on a per-platform basis. Tests not listed here will # not be run. # # Within each platform, tests are specified in up to 3 groups: (1) enabled # tests; (2) tests that are permanently disabled because they do not apply # to the given platform and hence should not be run; and (3) disabled tests # that need to be investigated/fixed. Tests are listed alphabetically within # each group. # # Test names can be specified in any of the following ways: # 1. as a module, in which case all tests in that module will be run # example: 'test_basic' # 2. or as a test case, in which case all tests in that test case will be run # example: 'test_basic.SimpleTest' # 3. or as an individual test # example: 'test_basic.SimpleTest.testCanOpenGoogle' # # Tests beginning with '-' will be excluded. This can be used to enforce # exclusions for a particular platform. # Names beginning with '@' will be treated as suite names and will be # recursively expanded. { # The CONTINUOUS suite includes tests that we expect to pass on a # continuous builder. # This is the suite that gets run on 'Chromium' builds. 'CONTINUOUS': { 'all': [ 'autofill', 'bookmark_bar', 'bookmarks', 'browser', 'browsing_data', 'codesign', 'content', 'cookies', 'crash_reporter', 'databases', 'downloads', 'execute_javascript', 'extensions', 'find_in_page', 'flash', 'history', 'https', 'imports', 'indexeddb', 'infobars', 'instant', 'multiprofile', 'navigation', 'notifications', 'ntp', 'omnibox', 'passwords', 'pdf', 'plugins', 'plugins_check', 'policy', 'policy_prefs_ui', 'popups', 'prefs', 'process_count', 'pyauto_webdriver', 'search_engines', 'shortcuts', 'special_tabs', 'sync.SyncTest', 'test_basic', 'test_pyauto', 'themes', 'youtube', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/83686 '-autofill.AutofillTest.testFillProfileCrazyCharacters', # crbug.com/99475 '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', # crbug.com/99506 '-notifications.NotificationsTest.testSpecialURLNotification', # crbug.com/71715 '-omnibox.OmniboxTest.testHistoryResult', # crbug.com/97140 '-plugins.PluginsTest.testDisableEnableAllPlugins', # crbug.com/82629 - real bug. '-popups.PopupsTest.testPopupsLaunchUponBrowserBackButton', # crbug.com/47935 '-popups.PopupsTest.testBlockedPopupNotShowInHistory', # crbug.com/83827 '-popups.PopupsTest.testUnblockedPopupAddedToOmnibox', # crbug.com/85600 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', # Flaky: crbug.com/67851 '-special_tabs.SpecialTabsTest.testAboutDNSTab', # crbug.com/102517 -- external webpage throws 500 '-special_tabs.SpecialTabsTest.testAboutAppCacheTab', # crbug.com/95140 '-translate.TranslateTest.testGoBackAndForwardToTranslatePage', # crbug.com/86524 '-plugins.PluginsTest.testAllowPluginException', # crbug.com/85307 '-plugins.PluginsTest.testKillAndReloadAllPlugins', # crbug.com/88574 '-special_tabs.SpecialTabsTest.testSpecialURLTabs', # crbug.com/88593 '-sync.SyncTest.testPersonalStuffSyncSection', # crbug.com/100593 '-nacl_sdk.NaClSDKTest.testVerifyNaClPlugin', # crbug.com/105560 '-notifications', ], 'win': [ # Enable sync.SyncIntegrationTest only for windows CONTINUIOUS, # which contains the sync protobufs. #'sync.SyncIntegrationTest', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # Disabled on vista in the actual test. crbug.com/89767 # -imports.ImportsTest.testImportFirefoxDataTwice # Disabled on vista in the actual test. crbug.com/89767 # -imports.ImportsTest.testFirefoxImportFromPrefs # Disabled on vista in the actual test. crbug.com/89767 # -imports.ImportsTest.testImportFirefoxPasswordsFromPrefs # crbug.com/70694, crbug.com/73078 '-passwords.PasswordTest.testInfoBarDisappearByReload', '-passwords.PasswordTest.testNeverSavePasswords', # crbug.com/91887 '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', '-passwords.PasswordTest.testClearFetchedCredForNewUserName', # crbug.com/104227 '-sync.SyncIntegrationTest', # crbug.com/88679 '-sync.SyncTest', # crbug.com/86949 '-sync.SyncTest.testRestartBrowser', # crbug.com/87642 '-sync.SyncTest.testDisableAndEnableDatatypes', # crbug.com/95621 '-youtube.YoutubeTest.testPlayerStatus', # crbug.com/98526 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', # crbug.com/95031 '-notifications.NotificationsTest.testOriginPrefsNotSavedInIncognito', # crbug.com/103379 '-policy.PolicyTest.testDisable3DAPIs', ], 'mac': [ # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # Keychain popups make autofill/password tests difficult: crbug.com/49378 '-autofill', # Autofill on Mac. crbug.com/95165 '-browsing_data.BrowsingDataTest.testClearAutofillData', # codesign tests should run *after* signing. crbug.com/50481 '-codesign', '-crash_reporter.CrashReporterTest.testRendererCrash', # crbug.com/69464 # crbug.com/75724 '-downloads.DownloadsTest.testDeclineDangerousDownload', # crbug.com/75724 '-downloads.DownloadsTest.testSaveDangerousFile', '-ntp.NTPTest.testLaunchAppNewWindow', # crbug.com/79812 # Causes browser crash. crbug.com/93548 '-notifications.NotificationsTest.testIncognitoNotification', '-notifications.NotificationsTest.testOriginPrefsNotSavedInIncognito', '-notifications.NotificationsTest.testNotificationOrderAfterClosingOne', # crbug.com/70437 '-omnibox.OmniboxTest.testHistoryResult', # crbug.com/91617 '-omnibox.OmniboxTest.testContentHistory', # Keychain popups make autofill/password tests difficult: crbug.com/49378 '-passwords', # crbug.com/97140 '-plugins.PluginsTest.testDisableEnableAllPlugins', # crbug.com/79263 '-popups.PopupsTest.testPopupsLaunchUponBrowserBackButton', # crbug.com/69619 '-search_engines.SearchEnginesTest.testDiscoverSearchEngine', # Keychain popups make sync tests difficult on Mac. crbug.com/49378 '-sync', # crbug.com/98526 '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser', '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox', # crbug.com/109035 '-youtube.YoutubeTest.testPlayerBytes', # crbug.com/109035 '-youtube.YoutubeTest.testPlayerResolution', ], 'linux': [ # linux != chromeos # Enabling here since it's disabled on other platforms. crbug.com/88574 'special_tabs.SpecialTabsTest.testSpecialURLTabs', # =========================== # Permanently-disabled tests. # =========================== '-browser.BrowserTest.testWindowResize', # Details at: crbug.com/44963 # crbug.com/104206 '-notifications.NotificationsTest.testKillNotificationProcess', # Several omnibox tests do not show omnibox results when run # under virtual X on linux. The CONTINUOUS suite runs under virtual X # on the chromium bots, so exlcude these from here. They're included # back in the FULL suite below. # crbug.com/85310 '-omnibox.OmniboxTest.testAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testAppNameWithNumberSearch', '-omnibox.OmniboxTest.testAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testAppSearchWithVeryLongAppName', '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', '-omnibox.OmniboxTest.testAutoCompleteForSearch', '-omnibox.OmniboxTest.testBasicAppSearch', '-omnibox.OmniboxTest.testBookmarkResultInNewTabAndWindow', '-omnibox.OmniboxTest.testCrazyFilenames', '-omnibox.OmniboxTest.testDifferentTypesOfResults', '-omnibox.OmniboxTest.testFetchHistoryResultItems', '-omnibox.OmniboxTest.testGoogleSearch', '-omnibox.OmniboxTest.testInlineAutoComplete', '-omnibox.OmniboxTest.testSelect', '-omnibox.OmniboxTest.testSuggest', '-omnibox.OmniboxTest.testSuggestPref', # System password manager obstructs password automation. '-passwords', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/85310 '-instant', # crbug.com/91033 '-omnibox.OmniboxTest.testOmniboxSearchHistory', # Gnome / kwallet popups on linux break sync tests. crbug.com/80329 '-sync', # crbug.com/66072 '-notifications.NotificationsTest.testNotificationOrderAfterClosingOne', # crbug.com/103379 '-policy.PolicyTest.testDisable3DAPIs', # crbug.com/109035 '-youtube.YoutubeTest.testPlayerBytes', # crbug.com/109035 '-youtube.YoutubeTest.testPlayerResolution', ], # ChromeOS is linux, but note that this section does not include the # entries in the linux section above. # These tests use the named automation interface to control the # primary chrome on ChromeOS. 'chromeos': [ 'chromeos_basic', 'chromeos_browser', 'chromeos_firmware_version_checker', 'chromeos_file_browser', 'chromeos_power', 'chromeos_prefs', 'chromeos_security', 'chromeos_time', 'chromeos_update', 'chromeos_wifi_sanity', # =========================== # Permanently-disabled tests. # =========================== # You cannot resize the browser window on ChromeOS. '-browser.BrowserTest.testWindowResize', # No codesign verification on ChromeOS. '-codesign', # Import tests are invalid on ChromeOS since Chrome is the only browser. '-imports', # Multi-profile doesn't apply to chromeos yet. '-multiprofile', # These require a custom setup on ChromeOS and have their own suite for # this platform. '-policy', '-policy_prefs_ui', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crosbug.com/19556 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', # FileBrowser test API as broken - crosbug.com/24257 '-chromeos_file_browser.ChromeosFileBrowserTest.testCopyFolderInDialog', '-chromeos_file_browser.ChromeosFileBrowserTest.testCopyFolderInFullPage', '-chromeos_file_browser.ChromeosFileBrowserTest.testCreateFolderInDialog', '-chromeos_file_browser.ChromeosFileBrowserTest.testCreateFolderInFullPage', '-chromeos_file_browser.ChromeosFileBrowserTest.testCutFolderInDialog', '-chromeos_file_browser.ChromeosFileBrowserTest.testCutFolderInFullPage', '-chromeos_file_browser.ChromeosFileBrowserTest.testDeleteFolderInDialog', '-chromeos_file_browser.ChromeosFileBrowserTest.testDeleteFolderInFullPage', '-chromeos_file_browser.ChromeosFileBrowserTest.testGetSelectedDirectorySizeStats', '-chromeos_file_browser.ChromeosFileBrowserTest.testRenameFolderInDialog', '-chromeos_file_browser.ChromeosFileBrowserTest.testRenameFolderInFullPage', '-chromeos_file_browser.ChromeosFileBrowserTest.testSelectMultipleFoldersInFullPage', # Downloads panel stays even after declining a download. # crosbug.com/14728 '-downloads.DownloadsTest.testDeclineDangerousDownload', # crosbug.com/19556 '-extensions.ExtensionsTest.testAllowAccessFileURLs', '-extensions.ExtensionsTest.testAllowIncognitoExtension', '-extensions.ExtensionsTest.testDisableEnableExtension', # Need internal data dirs in autotest package. crosbug.com/6855 '-find_in_page.FindMatchTests.testSearchInPDF', # crosbug.com/24496 '-infobars.InfobarTest.testPluginCrashForMultiTabs', # crbug.com/109035 '-infobars.InfobarTest.testPluginCrashInfobar', # Flaky: crosbug.com/14439 '-instant', # crosbug.com/14256 '-ntp.NTPTest.testLaunchAppFullScreen', # Content history broken in omnibox. crosbug.com/14416 '-omnibox.OmniboxTest.testContentHistory', # crbug.com/91033 '-omnibox.OmniboxTest.testOmniboxSearchHistory', # crosbug.com/19760 '-passwords.PasswordTest.testClearFetchedCredForNewUserName', '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', # onunload popups get created in the same window on chromeos # crosbug.com/14924 '-popups.PopupsTest.testPopupsLaunchWhenTabIsClosed', # Session restore not working with PyAuto. crosbug.com/12648 '-prefs.PrefsTest.testNavigationStateOnSessionRestore', '-prefs.PrefsTest.testSessionRestoreURLs', '-prefs.PrefsTest.testSessionRestore', # sync tests were running into a secondary passphrase. crosbug.com/15016. '-sync', # crbug.com/109035 '-plugins.PluginsTest.testBlockPluginException', # Deal with i18n chars. crosbug.com/12639 '-autofill.AutofillTest.testFillProfileCrazyCharacters', '-downloads.DownloadsTest.testCrazyFilenames', '-omnibox.OmniboxTest.testCrazyFilenames', # crosbug.com/16977 '-chromeos_wifi_sanity.ChromeosWifiSanity.testConnectToHiddenWiFiNonExistent', # crosbug.com/20025 '-chromeos_browser.ChromeosBrowserTest.testFullScreen', # crosbug.com/19556 '-pyauto_webdriver', # crbug.com/102549 '-notifications.NotificationsTest.testOriginPrefsNotSavedInIncognito', # crbug.com/103379 '-policy.PolicyTest.testDisable3DAPIs', ], }, # The CONTINUOUS_PUBLIC is a subset of CONTINUOUS suite # and excludes tests that depend on internal data 'CONTINUOUS_PUBLIC': { 'all': [ '@CONTINUOUS', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # These currently fail even with internal data '-chromeos_firmware_version_checker.ChromeosFirmwareVersionChecker.testECVersion', '-chromeos_firmware_version_checker.ChromeosFirmwareVersionChecker.testFirmwareVersion', '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows', # =========================== # Permanently-disabled tests. # =========================== '-infobars.InfobarTest.testPluginCrashForMultiTabs', '-chromeos_security.ChromeosSecurity.testBundledCrxPermissions', '-chromeos_security.ChromeosSecurity.testCannotViewLocalFiles', '-chromeos_security.ChromeosSecurity.testComponentExtensionPermissions', '-chromeos_security.ChromeosSecurity.testNoUnexpectedExtensions', '-flash.FlashTest.testCanLoadFlash', '-flash.FlashTest.testFlashLoadsAfterKill', '-flash.FlashTest.testFlashIncognitoMode', '-flash.FlashTest.testFlashWithMultipleTabs', '-flash.FlashTest.testSingleFlashPluginProcess', '-ntp.NTPTest.testGetAppsWhenInstallApp', '-ntp.NTPTest.testLaunchAppNewWindow', '-ntp.NTPTest.testLaunchAppPinnedTab', '-ntp.NTPTest.testLaunchAppRegularTab', '-ntp.NTPTest.testLaunchAppWithDefaultSettings', '-ntp.NTPTest.testUninstallApp', '-omnibox.OmniboxTest.testAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testAppNameWithNumberSearch', '-omnibox.OmniboxTest.testAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testAppSearchWithVeryLongAppName', '-omnibox.OmniboxTest.testBasicAppSearch', '-omnibox.OmniboxTest.testBeginningPartAppNameSearchInNewTab', '-omnibox.OmniboxTest.testEndPartAppNameSearchInNewTab', '-omnibox.OmniboxTest.testIncognitoAppComboNameWithSpecialCharSearch', '-omnibox.OmniboxTest.testIncognitoAppNameWithSpaceSearch', '-omnibox.OmniboxTest.testIncognitoAppSearchWithVeryLongAppName', '-omnibox.OmniboxTest.testRepeatedlyAppLaunchInTabs', '-passwords.PasswordTest.testDisplayAndSavePasswordInfobar', '-passwords.PasswordTest.testInfoBarDisappearByNavigatingPage', '-passwords.PasswordTest.testInfoBarDisappearByReload', '-passwords.PasswordTest.testNeverSavePasswords', '-plugins.PluginsTest.testBlockAllPlugins', '-popups.PopupsTest.testMultiplePopups', '-popups.PopupsTest.testPopupBlockedEverySec', '-process_count.ProcessCountTest.testProcessCountCombination', '-process_count.ProcessCountTest.testProcessCountFlashProcess', ], 'win': [ ], 'mac': [ ], 'linux': [ ], 'chromeos': [ # These tests fail due to crosbug.com/19556 '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs', # Media files test uses internal media files '-chromeos_file_browser.ChromeosFileBrowserTest.testOpenMediaFiles', ], }, # The FULL suite includes CONTINUOUS, and more. # This the suite that gets run on 'Google Chrome' builds. 'FULL': { 'all': [ '@CONTINUOUS', 'nacl_sdk', 'translate', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # This test fails due to crbug.com/35485 '-translate.TranslateTest.testBarNotVisibleOnSSLErrorPage', # This test fails due to crbug.com/50706 '-translate.TranslateTest.testToggleTranslateOption', # crbug.com/85601 '-translate.TranslateTest.testAlwaysTranslateInIncognito', # This test fails due to crbug.com/85633 '-translate.TranslateTest.testSessionRestore', # The source is behind. Waiting for dev to automate the update. '-nacl_sdk.NaClSDKTest.testVerifyNaClSDKChecksum', # crbug.com/98681 '-nacl_sdk.NaClSDKTest.testNaClSDK', '-nacl_sdk.NaClSDKTest.testVerifyPrereleaseGallery', # crbug.com/109148 '-extensions.ExtensionsTest.testTriggerBrowserAction', '-extensions.ExtensionsTest.testTriggerPageAction', '-extensions.ExtensionsTest.testTriggerPageActionWithPopup', # crbug.com/109160 '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtension', ], 'win': [ 'enterprise', 'gpu', # =========================== # Permanently-disabled tests. # =========================== # Can't use automation to change the policy data on Chrome builds. '-policy', '-policy_prefs_ui', # Preloaded extensions affect process count: crbug.com/94350 '-process_count', # Disable sync.SyncIntegrationTest on the QA bots in FULL, # until sync protobufs are copied over. '-sync.SyncIntegrationTest', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/101787 '-enterprise.EnterpriseTestReverse.testDefaultSearchProviderDisabled', # crbug.com/96896 '-enterprise.EnterpriseTest.testDisabledPlugins', '-enterprise.EnterpriseTest.testEnabledPlugins', # crbug.com/100893 '-enterprise.EnterpriseTestReverse.testEnableSPDY', # crbug.com/105948 '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum', '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry', ], 'mac': [ # =========================== # Permanently-disabled tests. # =========================== # Can't use automation to change the policy data on Chrome builds. '-policy', '-policy_prefs_ui', # Consistently failing on Mac QA bots only: crbug.com/97810 '-process_count', ], 'linux': [ 'enterprise', # Add back the omnibox tests excluded from the CONTINUOUS suite above # because they do not produce omnibox results under virtual X on linux. # crbug.com/85310 'instant', 'omnibox.OmniboxTest.testAppComboNameWithSpecialCharSearch', 'omnibox.OmniboxTest.testAppNameWithNumberSearch', 'omnibox.OmniboxTest.testAppNameWithSpaceSearch', 'omnibox.OmniboxTest.testAppSearchWithVeryLongAppName', 'omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch', 'omnibox.OmniboxTest.testAutoCompleteForSearch', 'omnibox.OmniboxTest.testBasicAppSearch', 'omnibox.OmniboxTest.testBookmarkResultInNewTabAndWindow', 'omnibox.OmniboxTest.testCrazyFilenames', 'omnibox.OmniboxTest.testDifferentTypesOfResults', 'omnibox.OmniboxTest.testFetchHistoryResultItems', 'omnibox.OmniboxTest.testGoogleSearch', 'omnibox.OmniboxTest.testInlineAutoComplete', 'omnibox.OmniboxTest.testSelect', 'omnibox.OmniboxTest.testSuggest', 'omnibox.OmniboxTest.testSuggestPref', # =========================== # Permanently-disabled tests. # =========================== # Can't use automation to change the policy data on Chrome builds. '-policy', '-policy_prefs_ui', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # crbug.com/101787 '-enterprise.EnterpriseTestReverse.testDefaultSearchProviderDisabled', # crbug.com/97179 '-enterprise.EnterpriseTest.testAlwaysAuthorizePlugins', ], 'chromeos': [ 'netflix', # =========================== # Permanently-disabled tests. # =========================== # No NaCl support on ChromeOS. '-nacl_sdk', # ================================================== # Disabled tests that need to be investigated/fixed. # ================================================== # Session restore not working with PyAuto. crosbug.com/12648 '-translate.TranslateTest.testSessionRestore', # Deal with i18n chars. crosbug.com/12639 '-translate.TranslateTest.testDownloadsNotTranslated', ], }, 'EMPTY': { }, 'CHROMEOS_CONNECTIVITY': { 'chromeos': [ 'chromeos_wifi_functional', 'chromeos_wifi_compliance', 'wifi_downloads', 'wifi_notification', ], }, # Subset of CONTINUOUS_PUBLIC to be run in a ChromeOS VM. 'CHROMEOS_VM_PUBLIC': { 'chromeos': [ '@CONTINUOUS_PUBLIC', '-chromeos_firmware_version_checker', '-chromeos_power', '-chromeos_prefs.ChromeosPrefsTest.testCaptureUserPhoto', '-chromeos_wifi_sanity', ], }, # ChromeOS flash tests. 'CHROMEOS_FLASH': { 'chromeos': [ 'flash', ], }, # ChromeOS login tests. 'CHROMEOS_LOGIN': { 'chromeos': [ 'chromeos_login', ], }, # ChromeOS policy tests. These are shared with Desktop Chrome tests, but # require custom setup on ChromeOS. 'CHROMEOS_POLICY': { 'chromeos': [ 'policy', 'policy_prefs_ui', ], }, # ChromeOS volume tests. 'CHROMEOS_VOLUME': { 'chromeos': [ 'chromeos_volume', ], }, # ChromeOS performance tests. 'CHROMEOS_PERF': { 'chromeos': [ 'perf', ], }, # Subset of performance tests for seaboard. 'CHROMEOS_PERF_SEABOARD': { 'chromeos': [ 'perf.ScrollTest.testBlankPageScroll', 'perf.ScrollTest.testGooglePlusScroll', 'perf.ScrollTest.testTextScroll', ], }, # HTML5 media performance tests. 'AV_PERF': { 'linux': [ 'media.media_constrained_network_perf', ], }, }