path: root/chrome/test/functional/PYAUTO_TESTS

# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This file lists the pyauto tests that run as a part of the functional test
# suite.
#
# Tests can be enabled on a per-platform basis. Tests not listed here will
# not be run.
#
# Within each platform, tests are specified in up to 3 groups: (1) enabled
# tests; (2) tests that are permanently disabled because they do not apply
# to the given platform and hence should not be run; and (3) disabled tests
# that need to be investigated/fixed.  Tests are listed alphabetically within
# each group.
#
# Test names can be specified in any of the following ways:
# 1. as a module, in which case all tests in that module will be run
#      example: 'test_basic'
# 2. or as a test case, in which case all tests in that test case will be run
#      example: 'test_basic.SimpleTest'
# 3. or as an individual test
#      example: 'test_basic.SimpleTest.testCanOpenGoogle'
#
# Tests beginning with '-' will be excluded.  This can be used to enforce
# exclusions for a particular platform.
# Names beginning with '@' will be treated as suite names and will be
# recursively expanded.

{
  # This suite gets run on 'Google Chrome' builds.
  'FULL': {
    'all': [
      'about_plugins_ui.AboutPluginsUITest',
      'about_plugins_ui.ChromeAboutPluginsUITest',
      'apptest',
      'autofill',
      'codesign',
      'crash_reporter',
      'execute_javascript',
      'extensions',
      'fullscreen_mouselock',
      'gtalk.test_basic',
      'imports',
      'infobars',
      'media.media_basic_playback',
      'multiprofile',
      'nacl_sdk',
      'ntp',
      'omnibox',
      'passwords',
      'prefs',
      'prefs_ui',
      'pyauto_webdriver',
      'search_engines',
      'special_tabs',
      'test_pyauto',

      # ===========================
      # Permanently-disabled tests.
      # ===========================

      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      # crbug.com/145006
      '-about_plugins_ui.ChromeAboutPluginsUITest.testEnableAndDisableFlashPlugin',
      # crbug.com/100365
      '-autofill.AutofillTest.testDisplayLineItemForEntriesWithNoCCNum',
      # crbug.com/171828
      '-autofill.AutofillTest.testNoDuplicatePhoneNumsInPrefs',
      # The source is behind.  Waiting for dev to automate the update.
      # crbug.com/109160
      '-execute_javascript.ExecuteJavascriptTest.testExecuteJavascriptInExtension',
      # crbug.com/123396
      '-fullscreen_mouselock.FullscreenMouselockTest.testPatternsForFSAndML',
      # crbug.com/132665
      '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavBackToPrevPage',
      '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSExitWhenNavToNewPage',
      '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavBackToPrevPage',
      '-fullscreen_mouselock.FullscreenMouselockTest.testMLExitWhenNavToNewPage',
      # crbug.com/140460
      '-fullscreen_mouselock.FullscreenMouselockTest.testNoTabFSExitWhenJSExitMouseLock',
      # crosbug.com/136875
      '-fullscreen_mouselock.FullscreenMouselockTest.testTabFSDoesNotExitForAnchorLinks',
      # crbug.com/179263
      '-gtalk.test_basic.BasicTest.testCurrentVersion',
      '-gtalk.test_basic.BasicTest.testRCVersion',
      # crbug.com/165796
      '-infobars.OneClickInfobarTest',
      # crbug.com/131874
      '-infobars.OneClickInfobarTest.testNoOneClickInfobarAfterCancel',
      # crbug.com/133315
      '-infobars.OneClickInfobarTest.testDisplayOneClickInfobarAfterDismiss',
      # Mysteriously broken?
      # crbug.com/138857
      '-multiprofile.MultiprofileTest.testMakeSearchEngineDefaultInMultiprofile',
      # crbug.com/179268
      '-ntp.NTPTest.testCannotUninstallWebStore',
      '-ntp.NTPTest.testGetAppsInNewProfile',
      '-ntp.NTPTest.testGetAppsWhenInstallApp',
      '-ntp.NTPTest.testGetAppsWhenInstallNonApps',
      '-ntp.NTPTest.testUninstallApp',
      # crbug.com/143308
      '-omnibox.OmniboxLiveTest.testGoogleSearch',
      # crbug.com/71715
      '-omnibox.OmniboxTest.testHistoryResult',
      # crbug.com/123019
      '-omnibox.OmniboxTest.testAutoCompleteForNonAsciiSearch',
      # crbug.com/137041
      '-omnibox.OmniboxTest.testDifferentTypesOfResults',
      # crbug.com/162341
      '-prefs.PrefsTest.testAllowSelectedGeoTracking',
      '-prefs.PrefsTest.testDismissedInfobarSavesNoEntry',
      '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied',
      '-prefs.PrefsTest.testGeolocationPref',
      # crbug.com/85600
      '-prefs.PrefsTest.testNavigationStateOnSessionRestore',
      # crbug.com/157271
      '-prefs_ui.BasicSettingsUITest.testCancelStartupURLSetting',
      '-prefs_ui.BasicSettingsUITest.testSetStartupPages',
      '-prefs_ui.BasicSettingsUITest.testUseCurrentPagesForStartup',
      # crbug.com/162341
      '-prefs_ui.PrefsUITest.testBehaviorValueCorrectlyDisplayed',
      # crbug.com/112051
      '-prefs_ui.PrefsUITest.testChangeExceptionBehaviorUI',
      # crbug.com/157271
      '-prefs_ui.PrefsUITest.testDeleteExceptionUI',
      # crbug.com/151973
      '-prefs_ui.PrefsUITest.testInitialLineEntryInIncognitoUI',
      # crbug.com/162341
      '-prefs_ui.PrefsUITest.testLocationSettingOptionsUI',
      '-prefs_ui.PrefsUITest.testNoInitialLineEntryInUI',
      # crbug.com/132285
      '-special_tabs.SpecialTabsTest.testSpecialURLTabs',
      # crbug.com/168081
      '-ntp.NTPTest.testCloseOneTab',
    ],

    'win': [
      'gpu',

      # ===========================
      # Permanently-disabled tests.
      # ===========================

      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      # crbug.com/105948
      '-autofill.AutofillTest.testPostalCodeAndStateLabelsBasedOnCountry',
      # crbug.com/111289
      '-extensions.ExtensionsTest.testAllowAccessFileURLs',
      # crbug.com/113090
      '-extensions.ExtensionsTest.testAllowIncognitoExtension',
      # crbug.com/94682
      '-gpu.GpuTest.test2dCanvas',
      # Disabled on vista in the actual test. crbug.com/89767
      # -imports.ImportsTest.testImportFirefoxDataTwice
      # Disabled on vista in the actual test. crbug.com/89767
      # -imports.ImportsTest.testFirefoxImportFromPrefs
      # Disabled on vista in the actual test. crbug.com/89767
      # -imports.ImportsTest.testImportFirefoxPasswordsFromPrefs
      # crbug.com/171490
      '-passwords.PasswordTest.testPasswdInfoNotStoredWhenAutocompleteOff',
      # crbug.com/117569
      '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows',
      # crbug.com/98526
      '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser',
      '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox',
    ],

    'mac': [
      # ===========================
      # Permanently-disabled tests.
      # ===========================

      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      # Keychain popups make autofill/password tests difficult: crbug.com/49378
      '-prefs_ui.PrefsUITest.testSetPasswordAndDelete',
      # codesign tests should run *after* signing. crbug.com/50481
      '-codesign',
      # crbug.com/124922
      '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenBrowserLoseFocus',
      # crbug.com/125989
      '-fullscreen_mouselock.FullscreenMouselockTest.testMouseLockExitWhenAlertDialogShow',
      # Fails on chrome-mac-10_7-qa only: crbug.com/124886
      '-fullscreen_mouselock.FullscreenMouselockTest.testPrefsForFullscreenExit',
      '-fullscreen_mouselock.FullscreenMouselockTest.testNoMouseLockWhenCancelFS',
      # crbug.com/114244
      '-imports.ImportsTest.testImportSafariDataTwice',
      # crbug.com/121484
      '-multiprofile.MultiprofileTest.test20NewProfiles',
      '-ntp.NTPTest.testLaunchAppNewWindow',  # crbug.com/79812
      # crbug.com/70437
      '-omnibox.OmniboxTest.testHistoryResult',
      # crbug.com/91617
      '-omnibox.OmniboxTest.testContentHistory',
      # Keychain popups make autofill/password tests difficult: crbug.com/49378
      '-passwords',
      # crbug.com/69619
      '-search_engines.SearchEnginesTest.testDiscoverSearchEngine',
      # crbug.com/98526
      '-pyauto_webdriver.PyAutoWebDriverTest.testCanConnectToRestartedBrowser',
      '-pyauto_webdriver.PyAutoWebDriverTest.testTypeIntoTextBox',
    ],

    'linux': [
      'test_clean_exit',

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      # System password manager obstructs password automation.
      '-passwords',

      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      # crbug.com/111289
      '-extensions.ExtensionsTest.testAllowAccessFileURLs',
      # crbug.com/91033
      '-omnibox.OmniboxTest.testOmniboxSearchHistory',
    ],

    'chromeos': [
      'chromeos_basic',
      'chromeos_browser',
      'chromeos_crosh',
      'chromeos_file_browser',
      'chromeos_offline',
      'chromeos_power',
      'chromeos_prefs',
      'chromeos_security',
      'chromeos_time',
      'chromeos_update',
      'chromeos_wifi_sanity',
      'doc_viewing',
      'secure_shell',
      'youtube',

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      # No codesign verification on ChromeOS.
      '-codesign',
      # Import tests are invalid on ChromeOS since Chrome is the only browser.
      '-imports',
      # Sync is already signed in with the login account.
      # So one-click infobar tests do tno apply to chromeos.
      '-infobars.OneClickInfobarTest',
      # Multi-profile doesn't apply to chromeos yet.
      '-multiprofile',
      '-ntp.NTPTest.testDifferentProfileNotAppearInMostVisited',
      # No NaCl support on ChromeOS.
      '-nacl_sdk',
      # You cannot adjust the downloads directory on ChromeOS.
      '-prefs.PrefsTest.testDownloadDirPref',

      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      # crbug.com/132337
      '-autofill.AutofillTest.testTabOrderForEditAddress',
      # crosbug.com/19556
      '-extensions.ExtensionsTest.testAllowAccessFileURLs',
      '-extensions.ExtensionsTest.testAllowIncognitoExtension',
      '-extensions.ExtensionsTest.testDisableEnableExtension',
      # crbug.com/134593
      '-gtalk.test_basic.BasicTest.testCurrentVersion',
      '-gtalk.test_basic.BasicTest.testRCVersion',
      # crosbug.com/24496
      '-infobars.InfobarTest.testPluginCrashForMultiTabs',
      # crbug.com/109035
      '-infobars.InfobarTest.testPluginCrashInfobar',
      # crosbug.com/14256
      '-ntp.NTPTest.testLaunchAppFullScreen',
      # Content history broken in omnibox.  crosbug.com/14416
      '-omnibox.OmniboxTest.testContentHistory',
      # crbug.com/91033
      '-omnibox.OmniboxTest.testOmniboxSearchHistory',
      # crosbug.com/19760
      '-passwords.PasswordTest.testClearFetchedCredForNewUserName',
      '-passwords.PasswordTest.testSavedPasswordInTabsAndWindows',
      # onunload popups get created in the same window on chromeos
      # Session restore not working with PyAuto.  crosbug.com/12648
      '-prefs.PrefsTest.testNavigationStateOnSessionRestore',
      '-prefs.PrefsTest.testSessionRestoreURLs',
      '-prefs.PrefsTest.testSessionRestore',
      # Deal with i18n chars.  crosbug.com/12639
      '-omnibox.OmniboxTest.testCrazyFilenames',
      # crosbug.com/16977
      '-chromeos_wifi_sanity.ChromeosWifiSanity.testConnectToHiddenWiFiNonExistent',
      # crosbug.com/20025
      '-chromeos_browser.ChromeosBrowserTest.testFullScreen',
      # Chrome driver does not work in Chrome OS.
      # crosbug.com/19556
      '-prefs_ui',
      '-prefs.PrefsTest.testGeolocationBlockedWhenTrackingDenied',
      '-prefs.PrefsTest.testGeolocationPref',
      '-pyauto_webdriver',
    ],
  },

  # Performance tests.
  'PERFORMANCE': {
    'all': [
      'perf',

      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      '-perf.HTML5BenchmarkTest',  # crbug.com/134476
      '-perf.LiveWebappLoadTest.testNewTabGmail',  # crbug.com/136554
      '-perf.ScrollTest.testGmailScroll',  # crbug.com/136554
      '-perf.WebGLTest.testWebGLField',  # crbug.com/132797

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      # Invoked outside of the pyauto_tests framework.
      '-perf.BenchmarkPerfTest.testSpaceport',
      '-perf.PopularSitesScrollTest.test2012Q3',
    ],
    'win': [
      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      '-perf.GPUPerfTest',  # Fails. Discuss with prachij@.
      '-perf.PageCyclerNetSimTest',  # Dependence missing: crbug.com/132559
      '-perf.LiveGamePerfTest',  # Requires linux /proc/stat.
      '-perf.YoutubePerfTest',  # AttributeError: AssertPlayingState.

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      '-perf.MemoryTest',  # Designed only for ChromeOS.
      '-perf.NetflixPerfTest',  # Designed only for ChromeOS.
    ],
    'mac': [
      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      '-perf.GPUPerfTest',  # Fails. Discuss with prachij@.
      '-perf.PageCyclerNetSimTest',  # Dependence missing: crbug.com/132559
      '-perf.LiveGamePerfTest',  # Requires linux /proc/stat.
      '-perf.YoutubePerfTest',  # AttributeError: AssertPlayingState.

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      '-perf.MemoryTest',  # Designed only for ChromeOS.
      '-perf.NetflixPerfTest',  # Designed only for ChromeOS.
    ],
    'linux': [
      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      '-perf.GPUPerfTest',  # Fails. Discuss with prachij@.
      '-perf.PageCyclerNetSimTest',  # Dependence missing: crbug.com/132559
      '-perf.WebGLTest',  # May not render WebGL; need to verify on QA bot.
      '-perf.YoutubePerfTest',  # AttributeError: AssertPlayingState.

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      '-perf.MemoryTest',  # Designed only for ChromeOS.
      '-perf.NetflixPerfTest',  # Designed only for ChromeOS.
    ],
    'chromeos': [
      # ==================================================
      # Disabled tests that need to be investigated/fixed.
      # ==================================================
      '-perf.NetflixPerfTest',  # crosbug.com/32320.
      '-perf.PageCyclerNetSimTest',  # Investigate Web Page Replay on ChromeOS.

      # ===========================
      # Permanently-disabled tests.
      # ===========================
      '-perf.GPUPerfTest',  # Designed only for Chrome desktop.
    ],
  },

  'PERFORMANCE_LAB': {
    'all': [
      'perf.BenchmarkPerfTest.testV8BenchmarkSuite',
      'perf.FlashTest.testFlashGaming',
      'perf.ScrollTest.testGooglePlusScroll',
      'perf.TabPerfTest.test20Tabs',
      'perf.WebGLTest.testWebGLSpaceRocks',
    ],
  },

  'EMPTY': {
  },

  'CHROMEOS_CONNECTIVITY': {
    'chromeos': [
      'chromeos_wifi_functional',
      'chromeos_wifi_compliance',
      'wifi_downloads',
      'wifi_notification',
    ],
  },

  # ChromeOS flash tests.
  'CHROMEOS_FLASH': {
    'chromeos': [
      'flash',
    ],
  },

  # ChromeOS login tests.
  'CHROMEOS_LOGIN': {
    'chromeos': [
      'chromeos_login',
      'chromeos_oobe',

      # crosbug.com/32583
      '-chromeos_login.ChromeosLoginCachedCredentialsUserPod.testCachedCredentialsUserPod',
    ],
  },

  # ChromeOS policy tests. Some of these are shared with Desktop Chrome tests
  # but require custom setup on ChromeOS. Others are specific to ChromeOS only.
  'CHROMEOS_POLICY': {
    'chromeos': [
      'chromeos_ephemeral',
      'chromeos_device_policy',
      'chromeos_onc',
      'chromeos_retail_mode',
    ],
  },

  # ChromeOS volume tests.
  'CHROMEOS_VOLUME': {
    'chromeos': [
      'chromeos_volume',
    ],
  },

  # ChromeOS Accessibility tests.
  'CHROMEOS_ACCESSIBILITY': {
    'chromeos': [
      'chromeos_accessibility',
    ],
  },

  # PGO performance suite.
  'PGO': {
    'chromeos': [
      'perf.BenchmarkPerfTest.testV8BenchmarkSuite',
      'perf.PageCyclerTest',
      'perf.ScrollTest.testGooglePlusScroll',
    ],
  },

  # Subset of ChromeOS performance tests for seaboard.
  'CHROMEOS_PERF_SEABOARD': {
    'chromeos': [
      'perf.ScrollTest.testBlankPageScroll',
      'perf.ScrollTest.testGooglePlusScroll',
      'perf.ScrollTest.testTextScroll',
    ],
  },

  # HTML5 media performance tests.
  'AV_PERF': {
     'linux': [
       'media.audio_latency_perf',
       'media.audio_playback_perf',
       'media.media_constrained_network_perf',
       'media.media_scrub_perf',
       'media.media_seek_perf',
       'media.media_stat_perf',
       'media.mixed_audio_latency_perf',
     ],
     'win': [
       'media.audio_latency_perf',
       'media.audio_playback_perf',
       'media.media_constrained_network_perf',
       'media.media_scrub_perf',
       'media.media_seek_perf',
       'media.media_stat_perf',
       'media.mixed_audio_latency_perf',
     ],
  },

  # WebRTC MediaStream tests. Requires webcam and audio device to be present on
  # the test machine.
  'WEBRTC': {
    'all': [
      'media_stream_infobar',
      'webrtc_brutality_test',
      'webrtc_call',
    ],
  },

  # WebRTC / AppRTC Integration tests.
  'WEBRTC_APPRTC': {
    'all': [
      'webrtc_apprtc_call',
    ],
  },

  # WebRTC quality tests. The video quality test requires a webcam, an audio
  # device and a special setup where the webcam plays a barcode-encoded video.
  # The webrtc_audio_quality test requires additional configuration as described
  # in the test's class comments.
  'WEBRTC_QUALITY': {
    'all': [
      'webrtc_video_quality',
      'webrtc_audio_quality',
    ],
    'mac': [
      '-webrtc_audio_quality',  # Not implemented.
    ],
  },

  # Trace event tests.
  'TRACING': {
    'all': [
      'tracing.tracing_smoke_test',
    ],
  },

  # Chromoting tests.
  'CHROMOTING': {
    'all': [
      'chromoting.auth',
      'chromoting.it2me_basic',
      'chromoting.me2me_connect',
      'chromoting.me2me_enable',
    ],
    'linux': [
      # TODO(yihongg): Me2me test against Linux is not working yet.
      '-chromoting.me2me_connect',
      '-chromoting.me2me_enable',
    ],
    'chromeos': [
      # ChromeOS doesn't yet support the chromoting host.
      '-chromoting.auth',
      '-chromoting.it2me_basic',
      '-chromoting.me2me_connect',
      '-chromoting.me2me_enable',
    ],
  },

  # Pyauto functional tests running on coverage bots.
  'CODE_COVERAGE': {
    'all': [
      '@FULL',
    ],
    'linux': [
      # These tests fail on coverage bots. Disabling for now.
      '-test_clean_exit',
      '-about_plugins_ui',
      '-autofill',
      '-pyauto_webdriver',
      '-extensions',
      '-flash',
      '-fullscreen_mouselock',
      '-gtalk.test_basic',
      '-infobars',
      '-multiprofile',
      '-prefs',
      '-prefs_ui',
      '-apptest',
      '-plugins',
      '-omnibox',
      '-special_tabs',
      '-ntp.NTPTest.testLaunchAppFullScreen',
      '-media.media_basic_playback.MediaBasicPlaybackTest.testBasicPlaybackMatrix',
      '-search_engines.SearchEnginesTest.testDiscoverSearchEngine',
      '-ntp.NTPTest.testUninstallApp',
      '-ntp.NTPTest.testLaunchAppWithDefaultSettings',
      '-ntp.NTPTest.testLaunchAppRegularTab',
      '-ntp.NTPTest.testLaunchAppPinnedTab',
      '-ntp.NTPTest.testGetAppsWhenInstallApp',
      '-ntp.NTPTest.testLaunchAppNewWindow',
    ],
  },
}