[Android] Refactor bb_host_steps to prepare for downstream usage.

- Refactored bb_host_steps.py to take a list of steps.
- Moved run step logic into bb_utils

BUG=249997
NOTRY=true

Review URL: https://chromiumcodereview.appspot.com/15817022

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@206555 0039d316-1c4b-4281-b951-d872f2087c98

3 files changed, 80 insertions, 78 deletions

diff --git a/build/android/buildbot/bb_host_steps.py b/build/android/buildbot/bb_host_steps.py
index 58d2e0e..0224116 100755
--- a/build/android/buildbot/bb_host_steps.py
+++ b/build/android/buildbot/bb_host_steps.py
@@ -25,7 +25,7 @@ def SrcPath(*path):
   return os.path.join(constants.DIR_SOURCE_ROOT, *path)
 
 
-def CheckWebViewLicenses():
+def CheckWebViewLicenses(_):
   buildbot_report.PrintNamedStep('check_licenses')
   RunCmd([SrcPath('android_webview', 'tools', 'webview_licenses.py'), 'scan'],
          warning_code=1)
@@ -48,43 +48,46 @@ def RunHooks(build_type):
   RunCmd(['gclient', 'runhooks'], halt_on_failure=True)
 
 
-def Compile(build_type, args, experimental=False):
+def Compile(options):
+  RunHooks(options.target)
   cmd = [os.path.join(SLAVE_SCRIPTS_DIR, 'compile.py'),
          '--build-tool=ninja',
          '--compiler=goma',
-         '--target=%s' % build_type,
+         '--target=%s' % options.target,
          '--goma-dir=%s' % bb_utils.GOMA_DIR]
-  if experimental:
-    for compile_target in args:
+  build_targets = options.build_targets.split(',')
+  buildbot_report.PrintNamedStep('compile')
+  for build_target in build_targets:
+    RunCmd(cmd + ['--build-args=%s' % build_target], halt_on_failure=True)
+  if options.experimental:
+    for compile_target in EXPERIMENTAL_TARGETS:
       buildbot_report.PrintNamedStep('Experimental Compile %s' % compile_target)
       RunCmd(cmd + ['--build-args=%s' % compile_target], flunk_on_failure=False)
-  else:
-    buildbot_report.PrintNamedStep('compile')
-    RunCmd(cmd + ['--build-args=%s' % ' '.join(args)], halt_on_failure=True)
 
 
-def ZipBuild(properties):
+def ZipBuild(options):
   buildbot_report.PrintNamedStep('zip_build')
   RunCmd([
       os.path.join(SLAVE_SCRIPTS_DIR, 'zip_build.py'),
       '--src-dir', constants.DIR_SOURCE_ROOT,
       '--build-dir', SrcPath('out'),
       '--exclude-files', 'lib.target,gen,android_webview,jingle_unittests']
-      + properties)
+      + bb_utils.EncodeProperties(options))
 
 
-def ExtractBuild(properties):
+def ExtractBuild(options):
   buildbot_report.PrintNamedStep('extract_build')
-  RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'extract_build.py'),
-          '--build-dir', SrcPath('build'),
-          '--build-output-dir', SrcPath('out')] + properties,
-         warning_code=1)
+  RunCmd(
+      [os.path.join(SLAVE_SCRIPTS_DIR, 'extract_build.py'),
+       '--build-dir', SrcPath('build'), '--build-output-dir',
+       SrcPath('out')] + bb_utils.EncodeProperties(options),
+       warning_code=1)
 
 
-def FindBugs(is_release):
+def FindBugs(options):
   buildbot_report.PrintNamedStep('findbugs')
   build_type = []
-  if is_release:
+  if options.target == 'Release':
     build_type = ['--release-build']
   RunCmd([SrcPath('build', 'android', 'findbugs_diff.py')] + build_type)
   RunCmd([SrcPath(
@@ -92,7 +95,7 @@ def FindBugs(is_release):
       'run_findbugs_plugin_tests.py')] + build_type)
 
 
-def BisectPerfRegression():
+def BisectPerfRegression(_):
   buildbot_report.PrintNamedStep('Bisect Perf Regression')
   RunCmd([SrcPath('tools', 'prepare-bisect-perf-regression.py'),
           '-w', os.path.join(constants.DIR_SOURCE_ROOT, os.pardir)])
@@ -101,50 +104,32 @@ def BisectPerfRegression():
           '-p', bb_utils.GOMA_DIR])
 
 
+def GetHostSteps():
+  return [
+      ('compile', Compile),
+      ('extract_build', ExtractBuild),
+      ('check_webview_licenses', CheckWebViewLicenses),
+      ('bisect_perf_regression', BisectPerfRegression),
+      ('findbugs', FindBugs),
+      ('zip_build', ZipBuild)
+  ]
+
+
 def main(argv):
   parser = bb_utils.GetParser()
-  parser.add_option('--host-tests', help='Comma separated list of host tests.')
-  parser.add_option('--build-args', default='All',
+  parser.add_option('--steps', help='Comma separated list of host tests.')
+  parser.add_option('--build-targets', default='All',
                     help='Comma separated list of build targets.')
-  parser.add_option('--compile', action='store_true',
-                    help='Indicate whether a compile step should be run.')
   parser.add_option('--experimental', action='store_true',
                     help='Indicate whether to compile experimental targets.')
-  parser.add_option('--zip-build', action='store_true',
-                    help='Indicate whether the build should be zipped.')
-  parser.add_option('--extract-build', action='store_true',
-                    help='Indicate whether a build should be downloaded.')
-  parser.add_option('--bisect-perf-regression', action='store_true',
-                    help='Bisect a perf regression.')
 
   options, args = parser.parse_args(argv[1:])
   if args:
     return sys.exit('Unused args %s' % args)
 
-  host_tests = []
-  if options.host_tests:
-    host_tests = options.host_tests.split(',')
-  unknown_tests = set(host_tests) - VALID_HOST_TESTS
-  if unknown_tests:
-    return sys.exit('Unknown host tests %s' % list(unknown_tests))
-
-  build_type = options.factory_properties.get('target', 'Debug')
-
-  if options.bisect_perf_regression:
-    BisectPerfRegression()
-  if options.compile:
-    if 'check_webview_licenses' in host_tests:
-      CheckWebViewLicenses()
-    RunHooks(build_type)
-    Compile(build_type, options.build_args.split(','))
-    if options.experimental:
-      Compile(build_type, EXPERIMENTAL_TARGETS, True)
-    if 'findbugs' in host_tests:
-      FindBugs(build_type == 'Release')
-    if options.zip_build:
-      ZipBuild(bb_utils.EncodeProperties(options))
-  if options.extract_build:
-    ExtractBuild(bb_utils.EncodeProperties(options))
+  setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
+
+  bb_utils.RunSteps(GetHostSteps(), options)
 
 
 if __name__ == '__main__':
diff --git a/build/android/buildbot/bb_run_bot.py b/build/android/buildbot/bb_run_bot.py
index 20a435a..89b272f 100755
--- a/build/android/buildbot/bb_run_bot.py
+++ b/build/android/buildbot/bb_run_bot.py
@@ -19,7 +19,8 @@ BotConfig = collections.namedtuple(
     'BotConfig', ['bot_id', 'host_obj', 'test_obj'])
 
 HostConfig = collections.namedtuple(
-    'HostConfig', ['host_step_args', 'extra_gyp_defines', 'target_arch'])
+    'HostConfig',
+    ['host_steps', 'extra_args', 'extra_gyp_defines', 'target_arch'])
 
 TestConfig = collections.namedtuple('Tests', ['tests', 'extra_args'])
 
@@ -84,7 +85,8 @@ def GetCommands(options, bot_config):
   """
   property_args = bb_utils.EncodeProperties(options)
   commands = [['build/android/buildbot/bb_host_steps.py'] +
-              bot_config.host_obj.host_step_args + property_args]
+              ['--steps=%s' % ','.join(bot_config.host_obj.host_steps)] +
+              property_args + (bot_config.host_obj.extra_args or [])]
 
   test_obj = bot_config.test_obj
   if test_obj:
@@ -99,10 +101,10 @@ def GetCommands(options, bot_config):
 
 
 def GetBotStepMap():
-  compile_opt = ['--compile']
-  std_host_tests = ['--host-tests=check_webview_licenses,findbugs']
-  std_build_opts = ['--compile', '--zip-build']
-  std_test_opts = ['--extract-build']
+  compile_step = ['compile']
+  std_host_tests = ['check_webview_licenses', 'findbugs']
+  std_build_steps = ['compile', 'zip_build']
+  std_test_steps = ['extract_build']
   std_tests = ['ui', 'unit']
   flakiness_server = '--upload-to-flakiness-server'
 
@@ -112,42 +114,42 @@ def GetBotStepMap():
   def T(tests, extra_args=None):
     return TestConfig(tests, extra_args)
 
-  def H(host_step_args, extra_gyp=None, target_arch=None):
-    return HostConfig(host_step_args, extra_gyp, target_arch)
+  def H(host_steps, extra_args=None, extra_gyp=None, target_arch=None):
+    return HostConfig(host_steps, extra_args, extra_gyp, target_arch)
 
   bot_configs = [
       # Main builders
-      B('main-builder-dbg', H(std_build_opts + std_host_tests)),
-      B('main-builder-rel', H(std_build_opts)),
+      B('main-builder-dbg', H(std_build_steps + std_host_tests)),
+      B('main-builder-rel', H(std_build_steps)),
       B('main-clang-builder',
-        H(compile_opt, 'clang=1 component=shared_library')),
-      B('main-clobber', H(compile_opt)),
-      B('main-tests', H(std_test_opts), T(std_tests, [flakiness_server])),
+        H(compile_step, extra_gyp='clang=1 component=shared_library')),
+      B('main-clobber', H(compile_step)),
+      B('main-tests', H(std_test_steps), T(std_tests, [flakiness_server])),
 
       # Other waterfalls
-      B('asan-builder-tests', H(compile_opt, 'asan=1'),
+      B('asan-builder-tests', H(compile_step, extra_gyp='asan=1'),
         T(std_tests, ['--asan'])),
-      B('chromedriver-fyi-tests-dbg', H(std_test_opts),
+      B('chromedriver-fyi-tests-dbg', H(std_test_steps),
         T(['chromedriver'], ['--install=ChromiumTestShell'])),
       B('fyi-builder-dbg',
-        H(std_build_opts + std_host_tests + ['--experimental'])),
-      B('fyi-builder-rel', H(std_build_opts + ['--experimental'])),
-      B('fyi-tests-dbg-ics-gn', H(compile_opt + [ '--experimental']),
+        H(std_build_steps + std_host_tests, ['--experimental'])),
+      B('fyi-builder-rel', H(std_build_steps,  ['--experimental'])),
+      B('fyi-tests-dbg-ics-gn', H(compile_step, [ '--experimental']),
         T(std_tests, ['--experimental', flakiness_server])),
-      B('fyi-tests', H(std_test_opts),
+      B('fyi-tests', H(std_test_steps),
         T(std_tests, ['--experimental', flakiness_server])),
       B('fyi-component-builder-tests-dbg',
-        H(compile_opt, 'component=shared_library'),
+        H(compile_step, extra_gyp='component=shared_library'),
         T(std_tests, ['--experimental', flakiness_server])),
-      B('perf-bisect-builder-tests-dbg', H(['--bisect-perf-regression'])),
-      B('perf-tests-rel', H(std_test_opts), T([], ['--install=ContentShell'])),
-      B('webkit-latest-webkit-tests', H(std_test_opts),
+      B('perf-bisect-builder-tests-dbg', H(['bisect_perf_regression'])),
+      B('perf-tests-rel', H(std_test_steps), T([], ['--install=ContentShell'])),
+      B('webkit-latest-webkit-tests', H(std_test_steps),
         T(['webkit_layout', 'webkit'])),
-      B('webkit-latest-contentshell', H(compile_opt), T(['webkit_layout'])),
-      B('builder-unit-tests', H(compile_opt), T(['unit'])),
+      B('webkit-latest-contentshell', H(compile_step), T(['webkit_layout'])),
+      B('builder-unit-tests', H(compile_step), T(['unit'])),
 
       # Generic builder config (for substring match).
-      B('builder', H(std_build_opts)),
+      B('builder', H(std_build_steps)),
   ]
 
   bot_map = dict((config.bot_id, config) for config in bot_configs)
diff --git a/build/android/buildbot/bb_utils.py b/build/android/buildbot/bb_utils.py
index 3f70caaa..74b2792 100644
--- a/build/android/buildbot/bb_utils.py
+++ b/build/android/buildbot/bb_utils.py
@@ -76,3 +76,18 @@ def GetParser():
 def EncodeProperties(options):
   return ['--factory-properties=%s' % json.dumps(options.factory_properties),
           '--build-properties=%s' % json.dumps(options.build_properties)]
+
+
+def RunSteps(all_steps, options):
+  if not options.steps:
+    return
+
+  steps = options.steps.split(',')
+  unknown_steps = set(steps) - set(step for step, _ in all_steps)
+  if unknown_steps:
+    print >> sys.stderr, 'FATAL: Unknown steps %s' % list(unknown_steps)
+    sys.exit(1)
+
+  for step, cmd in all_steps:
+    if step in steps:
+      cmd(options)