Revert of [telemetry] Script for finding all Telemetry dependencies. (https://codereview.chromium.org/103433002/)

Reason for revert:
May have caused a performance regression, reverting to confirm.
https://code.google.com/p/chromium/issues/detail?id=345374

Original issue's description:
> [telemetry] Script for finding all Telemetry dependencies.
> 
> Given a list of Python modules, it will find all:
> - bootstrap_deps folders, recursively.
> - Python dependencies.
> - Page sets in those modules' folders.
> - Serving dirs of those page sets.
> And print a list of all files, or zip them up.
> 
> Example bot command: tools/telemetry_tools/find_dependencies -v tools/perf/run_benchmark tools/perf/run_measurement tools/perf/record_wpr content/test/gpu/run_gpu_test -z telemetry.zip
> 
> 
> BUG=311380
> TEST=Ran it.
> 
> Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=251298

TBR=nduca@chromium.org,tonyg@chromium.org,achuith@chromium.org
NOTREECHECKS=true
NOTRY=true
BUG=311380

Review URL: https://codereview.chromium.org/170313010

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@252723 0039d316-1c4b-4281-b951-d872f2087c98

13 files changed, 24 insertions, 377 deletions

diff --git a/tools/perf/benchmarks/peacekeeper.py b/tools/perf/benchmarks/peacekeeper.py
index e74e421..2e71e56 100644
--- a/tools/perf/benchmarks/peacekeeper.py
+++ b/tools/perf/benchmarks/peacekeeper.py
@@ -22,7 +22,7 @@ from telemetry.page import page_measurement
 from telemetry.page import page_set
 from telemetry.value import merge_values
 
-class _PeaceKeeperMeasurement(page_measurement.PageMeasurement):
+class PeaceKeeperMeasurement(page_measurement.PageMeasurement):
 
   def WillNavigateToPage(self, page, tab):
     page.script_to_evaluate_on_commit = """
@@ -68,9 +68,9 @@ class _PeaceKeeperMeasurement(page_measurement.PageMeasurement):
     results.AddSummary('Score', 'score', total, 'Total')
 
 
-class _PeaceKeeperBenchmark(test.Test):
+class PeaceKeeperBenchmark(test.Test):
   """A base class for Peackeeper benchmarks."""
-  test = _PeaceKeeperMeasurement
+  test = PeaceKeeperMeasurement
 
   def CreatePageSet(self, options):
     """Makes a PageSet for PeaceKeeper benchmarks."""
@@ -98,7 +98,7 @@ class _PeaceKeeperBenchmark(test.Test):
     return page_set.PageSet.FromDict(page_set_dict, os.path.abspath(__file__))
 
 
-class PeaceKeeperRender(_PeaceKeeperBenchmark):
+class PeaceKeeperRender(PeaceKeeperBenchmark):
   """PeaceKeeper rendering benchmark suite.
 
   These tests measure your browser's ability to render and modify specific
@@ -113,7 +113,7 @@ class PeaceKeeperRender(_PeaceKeeperBenchmark):
                ]
 
 
-class PeaceKeeperData(_PeaceKeeperBenchmark):
+class PeaceKeeperData(PeaceKeeperBenchmark):
   """PeaceKeeper Data operations benchmark suite.
 
   These tests measure your browser's ability to add, remove and modify data
@@ -132,7 +132,7 @@ class PeaceKeeperData(_PeaceKeeperBenchmark):
                ]
 
 
-class PeaceKeeperDom(_PeaceKeeperBenchmark):
+class PeaceKeeperDom(PeaceKeeperBenchmark):
   """PeaceKeeper DOM operations benchmark suite.
 
   These tests emulate the methods used to create typical dynamic webpages.
@@ -172,7 +172,7 @@ class PeaceKeeperDom(_PeaceKeeperBenchmark):
                ]
 
 
-class PeaceKeeperTextParsing(_PeaceKeeperBenchmark):
+class PeaceKeeperTextParsing(PeaceKeeperBenchmark):
   """PeaceKeeper Text Parsing benchmark suite.
 
   These tests measure your browser's performance in typical text manipulations
@@ -201,7 +201,7 @@ class PeaceKeeperTextParsing(_PeaceKeeperBenchmark):
                ]
 
 
-class PeaceKeeperHTML5Canvas(_PeaceKeeperBenchmark):
+class PeaceKeeperHTML5Canvas(PeaceKeeperBenchmark):
   """PeaceKeeper HTML5 Canvas benchmark suite.
 
   These tests use HTML5 Canvas, which is a web technology for drawing and
@@ -218,7 +218,7 @@ class PeaceKeeperHTML5Canvas(_PeaceKeeperBenchmark):
                ]
 
 
-class PeaceKeeperHTML5Capabilities(_PeaceKeeperBenchmark):
+class PeaceKeeperHTML5Capabilities(PeaceKeeperBenchmark):
   """PeaceKeeper HTML5 Capabilities benchmark suite.
 
   These tests checks browser HTML5 capabilities support for WebGL, Video
diff --git a/tools/perf/generate_profile b/tools/perf/generate_profile
index 492ba7e4..312c8f3 100755
--- a/tools/perf/generate_profile
+++ b/tools/perf/generate_profile
@@ -5,8 +5,7 @@
 import os
 import sys
 
-sys.path.append(os.path.join(
-  os.path.dirname(os.path.realpath(__file__)), os.pardir, 'telemetry'))
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'telemetry'))
 
 from telemetry.page import profile_generator
 
diff --git a/tools/perf/record_android_profile.py b/tools/perf/record_android_profile.py
index 8b1fe6f..8c8c5ab 100755
--- a/tools/perf/record_android_profile.py
+++ b/tools/perf/record_android_profile.py
@@ -7,8 +7,7 @@ import os
 import sys
 import tempfile
 
-sys.path.append(os.path.join(
-  os.path.dirname(os.path.realpath(__file__)), os.pardir, 'telemetry'))
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'telemetry'))
 
 from telemetry.core import browser_finder
 from telemetry.core import browser_options
diff --git a/tools/perf/record_wpr b/tools/perf/record_wpr
index 5a581be..e12055f 100755
--- a/tools/perf/record_wpr
+++ b/tools/perf/record_wpr
@@ -7,7 +7,7 @@ import sys
 
 # Add telemetry to sys.path so that it can be imported below
 # Relative path from this file is ../telemetry
-_perf_dir = os.path.dirname(os.path.realpath(__file__))
+_perf_dir = os.path.dirname(__file__)
 _telemetry_path = os.path.join(_perf_dir, os.pardir, 'telemetry')
 sys.path.append(_telemetry_path)
 
diff --git a/tools/perf/run_benchmark b/tools/perf/run_benchmark
index 1676651..e6dd566 100755
--- a/tools/perf/run_benchmark
+++ b/tools/perf/run_benchmark
@@ -6,8 +6,7 @@
 import os
 import sys
 
-sys.path.append(os.path.join(
-  os.path.dirname(os.path.realpath(__file__)), os.pardir, 'telemetry'))
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'telemetry'))
 
 from telemetry import test_runner
 
diff --git a/tools/perf/run_measurement b/tools/perf/run_measurement
index d2f9e92c..b91b594 100755
--- a/tools/perf/run_measurement
+++ b/tools/perf/run_measurement
@@ -12,7 +12,7 @@ import urllib2
 BASE_URL = 'http://src.chromium.org/chrome/trunk/'
 DEPS_FILE = 'bootstrap_deps'
 
-SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
+SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
 # Directory containing src/ in a Chromium checkout.
 CHECKOUT_BASE_PATH = os.path.join(SCRIPT_PATH, os.pardir, os.pardir, os.pardir)
 # Directory in which to save bootstrap files.
diff --git a/tools/perf/run_tests b/tools/perf/run_tests
index c1c203d..77a5366 100755
--- a/tools/perf/run_tests
+++ b/tools/perf/run_tests
@@ -11,14 +11,13 @@ This script DOES NOT run benchmarks. run_benchmarks and run_measurement do that.
 import os
 import sys
 
-sys.path.append(os.path.join(
-  os.path.dirname(os.path.realpath(__file__)), os.pardir, 'telemetry'))
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'telemetry'))
 from telemetry.unittest import gtest_testrunner
 from telemetry.unittest import run_tests
 
 
 if __name__ == '__main__':
-  top_level_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+  top_level_dir = os.path.abspath(os.path.dirname(__file__))
   runner = gtest_testrunner.GTestTestRunner(print_result_after_run=False)
   ret = run_tests.Main(sys.argv[1:], top_level_dir, top_level_dir, runner)
 
diff --git a/tools/telemetry/run_tests b/tools/telemetry/run_tests
index 3e7e93a..e5c13e9 100755
--- a/tools/telemetry/run_tests
+++ b/tools/telemetry/run_tests
@@ -11,7 +11,7 @@ from telemetry.unittest import run_tests
 
 
 if __name__ == '__main__':
-  top_level_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
+  top_level_dir = os.path.abspath(os.path.dirname(__file__))
   runner = gtest_testrunner.GTestTestRunner(print_result_after_run=False)
   ret = run_tests.Main(sys.argv[1:], top_level_dir, top_level_dir, runner)
 
diff --git a/tools/telemetry/telemetry/core/util.py b/tools/telemetry/telemetry/core/util.py
index 018d8b1..55a2863 100644
--- a/tools/telemetry/telemetry/core/util.py
+++ b/tools/telemetry/telemetry/core/util.py
@@ -16,14 +16,14 @@ class TimeoutException(Exception):
 def GetBaseDir():
   main_module = sys.modules['__main__']
   if hasattr(main_module, '__file__'):
-    return os.path.dirname(os.path.realpath(main_module.__file__))
+    return os.path.dirname(os.path.abspath(main_module.__file__))
   else:
     return os.getcwd()
 
 
 def GetTelemetryDir():
   return os.path.normpath(os.path.join(
-      os.path.realpath(__file__), os.pardir, os.pardir, os.pardir))
+      __file__, os.pardir, os.pardir, os.pardir))
 
 
 def GetUnittestDataDir():
diff --git a/tools/telemetry/telemetry/page/cloud_storage.py b/tools/telemetry/telemetry/page/cloud_storage.py
index df103c1..75cd11e 100644
--- a/tools/telemetry/telemetry/page/cloud_storage.py
+++ b/tools/telemetry/telemetry/page/cloud_storage.py
@@ -27,7 +27,7 @@ _DOWNLOAD_PATH = os.path.join(util.GetTelemetryDir(), 'third_party', 'gsutil')
 class CloudStorageError(Exception):
   @staticmethod
   def _GetConfigInstructions(gsutil_path):
-    if SupportsProdaccess(gsutil_path):
+    if _SupportsProdaccess(gsutil_path):
       return 'Run prodaccess to authenticate.'
     else:
       return ('To configure your credentials:\n'
@@ -73,7 +73,7 @@ def _DownloadGsutil():
   return os.path.join(_DOWNLOAD_PATH, 'gsutil')
 
 
-def FindGsutil():
+def _FindGsutil():
   """Return the gsutil executable path. If we can't find it, download it."""
   # Look for a depot_tools installation.
   gsutil_path = _FindExecutableInPath(
@@ -90,7 +90,7 @@ def FindGsutil():
   return _DownloadGsutil()
 
 
-def SupportsProdaccess(gsutil_path):
+def _SupportsProdaccess(gsutil_path):
   def GsutilSupportsProdaccess():
     with open(gsutil_path, 'r') as gsutil:
       return 'prodaccess' in gsutil.read()
@@ -99,7 +99,8 @@ def SupportsProdaccess(gsutil_path):
 
 
 def _RunCommand(args):
-  gsutil_path = FindGsutil()
+  gsutil_path = _FindGsutil()
+
   gsutil = subprocess.Popen([sys.executable, gsutil_path] + args,
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   stdout, stderr = gsutil.communicate()
diff --git a/tools/telemetry_tools/find_dependencies b/tools/telemetry_tools/find_dependencies
deleted file mode 100755
index b970dd5..0000000
--- a/tools/telemetry_tools/find_dependencies
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import collections
-import fnmatch
-import imp
-import logging
-import modulefinder
-import optparse
-import os
-import sys
-import zipfile
-
-sys.path.append(os.path.join(
-    os.path.dirname(os.path.realpath(__file__)), os.pardir, 'telemetry'))
-from telemetry import test
-from telemetry.core import discover
-from telemetry.core import util
-from telemetry.page import cloud_storage
-
-import path_set
-import telemetry_bootstrap
-
-
-DEPS_FILE = 'bootstrap_deps'
-
-
-def _InDirectory(subdirectory, directory):
-  subdirectory = os.path.realpath(subdirectory)
-  directory = os.path.realpath(directory)
-  common_prefix = os.path.commonprefix([subdirectory, directory])
-  return common_prefix == directory
-
-
-def FindBootstrapDependencies(base_dir):
-  deps_file = os.path.join(base_dir, DEPS_FILE)
-  if not os.path.exists(deps_file):
-    return []
-  deps_paths = telemetry_bootstrap.ListAllDepsPaths(deps_file)
-  return set(
-      os.path.realpath(os.path.join(util.GetChromiumSrcDir(), os.pardir, path))
-      for path in deps_paths)
-
-
-def FindPythonDependencies(module_path):
-  logging.info('Finding Python dependencies of %s' % module_path)
-
-  # Load the module to inherit its sys.path modifications.
-  imp.load_source(
-      os.path.splitext(os.path.basename(module_path))[0], module_path)
-
-  # Analyze the module for its imports.
-  finder = modulefinder.ModuleFinder()
-  finder.run_script(module_path)
-
-  # Filter for only imports in Chromium.
-  for module in finder.modules.itervalues():
-    # If it's an __init__.py, module.__path__ gives the package's folder.
-    module_path = module.__path__[0] if module.__path__ else module.__file__
-    if not module_path:
-      continue
-
-    module_path = os.path.realpath(module_path)
-    if not _InDirectory(module_path, util.GetChromiumSrcDir()):
-      continue
-
-    yield module_path
-
-
-def FindPageSetDependencies(base_dir):
-  logging.info('Finding page sets in %s' % base_dir)
-
-  # Add base_dir to path so our imports relative to base_dir will work.
-  sys.path.append(base_dir)
-  tests = discover.DiscoverClasses(base_dir, base_dir, test.Test,
-                                   index_by_class_name=True)
-
-  for test_class in tests.itervalues():
-    test_obj = test_class()
-
-    # Ensure the test's default options are set if needed.
-    parser = optparse.OptionParser()
-    test_obj.AddTestCommandLineOptions(parser)
-    options = optparse.Values()
-    for k, v in parser.get_default_values().__dict__.iteritems():
-      options.ensure_value(k, v)
-
-    # Page set paths are relative to their runner script, not relative to us.
-    util.GetBaseDir = lambda: base_dir
-    # TODO: Loading the page set will automatically download its Cloud Storage
-    # deps. This is really expensive, and we don't want to do this by default.
-    try:
-      page_set = test_obj.CreatePageSet(options)
-    except NotImplementedError:
-      # The test is an abstract class with no page set, so skip it.
-      pass
-
-    # Add all of its serving_dirs as dependencies.
-    for serving_dir in page_set.serving_dirs:
-      yield serving_dir
-    for page in page_set:
-      if page.is_file:
-        yield page.serving_dir
-
-
-def FindExcludedFiles(files, options):
-  def MatchesConditions(path, conditions):
-    for condition in conditions:
-      if condition(path):
-        return True
-    return False
-
-  # Define some filters for files.
-  def IsHidden(path):
-    for pathname_component in path.split(os.sep):
-      if pathname_component.startswith('.'):
-        return True
-    return False
-  def IsPyc(path):
-    return os.path.splitext(path)[1] == '.pyc'
-  def IsInCloudStorage(path):
-    return os.path.exists(path + '.sha1')
-  def MatchesExcludeOptions(path):
-    for pattern in options.exclude:
-      if (fnmatch.fnmatch(path, pattern) or
-          fnmatch.fnmatch(os.path.basename(path), pattern)):
-        return True
-    return False
-
-  # Collect filters we're going to use to exclude files.
-  exclude_conditions = [
-      IsHidden,
-      IsPyc,
-      IsInCloudStorage,
-      MatchesExcludeOptions,
-  ]
-
-  # Check all the files against the filters.
-  for path in files:
-    if MatchesConditions(path, exclude_conditions):
-      yield path
-
-
-def FindDependencies(paths, options):
-  # Verify arguments.
-  for path in paths:
-    if not os.path.exists(path):
-      raise ValueError('Path does not exist: %s' % path)
-
-  dependencies = path_set.PathSet()
-
-  # Including this file will include Telemetry and its dependencies.
-  # If the user doesn't pass any arguments, we just have Telemetry.
-  dependencies |= FindPythonDependencies(os.path.realpath(__file__))
-
-  # Add dependencies.
-  for path in paths:
-    base_dir = os.path.dirname(os.path.realpath(path))
-
-    dependencies.add(base_dir)
-    dependencies |= FindBootstrapDependencies(base_dir)
-    dependencies |= FindPythonDependencies(path)
-    if options.include_page_set_data:
-      dependencies |= FindPageSetDependencies(base_dir)
-
-  # Remove excluded files.
-  dependencies -= FindExcludedFiles(set(dependencies), options)
-
-  return dependencies
-
-
-def ZipDependencies(paths, dependencies, options):
-  base_dir = os.path.dirname(os.path.realpath(util.GetChromiumSrcDir()))
-
-  with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
-    # Add dependencies to archive.
-    for path in dependencies:
-      path_in_archive = os.path.join(
-          'telemetry', os.path.relpath(path, base_dir))
-      zip_file.write(path, path_in_archive)
-
-    # Add symlinks to executable paths, for ease of use.
-    for path in paths:
-      link_info = zipfile.ZipInfo(
-          os.path.join('telemetry', os.path.basename(path)))
-      link_info.create_system = 3  # Unix attributes.
-      # 010 is regular file, 0111 is the permission bits rwxrwxrwx.
-      link_info.external_attr = 0100777 << 16  # Octal.
-
-      relative_path = os.path.relpath(path, base_dir)
-      link_script = (
-          '#!/usr/bin/env python\n\n'
-          'import os\n'
-          'import sys\n\n\n'
-          'script = os.path.join(os.path.dirname(__file__), \'%s\')\n'
-          'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])'
-        % relative_path)
-
-      zip_file.writestr(link_info, link_script)
-
-    # Add gsutil to the archive, if it's available. The gsutil in
-    # depot_tools is modified to allow authentication using prodaccess.
-    # TODO: If there's a gsutil in telemetry/third_party/, bootstrap_deps
-    # will include it. Then there will be two copies of gsutil at the same
-    # location in the archive. This can be confusing for users.
-    gsutil_path = cloud_storage.FindGsutil()
-    if cloud_storage.SupportsProdaccess(gsutil_path):
-      gsutil_dependencies = path_set.PathSet()
-      gsutil_dependencies.add(os.path.dirname(gsutil_path))
-      gsutil_dependencies -= FindExcludedFiles(
-          set(gsutil_dependencies), options)
-
-      gsutil_base_dir = os.path.join(os.path.dirname(gsutil_path), os.pardir)
-      for path in gsutil_dependencies:
-        path_in_archive = os.path.join(
-            'telemetry', os.path.relpath(util.GetTelemetryDir(), base_dir),
-            'third_party', os.path.relpath(path, gsutil_base_dir))
-        zip_file.write(path, path_in_archive)
-
-
-def ParseCommandLine():
-  parser = optparse.OptionParser()
-  parser.add_option(
-      '-v', '--verbose', action='count', dest='verbosity',
-      help='Increase verbosity level (repeat as needed).')
-
-  parser.add_option(
-      '-p', '--include-page-set-data', action='store_true', default=False,
-      help='Scan tests for page set data and include them.')
-
-  parser.add_option(
-      '-e', '--exclude', action='append', default=[],
-      help='Exclude paths matching EXCLUDE. Can be used multiple times.')
-
-  parser.add_option(
-      '-z', '--zip',
-      help='Store files in a zip archive at ZIP.')
-
-  options, args = parser.parse_args()
-
-  if options.verbosity >= 2:
-    logging.getLogger().setLevel(logging.DEBUG)
-  elif options.verbosity:
-    logging.getLogger().setLevel(logging.INFO)
-  else:
-    logging.getLogger().setLevel(logging.WARNING)
-
-  return options, args
-
-
-def main():
-  options, paths = ParseCommandLine()
-
-  dependencies = FindDependencies(paths, options)
-
-  if options.zip:
-    ZipDependencies(paths, dependencies, options)
-    print 'Zip archive written to %s.' % options.zip
-  else:
-    print '\n'.join(sorted(dependencies))
-
-
-if __name__ == '__main__':
-  main()
diff --git a/tools/telemetry_tools/path_set.py b/tools/telemetry_tools/path_set.py
deleted file mode 100644
index 0b46783..0000000
--- a/tools/telemetry_tools/path_set.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import collections
-import os
-
-
-class PathSet(collections.MutableSet):
-  """A set of paths.
-
-  All mutation methods can take both directories or individual files, but the
-  iterator yields the individual files. All paths are automatically normalized.
-  """
-  def __init__(self, iterable=None):
-    self._paths = set()
-    if iterable:
-      self |= iterable
-
-  def __contains__(self, path):
-    return os.path.realpath(path) in self._paths
-
-  def __iter__(self):
-    return iter(self._paths)
-
-  def __len__(self):
-    return len(self._paths)
-
-  def add(self, path):
-    path = os.path.realpath(path)
-    if os.path.isfile(path):
-      self._paths.add(path)
-    for root, _, files in os.walk(path):
-      for basename in files:
-        file_path = os.path.join(root, basename)
-        if os.path.isfile(file_path):
-          self._paths.add(file_path)
-
-  def discard(self, path):
-    path = os.path.realpath(path)
-    self._paths.discard(path)
-    for root, _, files in os.walk(path):
-      for basename in files:
-        self._paths.discard(os.path.join(root, basename))
diff --git a/tools/telemetry_tools/path_set_unittest.py b/tools/telemetry_tools/path_set_unittest.py
deleted file mode 100755
index 7451bee..0000000
--- a/tools/telemetry_tools/path_set_unittest.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import os
-import unittest
-
-import path_set
-
-
-class PathSetTest(unittest.TestCase):
-  def testCreate(self):
-    ps = path_set.PathSet()
-    self.assertEqual(len(ps), 0)
-    self.assertFalse(__file__ in ps)
-    for path in ps:
-      self.fail('New set is not empty.')
-
-    ps = path_set.PathSet([__file__])
-    self.assertEqual(len(ps), 1)
-    self.assertTrue(__file__ in ps)
-    self.assertEqual(ps.pop(), os.path.realpath(__file__))
-
-  def testAdd(self):
-    ps = path_set.PathSet()
-    ps.add(__file__)
-    self.assertEqual(len(ps), 1)
-    self.assertTrue(__file__ in ps)
-    self.assertEqual(ps.pop(), os.path.realpath(__file__))
-
-  def testDiscard(self):
-    ps = path_set.PathSet([__file__])
-    ps.discard(__file__)
-    self.assertEqual(len(ps), 0)
-    self.assertFalse(__file__ in ps)
-
-
-if __name__ == '__main__':
-  unittest.main()