Multiple improvements to trace_inputs.py.

Fix bool, None and string variables handling

Do not track svn or git directories and files with a space in their name.

Split trace_inputs_smoke_test.py child process into its own script.

R=rogerta@chromium.org
BUG=98834
TEST=


Review URL: http://codereview.chromium.org/10031008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@131981 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 109 insertions, 61 deletions

diff --git a/tools/isolate/data/trace_inputs/child1.py b/tools/isolate/data/trace_inputs/child1.py
new file mode 100644
index 0000000..9155867
--- /dev/null
+++ b/tools/isolate/data/trace_inputs/child1.py
@@ -0,0 +1,48 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import subprocess
+import sys
+
+
+def child():
+  """When the gyp argument is not specified, the command is started from
+  --root-dir directory.
+  """
+  print 'child from %s' % os.getcwd()
+  # Force file opening with a non-normalized path.
+  open(os.path.join('data', '..', 'trace_inputs.py'), 'rb').close()
+  open(os.path.join('data', '..', 'trace_inputs_smoke_test.py'), 'rb').close()
+  # Do not wait for the child to exit.
+  # Use relative directory.
+  subprocess.Popen(
+      ['python', 'child2.py'], cwd=os.path.join('data', 'trace_inputs'))
+  return 0
+
+
+def child_gyp():
+  """When the gyp argument is specified, the command is started from --cwd
+  directory.
+  """
+  print 'child_gyp from %s' % os.getcwd()
+  # Force file opening.
+  open(os.path.join('..', 'trace_inputs.py'), 'rb').close()
+  open(os.path.join('..', 'trace_inputs_smoke_test.py'), 'rb').close()
+  # Do not wait for the child to exit.
+  # Use relative directory.
+  subprocess.Popen(['python', 'child2.py'], cwd='trace_inputs')
+  return 0
+
+
+def main():
+  if sys.argv[1] == '--child':
+    return child()
+  if sys.argv[1] == '--child-gyp':
+    return child_gyp()
+  return 1
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/tools/isolate/trace_inputs.py b/tools/isolate/trace_inputs.py
index 0ee69b4..f181946 100755
--- a/tools/isolate/trace_inputs.py
+++ b/tools/isolate/trace_inputs.py
@@ -1098,8 +1098,20 @@ def extract_directories(files, root):
 
 
 def pretty_print(variables, stdout):
-  """Outputs a gyp compatible list from the decoded variables."""
-  # Similar to pprint.print() but with NIH syndrome.
+  """Outputs a gyp compatible list from the decoded variables.
+
+  Similar to pprint.print() but with NIH syndrome.
+  """
+  # Order the dictionary keys by these keys in priority.
+  ORDER = (
+      'variables', 'condition', 'command', 'relative_cwd', 'read_only',
+      KEY_TRACKED, KEY_UNTRACKED)
+
+  def sorting_key(x):
+    """Gives priority to 'most important' keys before the others."""
+    if x in ORDER:
+      return str(ORDER.index(x))
+    return x
 
   def loop_list(indent, items):
     for item in items:
@@ -1114,7 +1126,8 @@ def pretty_print(variables, stdout):
         stdout.write('%s[' % indent)
         for index, i in enumerate(item):
           if isinstance(i, basestring):
-            stdout.write('\'%s\', ' % i)
+            stdout.write(
+                '\'%s\', ' % i.replace('\\', '\\\\').replace('\'', '\\\''))
           elif isinstance(i, dict):
             stdout.write('{\n')
             loop_dict(indent + '  ', i)
@@ -1130,8 +1143,7 @@ def pretty_print(variables, stdout):
         assert False
 
   def loop_dict(indent, items):
-    # Use reversed sorting since it happens we always want it that way.
-    for key in sorted(items, reverse=True):
+    for key in sorted(items, key=sorting_key):
       item = items[key]
       stdout.write("%s'%s': " % (indent, key))
       if isinstance(item, dict):
@@ -1142,8 +1154,13 @@ def pretty_print(variables, stdout):
         stdout.write('[\n')
         loop_list(indent + '  ', item)
         stdout.write(indent + '],\n')
+      elif isinstance(item, basestring):
+        stdout.write(
+            '\'%s\',\n' % item.replace('\\', '\\\\').replace('\'', '\\\''))
+      elif item in (True, False, None):
+        stdout.write('%s\n' % item)
       else:
-        assert False
+        assert False, item
 
   stdout.write('{\n')
   loop_dict('  ', variables)
@@ -1222,9 +1239,15 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace):
     if returncode and not force_trace:
       return returncode
 
+  git_path = os.path.sep + '.git' + os.path.sep
+  svn_path = os.path.sep + '.svn' + os.path.sep
   def blacklist(f):
     """Strips ignored paths."""
-    return f.startswith(api.IGNORED) or f.endswith('.pyc')
+    return (
+        f.startswith(api.IGNORED) or
+        f.endswith('.pyc') or
+        git_path in f or
+        svn_path in f)
 
   print_if('Loading traces... %s' % logfile)
   files, non_existent = api.parse_log(logfile, blacklist)
@@ -1279,13 +1302,13 @@ def trace_inputs(logfile, cmd, root_dir, cwd_dir, product_dir, force_trace):
         return '<(DEPTH)/%s' % f
 
     corrected = [fix(f) for f in simplified]
-    files = [f for f in corrected if not f.endswith('/')]
-    dirs = [f for f in corrected if f.endswith('/')]
+    tracked = [f for f in corrected if not f.endswith('/') and ' ' not in f]
+    untracked = [f for f in corrected if f.endswith('/') or ' ' in f]
     variables = {}
-    if files:
-      variables[KEY_TRACKED] = files
-    if dirs:
-      variables[KEY_UNTRACKED] = dirs
+    if tracked:
+      variables[KEY_TRACKED] = tracked
+    if untracked:
+      variables[KEY_UNTRACKED] = untracked
     value = {
       'conditions': [
         ['OS=="%s"' % flavor, {
diff --git a/tools/isolate/trace_inputs_smoke_test.py b/tools/isolate/trace_inputs_smoke_test.py
index dec35e6..17b8c47 100755
--- a/tools/isolate/trace_inputs_smoke_test.py
+++ b/tools/isolate/trace_inputs_smoke_test.py
@@ -55,11 +55,11 @@ class TraceInputs(unittest.TestCase):
     if is_gyp:
       # When the gyp argument is specified, the command is started from --cwd
       # directory. In this case, 'data'.
-      cmd.extend([os.path.join('..', FILENAME), '--child-gyp'])
+      cmd.extend([os.path.join('trace_inputs', 'child1.py'), '--child-gyp'])
     else:
       # When the gyp argument is not specified, the command is started from
       # --root-dir directory.
-      cmd.extend([FILENAME, '--child'])
+      cmd.extend([os.path.join('data', 'trace_inputs', 'child1.py'), '--child'])
 
     # The current directory doesn't matter, the traced process will be called
     # from the correct cwd.
@@ -76,7 +76,7 @@ class TraceInputs(unittest.TestCase):
 
   def test_trace(self):
     expected_end = [
-      'Interesting: 4 reduced to 3',
+      'Interesting: 5 reduced to 3',
       '  data/trace_inputs/'.replace('/', os.path.sep),
       '  trace_inputs.py',
       '  %s' % FILENAME,
@@ -120,48 +120,7 @@ class TraceInputs(unittest.TestCase):
     self.assertEquals(expected_buffer.getvalue(), actual)
 
 
-def child():
-  """When the gyp argument is not specified, the command is started from
-  --root-dir directory.
-  """
-  print 'child from %s' % os.getcwd()
-  # Force file opening with a non-normalized path.
-  open(os.path.join('data', '..', 'trace_inputs.py'), 'rb').close()
-  # Do not wait for the child to exit.
-  # Use relative directory.
-  subprocess.Popen(
-      ['python', 'child2.py'], cwd=os.path.join('data', 'trace_inputs'))
-  return 0
-
-
-def child_gyp():
-  """When the gyp argument is specified, the command is started from --cwd
-  directory.
-  """
-  print 'child_gyp from %s' % os.getcwd()
-  # Force file opening.
-  open(os.path.join('..', 'trace_inputs.py'), 'rb').close()
-  # Do not wait for the child to exit.
-  # Use relative directory.
-  subprocess.Popen(['python', 'child2.py'], cwd='trace_inputs')
-  return 0
-
-
-def main():
-  global VERBOSE
+if __name__ == '__main__':
   VERBOSE = '-v' in sys.argv
-  level = logging.DEBUG if VERBOSE else logging.ERROR
-  logging.basicConfig(level=level)
-  if len(sys.argv) == 1:
-    unittest.main()
-
-  if sys.argv[1] == '--child':
-    return child()
-  if sys.argv[1] == '--child-gyp':
-    return child_gyp()
-
+  logging.basicConfig(level=logging.DEBUG if VERBOSE else logging.ERROR)
   unittest.main()
-
-
-if __name__ == '__main__':
-  sys.exit(main())
diff --git a/tools/isolate/trace_inputs_test.py b/tools/isolate/trace_inputs_test.py
index 3830b79..322ef8c 100755
--- a/tools/isolate/trace_inputs_test.py
+++ b/tools/isolate/trace_inputs_test.py
@@ -29,10 +29,17 @@ class TraceInputs(unittest.TestCase):
       'conditions': [
         ['OS=\"foo\"', {
           'variables': {
-            'isolate': [
+            trace_inputs.KEY_UNTRACKED: [
               'dir1',
               'dir2',
             ],
+            trace_inputs.KEY_TRACKED: [
+              'file4',
+              'file3',
+            ],
+            'command': ['python', '-c', 'print "H\\i\'"'],
+            'read_only': True,
+            'relative_cwd': 'isol\'at\\e',
           },
         }],
         ['OS=\"bar\"', {
@@ -53,7 +60,18 @@ class TraceInputs(unittest.TestCase):
         "  'conditions': [\n"
         "    ['OS=\"foo\"', {\n"
         "      'variables': {\n"
-        "        'isolate': [\n"
+        "        'command': [\n"
+        "          'python',\n"
+        "          '-c',\n"
+        "          'print \"H\\i\'\"',\n"
+        "        ],\n"
+        "        'relative_cwd': 'isol\\'at\\\\e',\n"
+        "        'read_only': True\n"
+        "        'isolate_dependency_tracked': [\n"
+        "          'file4',\n"
+        "          'file3',\n"
+        "        ],\n"
+        "        'isolate_dependency_untracked': [\n"
         "          'dir1',\n"
         "          'dir2',\n"
         "        ],\n"