Major refactoring of Croc.

Add support for scanning missing source files for executable lines.
Add support for HTML output.
Now reports percent coverage.

BUG=none
TEST=by hand on experimental buildbot
Review URL: http://codereview.chromium.org/113980

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@17141 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 1215 insertions, 220 deletions

diff --git a/tools/code_coverage/croc.py b/tools/code_coverage/croc.py
index b2bd33b..9d2f3ac 100644
--- a/tools/code_coverage/croc.py
+++ b/tools/code_coverage/croc.py
@@ -31,16 +31,19 @@
 
 """Crocodile - compute coverage numbers for Chrome coverage dashboard."""
 
+import optparse
 import os
 import re
 import sys
-from optparse import OptionParser
+import croc_html
+import croc_scan
 
 
-class CoverageError(Exception):
+class CrocError(Exception):
   """Coverage error."""
 
-class CoverageStatError(CoverageError):
+
+class CrocStatError(CrocError):
   """Error evaluating coverage stat."""
 
 #------------------------------------------------------------------------------
@@ -57,7 +60,7 @@ class CoverageStats(dict):
     """
     for k, v in coverage_stats.iteritems():
       if k in self:
-        self[k] = self[k] + v
+        self[k] += v
       else:
         self[k] = v
 
@@ -67,17 +70,19 @@ class CoverageStats(dict):
 class CoveredFile(object):
   """Information about a single covered file."""
 
-  def __init__(self, filename, group, language):
+  def __init__(self, filename, **kwargs):
     """Constructor.
 
     Args:
       filename: Full path to file, '/'-delimited.
-      group: Group file belongs to.
-      language: Language for file.
+      kwargs: Keyword args are attributes for file.
     """
     self.filename = filename
-    self.group = group
-    self.language = language
+    self.attrs = dict(kwargs)
+
+    # Move these to attrs?
+    self.local_path = None      # Local path to file
+    self.in_lcov = False        # Is file instrumented?
 
     # No coverage data for file yet
     self.lines = {}     # line_no -> None=executable, 0=instrumented, 1=covered
@@ -102,10 +107,9 @@ class CoveredFile(object):
     # Add conditional stats
     if cov:
       self.stats['files_covered'] = 1
-    if instr:
+    if instr or self.in_lcov:
       self.stats['files_instrumented'] = 1
 
-
 #------------------------------------------------------------------------------
 
 
@@ -128,7 +132,7 @@ class CoveredDir(object):
     self.subdirs = {}
 
     # Dict of CoverageStats objects summarizing all children, indexed by group
-    self.stats_by_group = {'all':CoverageStats()}
+    self.stats_by_group = {'all': CoverageStats()}
     # TODO: by language
 
   def GetTree(self, indent=''):
@@ -154,7 +158,8 @@ class CoveredDir(object):
           s.get('lines_executable', 0)))
 
     outline = '%s%-30s   %s' % (indent,
-                                self.dirpath + '/', '   '.join(groupstats))
+                                os.path.split(self.dirpath)[1] + '/',
+                                '   '.join(groupstats))
     dest.append(outline.rstrip())
 
     for d in sorted(self.subdirs):
@@ -172,17 +177,13 @@ class Coverage(object):
     """Constructor."""
     self.files = {}             # Map filename --> CoverageFile
     self.root_dirs = []         # (root, altname)
-    self.rules = []             # (regexp, include, group, language)
+    self.rules = []             # (regexp, dict of RHS attrs)
     self.tree = CoveredDir('')
     self.print_stats = []       # Dicts of args to PrintStat()
 
-    self.add_files_walk = os.walk       # Walk function for AddFiles()
-
-    # Must specify subdir rule, or AddFiles() won't find any files because it
-    # will prune out all the subdirs.  Since subdirs never match any code,
-    # they won't be reported in other stats, so this is ok.
-    self.AddRule('.*/$', language='subdir')
-
+    # Functions which need to be replaced for unit testing
+    self.add_files_walk = os.walk         # Walk function for AddFiles()
+    self.scan_file = croc_scan.ScanFile   # Source scanner for AddFiles()
 
   def CleanupFilename(self, filename):
     """Cleans up a filename.
@@ -208,8 +209,8 @@ class Coverage(object):
 
     # Replace alternate roots
     for root, alt_name in self.root_dirs:
-        filename = re.sub('^' + re.escape(root) + '(?=(/|$))',
-                          alt_name, filename)
+      filename = re.sub('^' + re.escape(root) + '(?=(/|$))',
+                        alt_name, filename)
     return filename
 
   def ClassifyFile(self, filename):
@@ -219,29 +220,17 @@ class Coverage(object):
       filename: Input filename.
 
     Returns:
-      (None, None) if the file is not included or has no group or has no
-          language.  Otherwise, a 2-tuple containing:
-      The group for the file (for example, 'source' or 'test').
-      The language of the file.
+      A dict of attributes for the file, accumulated from the right hand sides
+          of rules which fired.
     """
-    include = False
-    group = None
-    language = None
+    attrs = {}
 
     # Process all rules
-    for regexp, rule_include, rule_group, rule_language in self.rules:
+    for regexp, rhs_dict in self.rules:
       if regexp.match(filename):
-        # include/exclude source
-        if rule_include is not None:
-          include = rule_include
-        if rule_group is not None:
-          group = rule_group
-        if rule_language is not None:
-          language = rule_language
-
-    # TODO: Should have a debug mode which prints files which aren't excluded
-    # and why (explicitly excluded, no type, no language, etc.)
+        attrs.update(rhs_dict)
 
+    return attrs
     # TODO: Files can belong to multiple groups?
     #   (test/source)
     #   (mac/pc/win)
@@ -249,36 +238,43 @@ class Coverage(object):
     #   (small/med/large)
     # How to handle that?
 
-    # Return classification if the file is included and has a group and
-    # language
-    if include and group and language:
-      return group, language
-    else:
-      return None, None
-
-  def AddRoot(self, root_path, alt_name='#'):
+  def AddRoot(self, root_path, alt_name='_'):
     """Adds a root directory.
 
     Args:
       root_path: Root directory to add.
-      alt_name: If specified, name of root dir
+      alt_name: If specified, name of root dir.  Otherwise, defaults to '_'.
+
+    Raises:
+      ValueError: alt_name was blank.
     """
+    # Alt name must not be blank.  If it were, there wouldn't be a way to
+    # reverse-resolve from a root-replaced path back to the local path, since
+    # '' would always match the beginning of the candidate filename, resulting
+    # in an infinite loop.
+    if not alt_name:
+      raise ValueError('AddRoot alt_name must not be blank.')
+
     # Clean up root path based on existing rules
     self.root_dirs.append([self.CleanupFilename(root_path), alt_name])
 
-  def AddRule(self, path_regexp, include=None, group=None, language=None):
+  def AddRule(self, path_regexp, **kwargs):
     """Adds a rule.
 
     Args:
       path_regexp: Regular expression to match for filenames.  These are
           matched after root directory replacement.
+      kwargs: Keyword arguments are attributes to set if the rule applies.
+
+    Keyword arguments currently supported:
       include: If True, includes matches; if False, excludes matches.  Ignored
           if None.
       group: If not None, sets group to apply to matches.
       language: If not None, sets file language to apply to matches.
     """
+
     # Compile regexp ahead of time
-    self.rules.append([re.compile(path_regexp), include, group, language])
+    self.rules.append([re.compile(path_regexp), dict(kwargs)])
 
   def GetCoveredFile(self, filename, add=False):
     """Gets the CoveredFile object for the filename.
@@ -303,18 +299,29 @@ class Coverage(object):
     if not add:
       return None
 
-    # Check rules to see if file can be added
-    group, language = self.ClassifyFile(filename)
-    if not group:
+    # Check rules to see if file can be added.  Files must be included and
+    # have a group and language.
+    attrs = self.ClassifyFile(filename)
+    if not (attrs.get('include')
+            and attrs.get('group')
+            and attrs.get('language')):
       return None
 
     # Add the file
-    f = CoveredFile(filename, group, language)
+    f = CoveredFile(filename, **attrs)
     self.files[filename] = f
 
     # Return the newly covered file
     return f
 
+  def RemoveCoveredFile(self, cov_file):
+    """Removes the file from the covered file list.
+
+    Args:
+      cov_file: A file object returned by GetCoveredFile().
+    """
+    self.files.pop(cov_file.filename)
+
   def ParseLcovData(self, lcov_data):
     """Adds coverage from LCOV-formatted data.
 
@@ -331,6 +338,7 @@ class Coverage(object):
         cov_file = self.GetCoveredFile(line[3:], add=True)
         if cov_file:
           cov_lines = cov_file.lines
+          cov_file.in_lcov = True       # File was instrumented
       elif not cov_file:
         # Inside data for a file we don't care about - so skip it
         pass
@@ -372,13 +380,13 @@ class Coverage(object):
       group: File group to match; if 'all', matches all groups.
       default: Value to return if there was an error evaluating the stat.  For
           example, if the stat does not exist.  If None, raises
-          CoverageStatError.
+          CrocStatError.
 
     Returns:
       The evaluated stat, or None if error.
 
     Raises:
-      CoverageStatError: Error evaluating stat.
+      CrocStatError: Error evaluating stat.
     """
     # TODO: specify a subdir to get the stat from, then walk the tree to
     # print the stats from just that subdir
@@ -386,16 +394,16 @@ class Coverage(object):
     # Make sure the group exists
     if group not in self.tree.stats_by_group:
       if default is None:
-        raise CoverageStatError('Group %r not found.' % group)
+        raise CrocStatError('Group %r not found.' % group)
       else:
         return default
 
     stats = self.tree.stats_by_group[group]
     try:
-      return eval(stat, {'__builtins__':{'S':self.GetStat}}, stats)
+      return eval(stat, {'__builtins__': {'S': self.GetStat}}, stats)
     except Exception, e:
       if default is None:
-        raise CoverageStatError('Error evaluating stat %r: %s' % (stat, e))
+        raise CrocStatError('Error evaluating stat %r: %s' % (stat, e))
       else:
         return default
 
@@ -426,7 +434,7 @@ class Coverage(object):
       src_dir: Directory on disk at which to start search.  May be a relative
           path on disk starting with '.' or '..', or an absolute path, or a
           path relative to an alt_name for one of the roots
-          (for example, '#/src').  If the alt_name matches more than one root,
+          (for example, '_/src').  If the alt_name matches more than one root,
           all matches will be attempted.
 
     Note that dirs not underneath one of the root dirs and covered by an
@@ -451,8 +459,8 @@ class Coverage(object):
         # Add trailing '/' to directory names so dir-based regexps can match
         # '/' instead of needing to specify '(/|$)'.
         dpath = self.CleanupFilename(dirpath + '/' + d) + '/'
-        group, language = self.ClassifyFile(dpath)
-        if not group:
+        attrs = self.ClassifyFile(dpath)
+        if not attrs.get('include'):
           # Directory has been excluded, so don't traverse it
           # TODO: Document the slight weirdness caused by this: If you
           # AddFiles('./A'), and the rules include 'A/B/C/D' but not 'A/B',
@@ -463,12 +471,33 @@ class Coverage(object):
           dirnames.remove(d)
 
       for f in filenames:
-        covf = self.GetCoveredFile(dirpath + '/' + f, add=True)
-        # TODO: scan files for executable lines.  Add these to the file as
-        # 'executable', but not 'instrumented' or 'covered'.
-        # TODO: if a file has no executable lines, don't add it.
-        if covf:
+        local_path = dirpath + '/' + f
+
+        covf = self.GetCoveredFile(local_path, add=True)
+        if not covf:
+          continue
+
+        # Save where we found the file, for generating line-by-line HTML output
+        covf.local_path = local_path
+
+        if covf.in_lcov:
+          # File already instrumented and doesn't need to be scanned
+          continue
+
+        if not covf.attrs.get('add_if_missing', 1):
+          # Not allowed to add the file
+          self.RemoveCoveredFile(covf)
+          continue
+
+        # Scan file to find potentially-executable lines
+        lines = self.scan_file(covf.local_path, covf.attrs.get('language'))
+        if lines:
+          for l in lines:
+            covf.lines[l] = None
           covf.UpdateCoverage()
+        else:
+          # File has no executable lines, so don't count it
+          self.RemoveCoveredFile(covf)
 
   def AddConfig(self, config_data, lcov_queue=None, addfiles_queue=None):
     """Adds JSON-ish config data.
@@ -481,16 +510,14 @@ class Coverage(object):
           processing them immediately.
     """
     # TODO: All manner of error checking
-    cfg = eval(config_data, {'__builtins__':{}}, {})
+    cfg = eval(config_data, {'__builtins__': {}}, {})
 
     for rootdict in cfg.get('roots', []):
-      self.AddRoot(rootdict['root'], alt_name=rootdict.get('altname', '#'))
+      self.AddRoot(rootdict['root'], alt_name=rootdict.get('altname', '_'))
 
     for ruledict in cfg.get('rules', []):
-      self.AddRule(ruledict['regexp'],
-                   include=ruledict.get('include'),
-                   group=ruledict.get('group'),
-                   language=ruledict.get('language'))
+      regexp = ruledict.pop('regexp')
+      self.AddRule(regexp, **ruledict)
 
     for add_lcov in cfg.get('lcov_files', []):
       if lcov_queue is not None:
@@ -511,6 +538,7 @@ class Coverage(object):
 
     Args:
       filename: Config filename.
+      kwargs: Additional parameters to pass to AddConfig().
     """
     # TODO: All manner of error checking
     f = None
@@ -519,10 +547,18 @@ class Coverage(object):
       # Need to strip CR's from CRLF-terminated lines or posix systems can't
       # eval the data.
       config_data = f.read().replace('\r\n', '\n')
-      # TODO: some sort of include syntax.  Needs to be done at string-time
-      # rather than at eval()-time, so that it's possible to include parts of
-      # dicts.  Path from a file to its include should be relative to the dir
-      # containing the file.
+      # TODO: some sort of include syntax.
+      #
+      # Needs to be done at string-time rather than at eval()-time, so that
+      # it's possible to include parts of dicts.  Path from a file to its
+      # include should be relative to the dir containing the file.
+      #
+      # Or perhaps it could be done after eval.  In that case, there'd be an
+      # 'include' section with a list of files to include.  Those would be
+      # eval()'d and recursively pre- or post-merged with the including file.
+      #
+      # Or maybe just don't worry about it, since multiple configs can be
+      # specified on the command line.
       self.AddConfig(config_data, **kwargs)
     finally:
       if f:
@@ -531,17 +567,20 @@ class Coverage(object):
   def UpdateTreeStats(self):
     """Recalculates the tree stats from the currently covered files.
 
-    Also calculates coverage summary for files."""
+    Also calculates coverage summary for files.
+    """
     self.tree = CoveredDir('')
     for cov_file in self.files.itervalues():
       # Add the file to the tree
-      # TODO: Don't really need to create the tree unless we're creating HTML
       fdirs = cov_file.filename.split('/')
       parent = self.tree
       ancestors = [parent]
       for d in fdirs[:-1]:
         if d not in parent.subdirs:
-          parent.subdirs[d] = CoveredDir(d)
+          if parent.dirpath:
+            parent.subdirs[d] = CoveredDir(parent.dirpath + '/' + d)
+          else:
+            parent.subdirs[d] = CoveredDir(d)
         parent = parent.subdirs[d]
         ancestors.append(parent)
       # Final subdir actually contains the file
@@ -553,9 +592,10 @@ class Coverage(object):
         a.stats_by_group['all'].Add(cov_file.stats)
 
         # Add to group file belongs to
-        if cov_file.group not in a.stats_by_group:
-          a.stats_by_group[cov_file.group] = CoverageStats()
-        cbyg = a.stats_by_group[cov_file.group]
+        group = cov_file.attrs.get('group')
+        if group not in a.stats_by_group:
+          a.stats_by_group[group] = CoverageStats()
+        cbyg = a.stats_by_group[group]
         cbyg.Add(cov_file.stats)
 
   def PrintTree(self):
@@ -577,7 +617,7 @@ def Main(argv):
     exit code, 0 for normal exit.
   """
   # Parse args
-  parser = OptionParser()
+  parser = optparse.OptionParser()
   parser.add_option(
       '-i', '--input', dest='inputs', type='string', action='append',
       metavar='FILE',
@@ -600,6 +640,9 @@ def Main(argv):
   parser.add_option(
       '-u', '--uninstrumented', dest='uninstrumented', action='store_true',
       help='list uninstrumented files')
+  parser.add_option(
+      '-m', '--html', dest='html_out', type='string', metavar='PATH',
+      help='write HTML output to PATH')
 
   parser.set_defaults(
       inputs=[],
@@ -607,9 +650,10 @@ def Main(argv):
       configs=[],
       addfiles=[],
       tree=False,
+      html_out=None,
   )
 
-  (options, args) = parser.parse_args()
+  options = parser.parse_args(args=argv)[0]
 
   cov = Coverage()
 
@@ -647,9 +691,9 @@ def Main(argv):
     print 'Uninstrumented files:'
     for f in sorted(cov.files):
       covf = cov.files[f]
-      if not covf.stats.get('lines_instrumented'):
-        print '  %-6s %-6s %s' % (covf.group, covf.language, f)
-
+      if not covf.in_lcov:
+        print '  %-6s %-6s %s' % (covf.attrs.get('group'),
+                                  covf.attrs.get('language'), f)
 
   # Print tree stats
   if options.tree:
@@ -659,6 +703,11 @@ def Main(argv):
   for ps_args in cov.print_stats:
     cov.PrintStat(**ps_args)
 
+  # Generate HTML
+  if options.html_out:
+    html = croc_html.CrocHtml(cov, options.html_out)
+    html.Write()
+
   # Normal exit
   return 0
 
diff --git a/tools/code_coverage/croc_html.py b/tools/code_coverage/croc_html.py
new file mode 100644
index 0000000..79ec85d
--- /dev/null
+++ b/tools/code_coverage/croc_html.py
@@ -0,0 +1,453 @@
+#!/usr/bin/python2.4
+#
+# Copyright 2009, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#        * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#        * Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following disclaimer
+#     in the documentation and/or other materials provided with the
+#     distribution.
+#        * Neither the name of Google Inc. nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#     A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#     LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#     DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#     THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#     (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Crocodile HTML output."""
+
+import os
+import shutil
+import time
+import xml.dom
+
+
+class CrocHtmlError(Exception):
+  """Coverage HTML error."""
+
+
+class HtmlElement(object):
+  """Node in a HTML file."""
+
+  def __init__(self, doc, element):
+    """Constructor.
+
+    Args:
+      doc: XML document object.
+      element: XML element.
+    """
+    self.doc = doc
+    self.element = element
+
+  def E(self, name, **kwargs):
+    """Adds a child element.
+
+    Args:
+      name: Name of element.
+      kwargs: Attributes for element.  To use an attribute which is a python
+          reserved word (i.e. 'class'), prefix the attribute name with 'e_'.
+
+    Returns:
+      The child element.
+    """
+    he = HtmlElement(self.doc, self.doc.createElement(name))
+    element = he.element
+    self.element.appendChild(element)
+
+    for k, v in kwargs.iteritems():
+      if k.startswith('e_'):
+        # Remove prefix
+        element.setAttribute(k[2:], str(v))
+      else:
+        element.setAttribute(k, str(v))
+
+    return he
+
+  def Text(self, text):
+    """Adds a text node.
+
+    Args:
+      text: Text to add.
+
+    Returns:
+      self.
+    """
+    t = self.doc.createTextNode(str(text))
+    self.element.appendChild(t)
+    return self
+
+
+class HtmlFile(object):
+  """HTML file."""
+
+  def __init__(self, xml_impl, filename):
+    """Constructor.
+
+    Args:
+      xml_impl: DOMImplementation to use to create document.
+      filename: Path to file.
+    """
+    self.xml_impl = xml_impl
+    doctype = xml_impl.createDocumentType(
+        'HTML', '-//W3C//DTD HTML 4.01//EN',
+        'http://www.w3.org/TR/html4/strict.dtd')
+    self.doc = xml_impl.createDocument(None, 'html', doctype)
+    self.filename = filename
+
+    # Create head and body elements
+    root = HtmlElement(self.doc, self.doc.documentElement)
+    self.head = root.E('head')
+    self.body = root.E('body')
+
+  def Write(self, cleanup=True):
+    """Writes the file.
+
+    Args:
+      cleanup: If True, calls unlink() on the internal xml document.  This
+          frees up memory, but means that you can't use this file for anything
+          else.
+    """
+    f = open(self.filename, 'wt')
+    self.doc.writexml(f, encoding='UTF-8')
+    f.close()
+
+    if cleanup:
+      self.doc.unlink()
+      # Prevent future uses of the doc now that we've unlinked it
+      self.doc = None
+
+#------------------------------------------------------------------------------
+
+COV_TYPE_STRING = {None: 'm', 0: 'i', 1: 'E', 2: ' '}
+COV_TYPE_CLASS = {None: 'missing', 0: 'instr', 1: 'covered', 2: ''}
+
+
+class CrocHtml(object):
+  """Crocodile HTML output class."""
+
+  def __init__(self, cov, output_root):
+    """Constructor."""
+    self.cov = cov
+    self.output_root = output_root
+    self.xml_impl = xml.dom.getDOMImplementation()
+    self.time_string = 'Coverage information generated %s.' % time.asctime()
+
+  def CreateHtmlDoc(self, filename, title):
+    """Creates a new HTML document.
+
+    Args:
+      filename: Filename to write to, relative to self.output_root.
+      title: Title of page
+
+    Returns:
+      The document.
+    """
+    f = HtmlFile(self.xml_impl, self.output_root + '/' + filename)
+
+    f.head.E('title').Text(title)
+    f.head.E(
+        'link', rel='stylesheet', type='text/css',
+        href='../' * (len(filename.split('/')) - 1) + 'croc.css')
+
+    return f
+
+  def AddCaptionForFile(self, body, path):
+    """Adds a caption for the file, with links to each parent dir.
+
+    Args:
+      body: Body elemement.
+      path: Path to file.
+    """
+    # This is slightly different that for subdir, because it needs to have a
+    # link to the current directory's index.html.
+    hdr = body.E('h2')
+    hdr.Text('Coverage for ')
+    dirs = [''] + path.split('/')
+    num_dirs = len(dirs)
+    for i in range(num_dirs - 1):
+      hdr.E('a', href=(
+          '../' * (num_dirs - i - 2) + 'index.html')).Text(dirs[i] + '/')
+    hdr.Text(dirs[-1])
+
+  def AddCaptionForSubdir(self, body, path):
+    """Adds a caption for the subdir, with links to each parent dir.
+
+    Args:
+      body: Body elemement.
+      path: Path to subdir.
+    """
+    # Link to parent dirs
+    hdr = body.E('h2')
+    hdr.Text('Coverage for ')
+    dirs = [''] + path.split('/')
+    num_dirs = len(dirs)
+    for i in range(num_dirs - 1):
+      hdr.E('a', href=(
+          '../' * (num_dirs - i - 1) + 'index.html')).Text(dirs[i] + '/')
+    hdr.Text(dirs[-1] + '/')
+
+  def AddSectionHeader(self, table, caption, itemtype, is_file=False):
+    """Adds a section header to the coverage table.
+
+    Args:
+      table: Table to add rows to.
+      caption: Caption for section, if not None.
+      itemtype: Type of items in this section, if not None.
+      is_file: Are items in this section files?
+    """
+
+    if caption is not None:
+      table.E('tr').E('td', e_class='secdesc', colspan=8).Text(caption)
+
+    sec_hdr = table.E('tr')
+
+    if itemtype is not None:
+      sec_hdr.E('td', e_class='section').Text(itemtype)
+
+    sec_hdr.E('td', e_class='section').Text('Coverage')
+    sec_hdr.E('td', e_class='section', colspan=3).Text(
+        'Lines executed / instrumented / missing')
+
+    graph = sec_hdr.E('td', e_class='section')
+    graph.E('span', style='color:#00FF00').Text('exe')
+    graph.Text(' / ')
+    graph.E('span', style='color:#FFFF00').Text('inst')
+    graph.Text(' / ')
+    graph.E('span', style='color:#FF0000').Text('miss')
+
+    if is_file:
+      sec_hdr.E('td', e_class='section').Text('Language')
+      sec_hdr.E('td', e_class='section').Text('Group')
+    else:
+      sec_hdr.E('td', e_class='section', colspan=2)
+
+  def AddItem(self, table, itemname, stats, attrs, link=None):
+    """Adds a bar graph to the element.  This is a series of <td> elements.
+
+    Args:
+      table: Table to add item to.
+      itemname: Name of item.
+      stats: Stats object.
+      attrs: Attributes dictionary; if None, no attributes will be printed.
+      link: Destination for itemname hyperlink, if not None.
+    """
+    row = table.E('tr')
+
+    # Add item name
+    if itemname is not None:
+      item_elem = row.E('td')
+      if link is not None:
+        item_elem = item_elem.E('a', href=link)
+      item_elem.Text(itemname)
+
+    # Get stats
+    stat_exe = stats.get('lines_executable', 0)
+    stat_ins = stats.get('lines_instrumented', 0)
+    stat_cov = stats.get('lines_covered', 0)
+
+    percent = row.E('td')
+
+    # Add text
+    row.E('td', e_class='number').Text(stat_cov)
+    row.E('td', e_class='number').Text(stat_ins)
+    row.E('td', e_class='number').Text(stat_exe - stat_ins)
+
+    # Add percent and graph; only fill in if there's something in there
+    graph = row.E('td', e_class='graph', width=100)
+    if stat_exe:
+      percent_cov = 100.0 * stat_cov / stat_exe
+      percent_ins = 100.0 * stat_ins / stat_exe
+
+      # Color percent based on thresholds
+      percent.Text('%.1f%%' % percent_cov)
+      if percent_cov >= 80:
+        percent.element.setAttribute('class', 'high_pct')
+      elif percent_cov >= 60:
+        percent.element.setAttribute('class', 'mid_pct')
+      else:
+        percent.element.setAttribute('class', 'low_pct')
+
+      # Graphs use integer values
+      percent_cov = int(percent_cov)
+      percent_ins = int(percent_ins)
+
+      graph.Text('.')
+      graph.E('span', style='padding-left:%dpx' % percent_cov,
+              e_class='g_covered')
+      graph.E('span', style='padding-left:%dpx' % (percent_ins - percent_cov),
+              e_class='g_instr')
+      graph.E('span', style='padding-left:%dpx' % (100 - percent_ins),
+              e_class='g_missing')
+
+    if attrs:
+      row.E('td', e_class='stat').Text(attrs.get('language'))
+      row.E('td', e_class='stat').Text(attrs.get('group'))
+    else:
+      row.E('td', colspan=2)
+
+  def WriteFile(self, cov_file):
+    """Writes the HTML for a file.
+
+    Args:
+      cov_file: croc.CoveredFile to write.
+    """
+    print '  ' + cov_file.filename
+    title = 'Coverage for ' + cov_file.filename
+
+    f = self.CreateHtmlDoc(cov_file.filename + '.html', title)
+    body = f.body
+
+    # Write header section
+    self.AddCaptionForFile(body, cov_file.filename)
+
+    # Summary for this file
+    table = body.E('table')
+    self.AddSectionHeader(table, None, None, is_file=True)
+    self.AddItem(table, None, cov_file.stats, cov_file.attrs)
+
+    body.E('h2').Text('Line-by-line coverage:')
+
+    # Print line-by-line coverage
+    if cov_file.local_path:
+      code_table = body.E('table').E('tr').E('td').E('pre')
+
+      flines = open(cov_file.local_path, 'rt')
+      lineno = 0
+
+      for line in flines:
+        lineno += 1
+        line_cov = cov_file.lines.get(lineno, 2)
+        e_class = COV_TYPE_CLASS.get(line_cov)
+
+        code_table.E('span', e_class=e_class).Text('%4d  %s :  %s\n' % (
+            lineno,
+            COV_TYPE_STRING.get(line_cov),
+            line.rstrip()
+        ))
+
+    else:
+      body.Text('Line-by-line coverage not available.  Make sure the directory'
+                ' containing this file has been scanned via ')
+      body.E('B').Text('add_files')
+      body.Text(' in a configuration file, or the ')
+      body.E('B').Text('--addfiles')
+      body.Text(' command line option.')
+
+      # TODO: if file doesn't have a local path, try to find it by
+      # reverse-mapping roots and searching for the file.
+
+    body.E('p', e_class='time').Text(self.time_string)
+    f.Write()
+
+  def WriteSubdir(self, cov_dir):
+    """Writes the index.html for a subdirectory.
+
+    Args:
+      cov_dir: croc.CoveredDir to write.
+    """
+    print '  ' + cov_dir.dirpath + '/'
+
+    # Create the subdir if it doesn't already exist
+    subdir = self.output_root + '/' + cov_dir.dirpath
+    if not os.path.exists(subdir):
+      os.mkdir(subdir)
+
+    if cov_dir.dirpath:
+      title = 'Coverage for ' + cov_dir.dirpath + '/'
+      f = self.CreateHtmlDoc(cov_dir.dirpath + '/index.html', title)
+    else:
+      title = 'Coverage summary'
+      f = self.CreateHtmlDoc('index.html', title)
+
+    body = f.body
+
+    # Write header section
+    if cov_dir.dirpath:
+      self.AddCaptionForSubdir(body, cov_dir.dirpath)
+    else:
+      body.E('h2').Text(title)
+
+    table = body.E('table')
+
+    # Coverage by group
+    self.AddSectionHeader(table, 'Coverage by Group', 'Group')
+
+    for group in sorted(cov_dir.stats_by_group):
+      self.AddItem(table, group, cov_dir.stats_by_group[group], None)
+
+    # List subdirs
+    if cov_dir.subdirs:
+      self.AddSectionHeader(table, 'Subdirectories', 'Subdirectory')
+
+      for d in sorted(cov_dir.subdirs):
+        self.AddItem(table, d + '/', cov_dir.subdirs[d].stats_by_group['all'],
+                     None, link=d + '/index.html')
+
+    # List files
+    if cov_dir.files:
+      self.AddSectionHeader(table, 'Files in This Directory', 'Filename',
+                            is_file=True)
+
+      for filename in sorted(cov_dir.files):
+        cov_file = cov_dir.files[filename]
+        self.AddItem(table, filename, cov_file.stats, cov_file.attrs,
+                     link=filename + '.html')
+
+    body.E('p', e_class='time').Text(self.time_string)
+    f.Write()
+
+  def WriteRoot(self):
+    """Writes the files in the output root."""
+    # Find ourselves
+    src_dir = os.path.split(self.WriteRoot.func_code.co_filename)[0]
+
+    # Files to copy into output root
+    copy_files = [
+        'croc.css',
+    ]
+
+    # Copy files from our directory into the output directory
+    for copy_file in copy_files:
+      print '  Copying %s' % copy_file
+      shutil.copyfile(os.path.join(src_dir, copy_file),
+                      os.path.join(self.output_root, copy_file))
+
+  def Write(self):
+    """Writes HTML output."""
+
+    print 'Writing HTML to %s...' % self.output_root
+
+    # Loop through the tree and write subdirs, breadth-first
+    # TODO: switch to depth-first and sort values - makes nicer output?
+    todo = [self.cov.tree]
+    while todo:
+      cov_dir = todo.pop(0)
+
+      # Append subdirs to todo list
+      todo += cov_dir.subdirs.values()
+
+      # Write this subdir
+      self.WriteSubdir(cov_dir)
+
+      # Write files in this subdir
+      for cov_file in cov_dir.files.itervalues():
+        self.WriteFile(cov_file)
+
+    # Write files in root directory
+    self.WriteRoot()
+
diff --git a/tools/code_coverage/croc_scan.py b/tools/code_coverage/croc_scan.py
new file mode 100644
index 0000000..e40b45e
--- /dev/null
+++ b/tools/code_coverage/croc_scan.py
@@ -0,0 +1,191 @@
+#!/usr/bin/python2.4
+#
+# Copyright 2009, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#        * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#        * Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following disclaimer
+#     in the documentation and/or other materials provided with the
+#     distribution.
+#        * Neither the name of Google Inc. nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#     A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#     LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#     DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#     THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#     (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Crocodile source scanners."""
+
+
+import re
+
+
+class Scanner(object):
+  """Generic source scanner."""
+
+  def __init__(self):
+    """Constructor."""
+
+    self.re_token = re.compile('#')
+    self.comment_to_eol = ['#']
+    self.comment_start = None
+    self.comment_end = None
+
+  def ScanLines(self, lines):
+    """Scans the lines for executable statements.
+
+    Args:
+      lines: Iterator returning source lines.
+
+    Returns:
+      An array of line numbers which are executable.
+    """
+    exe_lines = []
+    lineno = 0
+
+    in_string = None
+    in_comment = None
+    comment_index = None
+
+    for line in lines:
+      lineno += 1
+      in_string_at_start = in_string
+
+      for t in self.re_token.finditer(line):
+        tokenstr = t.groups()[0]
+
+        if in_comment:
+          # Inside a multi-line comment, so look for end token
+          if tokenstr == in_comment:
+            in_comment = None
+            # Replace comment with spaces
+            line = (line[:comment_index]
+                    + ' ' * (t.end(0) - comment_index)
+                    + line[t.end(0):])
+
+        elif in_string:
+          # Inside a string, so look for end token
+          if tokenstr == in_string:
+            in_string = None
+
+        elif tokenstr in self.comment_to_eol:
+          # Single-line comment, so truncate line at start of token
+          line = line[:t.start(0)]
+          break
+
+        elif tokenstr == self.comment_start:
+          # Multi-line comment start - end token is comment_end
+          in_comment = self.comment_end
+          comment_index = t.start(0)
+
+        else:
+          # Starting a string - end token is same as start
+          in_string = tokenstr
+
+      # If still in comment at end of line, remove comment
+      if in_comment:
+        line = line[:comment_index]
+        # Next line, delete from the beginnine
+        comment_index = 0
+
+      # If line-sans-comments is not empty, claim it may be executable
+      if line.strip() or in_string_at_start:
+        exe_lines.append(lineno)
+
+    # Return executable lines
+    return exe_lines
+
+  def Scan(self, filename):
+    """Reads the file and scans its lines.
+
+    Args:
+      filename: Path to file to scan.
+
+    Returns:
+      An array of line numbers which are executable.
+    """
+
+    # TODO: All manner of error checking
+    f = None
+    try:
+      f = open(filename, 'rt')
+      return self.ScanLines(f)
+    finally:
+      if f:
+        f.close()
+
+
+class PythonScanner(Scanner):
+  """Python source scanner."""
+
+  def __init__(self):
+    """Constructor."""
+    Scanner.__init__(self)
+
+    # TODO: This breaks for strings ending in more than 2 backslashes.  Need
+    # a pattern which counts only an odd number of backslashes, so the last
+    # one thus escapes the quote.
+    self.re_token = re.compile(r'(#|\'\'\'|"""|(?<!(?<!\\)\\)["\'])')
+    self.comment_to_eol = ['#']
+    self.comment_start = None
+    self.comment_end = None
+
+
+class CppScanner(Scanner):
+  """C / C++ / ObjC / ObjC++ source scanner."""
+
+  def __init__(self):
+    """Constructor."""
+    Scanner.__init__(self)
+
+    # TODO: This breaks for strings ending in more than 2 backslashes.  Need
+    # a pattern which counts only an odd number of backslashes, so the last
+    # one thus escapes the quote.
+    self.re_token = re.compile(r'(^\s*#|//|/\*|\*/|(?<!(?<!\\)\\)["\'])')
+
+    # TODO: Treat '\' at EOL as a token, and handle it as continuing the
+    # previous line.  That is, if in a comment-to-eol, this line is a comment
+    # too.
+
+    # Note that we treat # at beginning of line as a comment, so that we ignore
+    # preprocessor definitions
+    self.comment_to_eol = ['//', '#']
+
+    self.comment_start = '/*'
+    self.comment_end = '*/'
+
+
+def ScanFile(filename, language):
+  """Scans a file for executable lines.
+
+  Args:
+    filename: Path to file to scan.
+    language: Language for file ('C', 'C++', 'python', 'ObjC', 'ObjC++')
+
+  Returns:
+    A list of executable lines, or an empty list if the file was not a handled
+        language.
+  """
+
+  if language == 'python':
+    return PythonScanner().Scan(filename)
+  elif language in ['C', 'C++', 'ObjC', 'ObjC++']:
+    return CppScanner().Scan(filename)
+
+  # Something we don't handle
+  return []
diff --git a/tools/code_coverage/croc_scan_test.py b/tools/code_coverage/croc_scan_test.py
new file mode 100644
index 0000000..17fc7fa
--- /dev/null
+++ b/tools/code_coverage/croc_scan_test.py
@@ -0,0 +1,219 @@
+#!/usr/bin/python2.4
+#
+# Copyright 2009, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#        * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#        * Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following disclaimer
+#     in the documentation and/or other materials provided with the
+#     distribution.
+#        * Neither the name of Google Inc. nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#     A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#     LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#     DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#     THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#     (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for croc_scan.py."""
+
+#import os
+import re
+#import sys
+#import StringIO
+import unittest
+import croc_scan
+
+#------------------------------------------------------------------------------
+
+
+class TestScanner(unittest.TestCase):
+  """Tests for croc_scan.Scanner."""
+
+  def testInit(self):
+    """Test __init()__."""
+    s = croc_scan.Scanner()
+
+    self.assertEqual(s.re_token.pattern, '#')
+    self.assertEqual(s.comment_to_eol, ['#'])
+    self.assertEqual(s.comment_start, None)
+    self.assertEqual(s.comment_end, None)
+
+  def testScanLines(self):
+    """Test ScanLines()."""
+    s = croc_scan.Scanner()
+    # Set up imaginary language:
+    #   ':' = comment to EOL
+    #   '"' = string start/end
+    #   '(' = comment start
+    #   ')' = comment end
+    s.re_token = re.compile(r'([\:\"\(\)])')
+    s.comment_to_eol = [':']
+    s.comment_start = '('
+    s.comment_end = ')'
+
+    # No input file = no output lines
+    self.assertEqual(s.ScanLines([]), [])
+
+    # Empty lines and lines with only whitespace are ignored
+    self.assertEqual(s.ScanLines([
+        '',             # 1
+        'line',         # 2 exe
+        ' \t  ',        # 3
+    ]), [2])
+
+    # Comments to EOL are stripped, but not inside strings
+    self.assertEqual(s.ScanLines([
+        'test',                                                 # 1 exe
+        '   : A comment',                                       # 2
+        '"a : in a string"',                                    # 3 exe
+        'test2 : with comment to EOL',                          # 4 exe
+        'foo = "a multiline string with an empty line',         # 5 exe
+        '',                                                     # 6 exe
+        ': and a comment-to-EOL character"',                    # 7 exe
+        ': done',                                               # 8
+    ]), [1, 3, 4, 5, 6, 7])
+
+    # Test Comment start/stop detection
+    self.assertEqual(s.ScanLines([
+        '( a comment on one line)',                     # 1
+        'text (with a comment)',                        # 2 exe
+        '( a comment with a : in the middle)',          # 3
+        '( a multi-line',                               # 4
+        '  comment)',                                   # 5
+        'a string "with a ( in it"',                    # 6 exe
+        'not in a multi-line comment',                  # 7 exe
+        '(a comment with a " in it)',                   # 8
+        ': not in a string, so this gets stripped',     # 9
+        'more text "with an uninteresting string"',     # 10 exe
+    ]), [2, 6, 7, 10])
+
+  # TODO: Test Scan().  Low priority, since it just wraps ScanLines().
+
+
+class TestPythonScanner(unittest.TestCase):
+  """Tests for croc_scan.PythonScanner."""
+
+  def testScanLines(self):
+    """Test ScanLines()."""
+    s = croc_scan.PythonScanner()
+
+    # No input file = no output lines
+    self.assertEqual(s.ScanLines([]), [])
+
+    self.assertEqual(s.ScanLines([
+        '# a comment',                                  # 1
+        '',                                             # 2
+        '"""multi-line string',                         # 3 exe
+        '# not a comment',                              # 4 exe
+        'end of multi-line string"""',                  # 5 exe
+        '  ',                                           # 6
+        '"single string with #comment"',                # 7 exe
+        '',                                             # 8
+        '\'\'\'multi-line string, single-quote',        # 9 exe
+        '# not a comment',                              # 10 exe
+        'end of multi-line string\'\'\'',               # 11 exe
+        '',                                             # 12
+        '"string with embedded \\" is handled"',        # 13 exe
+        '# quoted "',                                   # 14
+        '"\\""',                                        # 15 exe
+        '# quoted backslash',                           # 16
+        '"\\\\"',                                       # 17 exe
+        'main()',                                       # 18 exe
+        '# end',                                        # 19
+    ]), [3, 4, 5, 7, 9, 10, 11, 13, 15, 17, 18])
+
+
+class TestCppScanner(unittest.TestCase):
+  """Tests for croc_scan.CppScanner."""
+
+  def testScanLines(self):
+    """Test ScanLines()."""
+    s = croc_scan.CppScanner()
+
+    # No input file = no output lines
+    self.assertEqual(s.ScanLines([]), [])
+
+    self.assertEqual(s.ScanLines([
+        '// a comment',                                 # 1
+        '# a preprocessor define',                      # 2
+        '',                                             # 3
+        '\'#\', \'"\'',                                 # 4 exe
+        '',                                             # 5
+        '/* a multi-line comment',                      # 6
+        'with a " in it',                               # 7
+        '*/',                                           # 8
+        '',                                             # 9
+        '"a string with /* and \' in it"',              # 10 exe
+        '',                                             # 11
+        '"a multi-line string\\',                       # 12 exe
+        '// not a comment\\',                           # 13 exe
+        'ending here"',                                 # 14 exe
+        '',                                             # 15
+        '"string with embedded \\" is handled"',        # 16 exe
+        '',                                             # 17
+        'main()',                                       # 18 exe
+        '// end',                                       # 19
+    ]), [4, 10, 12, 13, 14, 16, 18])
+
+
+class TestScanFile(unittest.TestCase):
+  """Tests for croc_scan.ScanFile()."""
+
+  class MockScanner(object):
+    """Mock scanner."""
+
+    def __init__(self, language):
+      """Constructor."""
+      self.language = language
+
+    def Scan(self, filename):
+      """Mock Scan() method."""
+      return 'scan %s %s' % (self.language, filename)
+
+  def MockPythonScanner(self):
+    return self.MockScanner('py')
+
+  def MockCppScanner(self):
+    return self.MockScanner('cpp')
+
+  def setUp(self):
+    """Per-test setup."""
+    # Hook scanners
+    self.old_python_scanner = croc_scan.PythonScanner
+    self.old_cpp_scanner = croc_scan.CppScanner
+    croc_scan.PythonScanner = self.MockPythonScanner
+    croc_scan.CppScanner = self.MockCppScanner
+
+  def tearDown(self):
+    """Per-test cleanup."""
+    croc_scan.PythonScanner = self.old_python_scanner
+    croc_scan.CppScanner = self.old_cpp_scanner
+
+  def testScanFile(self):
+    """Test ScanFile()."""
+    self.assertEqual(croc_scan.ScanFile('foo', 'python'), 'scan py foo')
+    self.assertEqual(croc_scan.ScanFile('bar1', 'C'), 'scan cpp bar1')
+    self.assertEqual(croc_scan.ScanFile('bar2', 'C++'), 'scan cpp bar2')
+    self.assertEqual(croc_scan.ScanFile('bar3', 'ObjC'), 'scan cpp bar3')
+    self.assertEqual(croc_scan.ScanFile('bar4', 'ObjC++'), 'scan cpp bar4')
+    self.assertEqual(croc_scan.ScanFile('bar', 'fortran'), [])
+
+#------------------------------------------------------------------------------
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/tools/code_coverage/croc_test.py b/tools/code_coverage/croc_test.py
index 028521c..035fcf9 100644
--- a/tools/code_coverage/croc_test.py
+++ b/tools/code_coverage/croc_test.py
@@ -32,14 +32,13 @@
 """Unit tests for Crocodile."""
 
 import os
-import re
-import sys
 import StringIO
 import unittest
 import croc
 
 #------------------------------------------------------------------------------
 
+
 class TestCoverageStats(unittest.TestCase):
   """Tests for croc.CoverageStats."""
 
@@ -53,23 +52,24 @@ class TestCoverageStats(unittest.TestCase):
     # Add items
     c['a'] = 1
     c['b'] = 0
-    self.assertEqual(c, {'a':1, 'b':0})
+    self.assertEqual(c, {'a': 1, 'b': 0})
 
     # Add dict with non-overlapping items
-    c.Add({'c':5})
-    self.assertEqual(c, {'a':1, 'b':0, 'c':5})
+    c.Add({'c': 5})
+    self.assertEqual(c, {'a': 1, 'b': 0, 'c': 5})
 
     # Add dict with overlapping items
-    c.Add({'a':4, 'd':3})
-    self.assertEqual(c, {'a':5, 'b':0, 'c':5, 'd':3})
+    c.Add({'a': 4, 'd': 3})
+    self.assertEqual(c, {'a': 5, 'b': 0, 'c': 5, 'd': 3})
 
 #------------------------------------------------------------------------------
 
+
 class TestCoveredFile(unittest.TestCase):
   """Tests for croc.CoveredFile."""
 
   def setUp(self):
-    self.cov_file = croc.CoveredFile('bob.cc', 'source', 'C++')
+    self.cov_file = croc.CoveredFile('bob.cc', group='source', language='C++')
 
   def testInit(self):
     """Test init."""
@@ -77,63 +77,76 @@ class TestCoveredFile(unittest.TestCase):
 
     # Check initial values
     self.assertEqual(f.filename, 'bob.cc')
-    self.assertEqual(f.group, 'source')
-    self.assertEqual(f.language, 'C++')
+    self.assertEqual(f.attrs, {'group': 'source', 'language': 'C++'})
     self.assertEqual(f.lines, {})
     self.assertEqual(f.stats, {})
+    self.assertEqual(f.local_path, None)
+    self.assertEqual(f.in_lcov, False)
 
   def testUpdateCoverageEmpty(self):
     """Test updating coverage when empty."""
     f = self.cov_file
     f.UpdateCoverage()
     self.assertEqual(f.stats, {
-        'lines_executable':0,
-        'lines_instrumented':0,
-        'lines_covered':0,
-        'files_executable':1,
+        'lines_executable': 0,
+        'lines_instrumented': 0,
+        'lines_covered': 0,
+        'files_executable': 1,
     })
 
   def testUpdateCoverageExeOnly(self):
     """Test updating coverage when no lines are instrumented."""
     f = self.cov_file
-    f.lines = {1:None, 2:None, 4:None}
+    f.lines = {1: None, 2: None, 4: None}
     f.UpdateCoverage()
     self.assertEqual(f.stats, {
-        'lines_executable':3,
-        'lines_instrumented':0,
-        'lines_covered':0,
-        'files_executable':1,
+        'lines_executable': 3,
+        'lines_instrumented': 0,
+        'lines_covered': 0,
+        'files_executable': 1,
+    })
+
+    # Now mark the file instrumented via in_lcov
+    f.in_lcov = True
+    f.UpdateCoverage()
+    self.assertEqual(f.stats, {
+        'lines_executable': 3,
+        'lines_instrumented': 0,
+        'lines_covered': 0,
+        'files_executable': 1,
+        'files_instrumented': 1,
     })
 
   def testUpdateCoverageExeAndInstr(self):
     """Test updating coverage when no lines are covered."""
     f = self.cov_file
-    f.lines = {1:None, 2:None, 4:0, 5:0, 7:None}
+    f.lines = {1: None, 2: None, 4: 0, 5: 0, 7: None}
     f.UpdateCoverage()
     self.assertEqual(f.stats, {
-        'lines_executable':5,
-        'lines_instrumented':2,
-        'lines_covered':0,
-        'files_executable':1,
-        'files_instrumented':1,
+        'lines_executable': 5,
+        'lines_instrumented': 2,
+        'lines_covered': 0,
+        'files_executable': 1,
+        'files_instrumented': 1,
     })
 
   def testUpdateCoverageWhenCovered(self):
     """Test updating coverage when lines are covered."""
     f = self.cov_file
-    f.lines = {1:None, 2:None, 3:1, 4:0, 5:0, 6:1, 7:None}
+    f.lines = {1: None, 2: None, 3: 1, 4: 0, 5: 0, 6: 1, 7: None}
     f.UpdateCoverage()
     self.assertEqual(f.stats, {
-        'lines_executable':7,
-        'lines_instrumented':4,
-        'lines_covered':2,
-        'files_executable':1,
-        'files_instrumented':1,
-        'files_covered':1,
+        'lines_executable': 7,
+        'lines_instrumented': 4,
+        'lines_covered': 2,
+        'files_executable': 1,
+        'files_instrumented': 1,
+        'files_covered': 1,
     })
 
 #------------------------------------------------------------------------------
 
+
 class TestCoveredDir(unittest.TestCase):
   """Tests for croc.CoveredDir."""
 
@@ -148,12 +161,12 @@ class TestCoveredDir(unittest.TestCase):
     self.assertEqual(d.dirpath, '/a/b/c')
     self.assertEqual(d.files, {})
     self.assertEqual(d.subdirs, {})
-    self.assertEqual(d.stats_by_group, {'all':{}})
+    self.assertEqual(d.stats_by_group, {'all': {}})
 
   def testGetTreeEmpty(self):
     """Test getting empty tree."""
     d = self.cov_dir
-    self.assertEqual(d.GetTree(), '/a/b/c/')
+    self.assertEqual(d.GetTree(), 'c/')
 
   def testGetTreeStats(self):
     """Test getting tree with stats."""
@@ -165,8 +178,9 @@ class TestCoveredDir(unittest.TestCase):
     d.stats_by_group['foo'] = croc.CoverageStats(
         lines_executable=33, lines_instrumented=22, lines_covered=11)
     # 'bar' group is skipped because it has no executable lines
-    self.assertEqual(d.GetTree(),
-        '/a/b/c/                          all:20/30/50   foo:11/22/33')
+    self.assertEqual(
+        d.GetTree(),
+        'c/                               all:20/30/50   foo:11/22/33')
 
   def testGetTreeSubdir(self):
     """Test getting tree with subdirs."""
@@ -175,13 +189,13 @@ class TestCoveredDir(unittest.TestCase):
     d3 = self.cov_dir = croc.CoveredDir('/a/c')
     d4 = self.cov_dir = croc.CoveredDir('/a/b/d')
     d5 = self.cov_dir = croc.CoveredDir('/a/b/e')
-    d1.subdirs = {'/a/b':d2, '/a/c':d3}
-    d2.subdirs = {'/a/b/d':d4, '/a/b/e':d5}
-    self.assertEqual(d1.GetTree(),
-                     '/a/\n  /a/b/\n    /a/b/d/\n    /a/b/e/\n  /a/c/')
+    d1.subdirs = {'/a/b': d2, '/a/c': d3}
+    d2.subdirs = {'/a/b/d': d4, '/a/b/e': d5}
+    self.assertEqual(d1.GetTree(), 'a/\n  b/\n    d/\n    e/\n  c/')
 
 #------------------------------------------------------------------------------
 
+
 class TestCoverage(unittest.TestCase):
   """Tests for croc.Coverage."""
 
@@ -197,8 +211,24 @@ class TestCoverage(unittest.TestCase):
     self.mock_walk_calls.append(src_dir)
     return self.mock_walk_return
 
+  def MockScanFile(self, filename, language):
+    """Mock for croc_scan.ScanFile().
+
+    Args:
+      filename: Path to file to scan.
+      language: Language for file.
+
+    Returns:
+      A list of executable lines.
+    """
+    self.mock_scan_calls.append([filename, language])
+    if filename in self.mock_scan_return:
+      return self.mock_scan_return[filename]
+    else:
+      return self.mock_scan_return['default']
+
   def setUp(self):
-    """Per-test setup"""
+    """Per-test setup."""
 
     # Empty coverage object
     self.cov = croc.Coverage()
@@ -207,26 +237,25 @@ class TestCoverage(unittest.TestCase):
     self.cov_minimal = croc.Coverage()
     self.cov_minimal.AddRoot('/src')
     self.cov_minimal.AddRoot('c:\\source')
-    self.cov_minimal.AddRule('^#/', include=1, group='my')
+    self.cov_minimal.AddRule('^_/', include=1, group='my')
     self.cov_minimal.AddRule('.*\\.c$', language='C')
-    self.cov_minimal.AddRule('.*\\.c##$', language='C##') # sharper than thou
+    self.cov_minimal.AddRule('.*\\.c##$', language='C##')  # sharper than thou
 
     # Data for MockWalk()
     self.mock_walk_calls = []
     self.mock_walk_return = []
 
+    # Data for MockScanFile()
+    self.mock_scan_calls = []
+    self.mock_scan_return = {'default': [1]}
+
   def testInit(self):
     """Test init."""
     c = self.cov
     self.assertEqual(c.files, {})
     self.assertEqual(c.root_dirs, [])
     self.assertEqual(c.print_stats, [])
-
-    # Check for the initial subdir rule
-    self.assertEqual(len(c.rules), 1)
-    r0 = c.rules[0]
-    self.assertEqual(r0[0].pattern, '.*/$')
-    self.assertEqual(r0[1:], [None, None, 'subdir'])
+    self.assertEqual(c.rules, [])
 
   def testAddRoot(self):
     """Test AddRoot() and CleanupFilename()."""
@@ -255,91 +284,108 @@ class TestCoverage(unittest.TestCase):
                      c.CleanupFilename(os.path.abspath('../../a/b/c')))
 
     # Replace alt roots
-    c.AddRoot('foo', '#')
-    self.assertEqual(c.CleanupFilename('foo'), '#')
-    self.assertEqual(c.CleanupFilename('foo/bar/baz'), '#/bar/baz')
+    c.AddRoot('foo')
+    self.assertEqual(c.CleanupFilename('foo'), '_')
+    self.assertEqual(c.CleanupFilename('foo/bar/baz'), '_/bar/baz')
     self.assertEqual(c.CleanupFilename('aaa/foo'), 'aaa/foo')
 
     # Alt root replacement is applied for all roots
-    c.AddRoot('foo/bar', '#B')
-    self.assertEqual(c.CleanupFilename('foo/bar/baz'), '#B/baz')
+    c.AddRoot('foo/bar', '_B')
+    self.assertEqual(c.CleanupFilename('foo/bar/baz'), '_B/baz')
 
     # Can use previously defined roots in cleanup
-    c.AddRoot('#/nom/nom/nom', '#CANHAS')
+    c.AddRoot('_/nom/nom/nom', '_CANHAS')
     self.assertEqual(c.CleanupFilename('foo/nom/nom/nom/cheezburger'),
-                     '#CANHAS/cheezburger')
+                     '_CANHAS/cheezburger')
 
     # Verify roots starting with UNC paths or drive letters work, and that
     # more than one root can point to the same alt_name
-    c.AddRoot('/usr/local/foo', '#FOO')
-    c.AddRoot('D:\\my\\foo', '#FOO')
-    self.assertEqual(c.CleanupFilename('/usr/local/foo/a/b'), '#FOO/a/b')
-    self.assertEqual(c.CleanupFilename('D:\\my\\foo\\c\\d'), '#FOO/c/d')
+    c.AddRoot('/usr/local/foo', '_FOO')
+    c.AddRoot('D:\\my\\foo', '_FOO')
+    self.assertEqual(c.CleanupFilename('/usr/local/foo/a/b'), '_FOO/a/b')
+    self.assertEqual(c.CleanupFilename('D:\\my\\foo\\c\\d'), '_FOO/c/d')
+
+    # Cannot specify a blank alt_name
+    self.assertRaises(ValueError, c.AddRoot, 'some_dir', '')
 
   def testAddRule(self):
     """Test AddRule() and ClassifyFile()."""
     c = self.cov
 
     # With only the default rule, nothing gets kept
-    self.assertEqual(c.ClassifyFile('#/src/'), (None, None))
-    self.assertEqual(c.ClassifyFile('#/src/a.c'), (None, None))
+    self.assertEqual(c.ClassifyFile('_/src/'), {})
+    self.assertEqual(c.ClassifyFile('_/src/a.c'), {})
 
     # Add rules to include a tree and set a default group
-    c.AddRule('^#/src/', include=1, group='source')
-    # Now the subdir matches, but source doesn't, since no languages are
-    # defined yet
-    self.assertEqual(c.ClassifyFile('#/src/'), ('source', 'subdir'))
-    self.assertEqual(c.ClassifyFile('#/notsrc/'), (None, None))
-    self.assertEqual(c.ClassifyFile('#/src/a.c'), (None, None))
+    c.AddRule('^_/src/', include=1, group='source')
+    self.assertEqual(c.ClassifyFile('_/src/'),
+                     {'include': 1, 'group': 'source'})
+    self.assertEqual(c.ClassifyFile('_/notsrc/'), {})
+    self.assertEqual(c.ClassifyFile('_/src/a.c'),
+                     {'include': 1, 'group': 'source'})
 
     # Define some languages and groups
     c.AddRule('.*\\.(c|h)$', language='C')
     c.AddRule('.*\\.py$', language='Python')
     c.AddRule('.*_test\\.', group='test')
-    self.assertEqual(c.ClassifyFile('#/src/a.c'), ('source', 'C'))
-    self.assertEqual(c.ClassifyFile('#/src/a.h'), ('source', 'C'))
-    self.assertEqual(c.ClassifyFile('#/src/a.cpp'), (None, None))
-    self.assertEqual(c.ClassifyFile('#/src/a_test.c'), ('test', 'C'))
-    self.assertEqual(c.ClassifyFile('#/src/test_a.c'), ('source', 'C'))
-    self.assertEqual(c.ClassifyFile('#/src/foo/bar.py'), ('source', 'Python'))
-    self.assertEqual(c.ClassifyFile('#/src/test.py'), ('source', 'Python'))
+    self.assertEqual(c.ClassifyFile('_/src/a.c'),
+                     {'include': 1, 'group': 'source', 'language': 'C'})
+    self.assertEqual(c.ClassifyFile('_/src/a.h'),
+                     {'include': 1, 'group': 'source', 'language': 'C'})
+    self.assertEqual(c.ClassifyFile('_/src/a.cpp'),
+                     {'include': 1, 'group': 'source'})
+    self.assertEqual(c.ClassifyFile('_/src/a_test.c'),
+                     {'include': 1, 'group': 'test', 'language': 'C'})
+    self.assertEqual(c.ClassifyFile('_/src/test_a.c'),
+                     {'include': 1, 'group': 'source', 'language': 'C'})
+    self.assertEqual(c.ClassifyFile('_/src/foo/bar.py'),
+                     {'include': 1, 'group': 'source', 'language': 'Python'})
+    self.assertEqual(c.ClassifyFile('_/src/test.py'),
+                     {'include': 1, 'group': 'source', 'language': 'Python'})
 
     # Exclude a path (for example, anything in a build output dir)
     c.AddRule('.*/build/', include=0)
     # But add back in a dir which matched the above rule but isn't a build
     # output dir
-    c.AddRule('#/src/tools/build/', include=1)
-    self.assertEqual(c.ClassifyFile('#/src/build.c'), ('source', 'C'))
-    self.assertEqual(c.ClassifyFile('#/src/build/'), (None, None))
-    self.assertEqual(c.ClassifyFile('#/src/build/a.c'), (None, None))
-    self.assertEqual(c.ClassifyFile('#/src/tools/build/'), ('source', 'subdir'))
-    self.assertEqual(c.ClassifyFile('#/src/tools/build/t.c'), ('source', 'C'))
+    c.AddRule('_/src/tools/build/', include=1)
+    self.assertEqual(c.ClassifyFile('_/src/build.c').get('include'), 1)
+    self.assertEqual(c.ClassifyFile('_/src/build/').get('include'), 0)
+    self.assertEqual(c.ClassifyFile('_/src/build/a.c').get('include'), 0)
+    self.assertEqual(c.ClassifyFile('_/src/tools/build/').get('include'), 1)
+    self.assertEqual(c.ClassifyFile('_/src/tools/build/t.c').get('include'), 1)
 
   def testGetCoveredFile(self):
     """Test GetCoveredFile()."""
     c = self.cov_minimal
 
     # Not currently any covered files
-    self.assertEqual(c.GetCoveredFile('#/a.c'), None)
+    self.assertEqual(c.GetCoveredFile('_/a.c'), None)
 
     # Add some files
-    a_c = c.GetCoveredFile('#/a.c', add=True)
-    b_c = c.GetCoveredFile('#/b.c##', add=True)
-    self.assertEqual(a_c.filename, '#/a.c')
-    self.assertEqual(a_c.group, 'my')
-    self.assertEqual(a_c.language, 'C')
-    self.assertEqual(b_c.filename, '#/b.c##')
-    self.assertEqual(b_c.group, 'my')
-    self.assertEqual(b_c.language, 'C##')
+    a_c = c.GetCoveredFile('_/a.c', add=True)
+    b_c = c.GetCoveredFile('_/b.c##', add=True)
+    self.assertEqual(a_c.filename, '_/a.c')
+    self.assertEqual(a_c.attrs, {'include': 1, 'group': 'my', 'language': 'C'})
+    self.assertEqual(b_c.filename, '_/b.c##')
+    self.assertEqual(b_c.attrs,
+                     {'include': 1, 'group': 'my', 'language': 'C##'})
 
     # Specifying the same filename should return the existing object
-    self.assertEqual(c.GetCoveredFile('#/a.c'), a_c)
-    self.assertEqual(c.GetCoveredFile('#/a.c', add=True), a_c)
+    self.assertEqual(c.GetCoveredFile('_/a.c'), a_c)
+    self.assertEqual(c.GetCoveredFile('_/a.c', add=True), a_c)
 
     # Filenames get cleaned on the way in, as do root paths
     self.assertEqual(c.GetCoveredFile('/src/a.c'), a_c)
     self.assertEqual(c.GetCoveredFile('c:\\source\\a.c'), a_c)
 
+    # TODO: Make sure that covered files require language, group, and include
+    # (since that checking is now done in GetCoveredFile() rather than
+    # ClassifyFile())
+
+  def testRemoveCoveredFile(self):
+    """Test RemoveCoveredFile()."""
+    # TODO: TEST ME!
+
   def testParseLcov(self):
     """Test ParseLcovData()."""
     c = self.cov_minimal
@@ -350,7 +396,7 @@ class TestCoverage(unittest.TestCase):
         'SF:/src/a.c',
         'DA:10,1',
         'DA:11,0',
-        'DA:12,1   \n', # Trailing whitespace should get stripped
+        'DA:12,1   \n',  # Trailing whitespace should get stripped
         'end_of_record',
         # File we should ignore
         'SF:/not_src/a.c',
@@ -368,13 +414,16 @@ class TestCoverage(unittest.TestCase):
         'SF:/src/b.c',
         'DA:50,0',
         'end_of_record',
+        # Empty file (instrumented but no executable lines)
+        'SF:c:\\source\\c.c',
+        'end_of_record',
     ])
 
-    # We should know about two files
-    self.assertEqual(sorted(c.files), ['#/a.c', '#/b.c'])
+    # We should know about three files
+    self.assertEqual(sorted(c.files), ['_/a.c', '_/b.c', '_/c.c'])
 
     # Check expected contents
-    a_c = c.GetCoveredFile('#/a.c')
+    a_c = c.GetCoveredFile('_/a.c')
     self.assertEqual(a_c.lines, {10: 1, 11: 0, 12: 1, 30: 1})
     self.assertEqual(a_c.stats, {
         'files_executable': 1,
@@ -384,7 +433,9 @@ class TestCoverage(unittest.TestCase):
         'lines_executable': 4,
         'lines_covered': 3,
     })
-    b_c = c.GetCoveredFile('#/b.c')
+    self.assertEqual(a_c.in_lcov, True)
+
+    b_c = c.GetCoveredFile('_/b.c')
     self.assertEqual(b_c.lines, {50: 0})
     self.assertEqual(b_c.stats, {
         'files_executable': 1,
@@ -393,6 +444,23 @@ class TestCoverage(unittest.TestCase):
         'lines_executable': 1,
         'lines_covered': 0,
     })
+    self.assertEqual(b_c.in_lcov, True)
+
+    c_c = c.GetCoveredFile('_/c.c')
+    self.assertEqual(c_c.lines, {})
+    self.assertEqual(c_c.stats, {
+        'files_executable': 1,
+        'files_instrumented': 1,
+        'lines_instrumented': 0,
+        'lines_executable': 0,
+        'lines_covered': 0,
+    })
+    self.assertEqual(c_c.in_lcov, True)
+
+    # TODO: Test that files are marked as instrumented if they come from lcov,
+    # even if they don't have any instrumented lines.  (and that in_lcov is set
+    # for those files - probably should set that via some method rather than
+    # directly...)
 
   def testGetStat(self):
     """Test GetStat() and PrintStat()."""
@@ -413,10 +481,10 @@ class TestCoverage(unittest.TestCase):
     }
 
     # Test missing stats and groups
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'nosuch')
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'baz')
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'foo', group='tests')
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'foo', group='nosuch')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'nosuch')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'baz')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'foo', group='tests')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'foo', group='nosuch')
 
     # Test returning defaults
     self.assertEqual(c.GetStat('nosuch', default=13), 13)
@@ -434,13 +502,13 @@ class TestCoverage(unittest.TestCase):
     self.assertEqual(c.GetStat('100.0 * count_a / count_b', group='tests'),
                      40.0)
     # Should catch eval errors
-    self.assertRaises(croc.CoverageStatError, c.GetStat, '100 / 0')
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'count_a -')
+    self.assertRaises(croc.CrocStatError, c.GetStat, '100 / 0')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'count_a -')
 
     # Test nested stats via S()
     self.assertEqual(c.GetStat('count_a - S("count_a", group="tests")'), 8)
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'S()')
-    self.assertRaises(croc.CoverageStatError, c.GetStat, 'S("nosuch")')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'S()')
+    self.assertRaises(croc.CrocStatError, c.GetStat, 'S("nosuch")')
 
     # Test PrintStat()
     # We won't see the first print, but at least verify it doesn't assert
@@ -476,14 +544,14 @@ GetStat('nosuch') = 42
     c.AddConfig("""{
         'roots' : [
           {'root' : '/foo'},
-          {'root' : '/bar', 'altname' : '#BAR'},
+          {'root' : '/bar', 'altname' : 'BAR'},
         ],
         'rules' : [
-          {'regexp' : '^#', 'group' : 'apple'},
+          {'regexp' : '^_/', 'group' : 'apple'},
           {'regexp' : 're2', 'include' : 1, 'language' : 'elvish'},
         ],
         'lcov_files' : ['a.lcov', 'b.lcov'],
-        'add_files' : ['/src', '#BAR/doo'],
+        'add_files' : ['/src', 'BAR/doo'],
         'print_stats' : [
           {'stat' : 'count_a'},
           {'stat' : 'count_b', 'group' : 'tests'},
@@ -492,8 +560,8 @@ GetStat('nosuch') = 42
     }""", lcov_queue=lcov_queue, addfiles_queue=addfiles_queue)
 
     self.assertEqual(lcov_queue, ['a.lcov', 'b.lcov'])
-    self.assertEqual(addfiles_queue, ['/src', '#BAR/doo'])
-    self.assertEqual(c.root_dirs, [['/foo', '#'], ['/bar', '#BAR']])
+    self.assertEqual(addfiles_queue, ['/src', 'BAR/doo'])
+    self.assertEqual(c.root_dirs, [['/foo', '_'], ['/bar', 'BAR']])
     self.assertEqual(c.print_stats, [
         {'stat': 'count_a'},
         {'stat': 'count_b', 'group': 'tests'},
@@ -501,30 +569,35 @@ GetStat('nosuch') = 42
     # Convert compiled re's back to patterns for comparison
     rules = [[r[0].pattern] + r[1:] for r in c.rules]
     self.assertEqual(rules, [
-        ['.*/$', None, None, 'subdir'],
-        ['^#', None, 'apple', None],
-        ['re2', 1, None, 'elvish'],
+        ['^_/', {'group': 'apple'}],
+        ['re2', {'include': 1, 'language': 'elvish'}],
     ])
 
   def testAddFilesSimple(self):
     """Test AddFiles() simple call."""
     c = self.cov_minimal
     c.add_files_walk = self.MockWalk
+    c.scan_file = self.MockScanFile
+
     c.AddFiles('/a/b/c')
     self.assertEqual(self.mock_walk_calls, ['/a/b/c'])
+    self.assertEqual(self.mock_scan_calls, [])
     self.assertEqual(c.files, {})
 
   def testAddFilesRootMap(self):
     """Test AddFiles() with root mappings."""
     c = self.cov_minimal
     c.add_files_walk = self.MockWalk
-    c.AddRoot('#/subdir', '#SUBDIR')
+    c.scan_file = self.MockScanFile
+
+    c.AddRoot('_/subdir', 'SUBDIR')
 
-    # AddFiles() should replace the '#SUBDIR' alt_name, then match both
-    # possible roots for the '#' alt_name.
-    c.AddFiles('#SUBDIR/foo')
+    # AddFiles() should replace the 'SUBDIR' alt_name, then match both
+    # possible roots for the '_' alt_name.
+    c.AddFiles('SUBDIR/foo')
     self.assertEqual(self.mock_walk_calls,
                      ['/src/subdir/foo', 'c:/source/subdir/foo'])
+    self.assertEqual(self.mock_scan_calls, [])
     self.assertEqual(c.files, {})
 
   def testAddFilesNonEmpty(self):
@@ -532,16 +605,20 @@ GetStat('nosuch') = 42
 
     c = self.cov_minimal
     c.add_files_walk = self.MockWalk
+    c.scan_file = self.MockScanFile
 
     # Add a rule to exclude a subdir
-    c.AddRule('^#/proj1/excluded/', include=0)
+    c.AddRule('^_/proj1/excluded/', include=0)
 
-    # Set data for mock walk
+    # Add a rule to exclude adding some fiels
+    c.AddRule('.*noscan.c$', add_if_missing=0)
+
+    # Set data for mock walk and scan
     self.mock_walk_return = [
         [
             '/src/proj1',
             ['excluded', 'subdir'],
-            ['a.c', 'no.f', 'yes.c'],
+            ['a.c', 'no.f', 'yes.c', 'noexe.c', 'bob_noscan.c'],
         ],
         [
             '/src/proj1/subdir',
@@ -550,15 +627,24 @@ GetStat('nosuch') = 42
         ],
     ]
 
+    # Add a file with no executable lines; it should be scanned but not added
+    self.mock_scan_return['/src/proj1/noexe.c'] = []
+
     c.AddFiles('/src/proj1')
 
     self.assertEqual(self.mock_walk_calls, ['/src/proj1'])
+    self.assertEqual(self.mock_scan_calls, [
+        ['/src/proj1/a.c', 'C'],
+        ['/src/proj1/yes.c', 'C'],
+        ['/src/proj1/noexe.c', 'C'],
+        ['/src/proj1/subdir/cherry.c', 'C'],
+    ])
 
     # Include files from the main dir and subdir
     self.assertEqual(sorted(c.files), [
-        '#/proj1/a.c',
-        '#/proj1/subdir/cherry.c',
-        '#/proj1/yes.c'])
+        '_/proj1/a.c',
+        '_/proj1/subdir/cherry.c',
+        '_/proj1/yes.c'])
 
     # Excluded dir should have been pruned from the mock walk data dirnames.
     # In the real os.walk() call this prunes the walk.
@@ -568,9 +654,6 @@ GetStat('nosuch') = 42
     """Test UpdateTreeStats()."""
 
     c = self.cov_minimal
-
-
-
     c.AddRule('.*_test', group='test')
 
     # Fill the files list
@@ -593,7 +676,7 @@ GetStat('nosuch') = 42
     t = c.tree
     self.assertEqual(t.dirpath, '')
     self.assertEqual(sorted(t.files), [])
-    self.assertEqual(sorted(t.subdirs), ['#'])
+    self.assertEqual(sorted(t.subdirs), ['_'])
     self.assertEqual(t.stats_by_group, {
         'all': {
             'files_covered': 3,
@@ -621,8 +704,8 @@ GetStat('nosuch') = 42
         },
     })
 
-    t = t.subdirs['#']
-    self.assertEqual(t.dirpath, '#')
+    t = t.subdirs['_']
+    self.assertEqual(t.dirpath, '_')
     self.assertEqual(sorted(t.files), ['a.c', 'a_test.c'])
     self.assertEqual(sorted(t.subdirs), ['foo'])
     self.assertEqual(t.stats_by_group, {
@@ -653,7 +736,7 @@ GetStat('nosuch') = 42
     })
 
     t = t.subdirs['foo']
-    self.assertEqual(t.dirpath, 'foo')
+    self.assertEqual(t.dirpath, '_/foo')
     self.assertEqual(sorted(t.files), ['b.c', 'b_test.c'])
     self.assertEqual(sorted(t.subdirs), [])
     self.assertEqual(t.stats_by_group, {