Adding section to parse the lcov data and post the new

coverage dataset to the code coverage dashboard.
Review URL: http://codereview.chromium.org/8693

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@4253 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 179 insertions, 2 deletions

diff --git a/tools/code_coverage/process_coverage.py b/tools/code_coverage/process_coverage.py
index e52fd15..e42e95b 100644
--- a/tools/code_coverage/process_coverage.py
+++ b/tools/code_coverage/process_coverage.py
@@ -39,9 +39,11 @@ TODO(niranjan): Add usage information here
 import optparse
 import os
 import shutil
+import subprocess
 import sys
 import tempfile
-import subprocess
+import time
+import urllib2
 
 
 # These are source files that were generated during compile time. We want to
@@ -52,6 +54,13 @@ win32_srcs_exclude = ['parse.y',
                       'cssgrammar.cpp',
                       'csspropertynames.gperf']
 
+# Number of lines of a new coverage data set 
+# to send at a time to the dashboard.
+POST_CHUNK_SIZE = 50
+
+# Number of post request failures to allow before exiting.
+MAX_FAILURES = 5
+
 def CleanPathNames(dir):
   """Clean the pathnames of the HTML generated by genhtml.
 
@@ -203,6 +212,164 @@ def CleanWin32Lcov(lcov_path, src_root):
   shutil.move(tmpfile_name, lcov_path)
 
 
+def ParseCoverageDataForDashboard(lcov_path):
+  """Parse code coverage data into coverage results per source node.
+
+  Use lcov and linecount data to create a map of source nodes to
+  corresponding total and tested line counts.
+
+  Args:
+    lcov_path: File path to lcov coverage data.
+ 
+  Returns:
+    List of strings with comma separated source node and coverage.
+  """
+  results = {}
+  linecount_path = lcov_path + '.csv'
+  assert(os.path.exists(linecount_path),
+         'linecount csv does not exist at: %s' % linecount_path)
+  csv_file = open(linecount_path, 'r')
+  linecounts = csv_file.readlines()
+  csv_file.close()
+  lcov_file = open(lcov_path, 'r')
+  srcfile_index = 0
+  for line in lcov_file:
+    line = line.strip()
+
+    # Set the current srcfile name for a new src file declaration.
+    if line[:len('SF:')] == 'SF:':
+      instrumented_set = {}
+      executed_set = {}
+      srcfile_name = line[len('SF:'):]
+    
+    # Mark coverage data points hashlist style for the current src file.
+    if line[:len('DA:')] == 'DA:':
+      line_info = line[len('DA:'):].split(',')
+      assert(len(line_info) == 2, 'DA: line format unexpected - %s' % line)
+      (line_num, line_was_executed) = line_info
+      instrumented_set[line_num] = True
+      # line_was_executed is '0' or '1'
+      if int(line_was_executed):
+        executed_set[line_num] = True
+    
+    # Update results for the current src file at record end.
+    if line == 'end_of_record':
+      instrumented = len(instrumented_set.keys())
+      executed = len(executed_set.keys())
+      parent_directory = srcfile_name[:srcfile_name.rfind('/') + 1]
+      linecount_point = linecounts[srcfile_index].strip().split(',')
+      assert(len(linecount_point) == 2, 
+             'lintcount format unexpected - %s' % linecounts[srcfile_index])
+      (linecount_path, linecount_count) = linecount_point
+      srcfile_index += 1
+      
+      # Sanity check that path names in the lcov and linecount are lined up.
+      if linecount_path[-10:] != srcfile_name[-10:]:
+        print 'NAME MISMATCH: %s :: %s' % (srcfile_name, linecount_path)
+      if instrumented > int(linecount_count):
+        linecount_count = instrumented
+
+      # Keep counts the same way that it is done in the genhtml utility.
+      # Count the coverage of a file towards the file,
+      # the parent directory, and the source root.
+      AddResults(results, srcfile_name, int(linecount_count), executed)
+      AddResults(results, parent_directory, int(linecount_count), executed)
+      AddResults(results, '/', instrumented, executed)
+
+  lcov_file.close()
+  keys = results.keys()
+  keys.sort()
+  # The first key (sorted) will be the base directory '/'
+  # but its full path may be '/mnt/chrome_src/src/'
+  # using this offset will ignore the part '/mnt/chrome_src/src'.
+  # Offset is the last '/' that isn't the last character for the 
+  # first directory name in results (position 1 in keys).
+  offset = len(keys[1][:keys[1][:-1].rfind('/')])
+  lines = []
+  for key in keys:
+    if len(key) > offset:
+      node_path = key[offset:]
+    else:
+      node_path = key
+    (total, covered) = results[key]
+    percent = float(covered) * 100 / total
+    lines.append('%s,%.2f' % (node_path, percent))
+  return lines
+
+
+def AddResults(results, location, lines_total, lines_executed):
+  """Add resulting line tallies to a location's total.
+  
+  Args:
+    results: Map of node location to corresponding coverage data.
+    location: Source node string.
+    lines_total: Number of lines to add to the total count for this node.
+    lines_executed: Number of lines to add to the executed count for this node.
+  """
+  if results.has_key(location):
+    (i, e) = results[location]
+    results[location] = (i + lines_total, e + lines_executed)
+  else:
+    results[location] = (lines_total, lines_executed)
+
+
+def PostResultsToDashboard(lcov_path, results, post_url):
+  """Post coverage results to coverage dashboard.
+  
+  Args:
+    lcov_path: File path for lcov data in the expected format:
+        <project>_<platform>_<cl#>.coverage.lcov
+    results: string list in the appropriate posting format.
+  """
+  project_platform_cl = lcov_path.split('.')[0].split('_')
+  assert(len(project_platform_cl) == 3,
+         'lcov_path not in expected format: %s' % lcov_path)
+  (project, platform, cl_string) = project_platform_cl
+  project_name = '%s-%s' % (project, platform)
+  url = '%s/newdata.do?project=%s&cl=%s' % (post_url, project_name, cl_string)
+
+  # Send POSTs of POST_CHUNK_SIZE lines of the result set until
+  # there is no more data and last_loop is set to True.
+  last_loop = False
+  cur_line = 0
+  while not last_loop:
+    body = '\n'.join(results[cur_line:cur_line + POST_CHUNK_SIZE])
+    cur_line += POST_CHUNK_SIZE
+    last_loop = (cur_line >= len(results))
+    req = urllib2.Request('%s&last=%s' % (url, str(last_loop)), body)
+    req.add_header('Content-Type', 'text/plain')
+    SendPost(req)
+
+
+# Global counter for the current number of request failures.
+num_fails = 0
+
+def SendPost(req):
+  """Execute a post request and retry for up to MAX_FAILURES.
+  
+  Args:
+    req: A urllib2 request object.
+
+  Raises:
+    URLError: If urlopen throws after too many retries.
+    HTTPError: If urlopen throws after too many retries.
+  """
+  global num_fails
+  try:
+    urllib2.urlopen(req)
+    # Reset failure count.
+    num_fails = 0
+  except (urllib2.URLError, urllib2.HTTPError):
+    num_fails += 1
+    if num_fails < MAX_FAILURES:
+      print 'fail, retrying (%d)' % num_fails
+      time.sleep(5)
+      SendPost(req)
+    else:
+      print 'POST request exceeded allowed retries.'
+      raise
+
+
 def main():
   if sys.platform[:5] != 'linux': # Run this only on Linux
     print 'This script is supported only on Linux'
@@ -231,6 +398,11 @@ def main():
                     dest='lcov_path',
                     default=None,
                     help='Location of the LCOV file to process')
+  parser.add_option('-u',
+                    '--post_url',
+                    dest='post_url',
+                    default=None,
+                    help='Base URL of the coverage dashboard')
   (options, args) = parser.parse_args()
 
   if options.platform == None:
@@ -241,6 +413,8 @@ def main():
     parser.error('Source directory not specified')
   if options.dash_root == None:
     parser.error('Dashboard root not specified')
+  if options.post_url == None:
+    parser.error('Post URL not specified')
   if options.platform == 'win32':
     CleanWin32Lcov(options.lcov_path, options.src_dir)
     percent = GenerateHtml(options.lcov_path, options.dash_root)
@@ -254,8 +428,11 @@ def main():
   else:
     print 'Unsupported platform'
     os.exit(1)
+  
+  # Prep coverage results for dashboard and post new set.
+  parsed_data = ParseCoverageDataForDashboard(options.lcov_path)
+  PostResultsToDashboard(options.lcov_path, parsed_data, options.post_url)
 
 
 if __name__ == '__main__':
   main()
-