[Findit] Support sending cookies for http requests.

BUG=
R=aarya@google.com, mbarbella@chromium.org

Review URL: https://codereview.chromium.org/465403004

Cr-Commit-Position: refs/heads/master@{#290064}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@290064 0039d316-1c4b-4281-b951-d872f2087c98

11 files changed, 118 insertions, 47 deletions

diff --git a/tools/findit/blame.py b/tools/findit/blame.py
index 9d6e989..63069749 100644
--- a/tools/findit/blame.py
+++ b/tools/findit/blame.py
@@ -4,7 +4,7 @@
 
 from threading import Lock, Thread
 
-import utils
+from common import utils
 
 
 class Blame(object):
diff --git a/tools/findit/chromium_deps.py b/tools/findit/chromium_deps.py
index 6bad2c4..0b187f588 100644
--- a/tools/findit/chromium_deps.py
+++ b/tools/findit/chromium_deps.py
@@ -3,8 +3,8 @@
 # found in the LICENSE file.
 
 import base64
-import https
-import utils
+
+from common import utils
 
 
 DEPS_FILE_URL_SVN = 'https://src.chromium.org/chrome/trunk/src/DEPS?p=%s'
@@ -78,7 +78,8 @@ def _GetContentOfDEPS(chromium_revision):
     url = DEPS_FILE_URL_GIT
   else:
     url = DEPS_FILE_URL_SVN
-  return https.SendRequest(url % chromium_revision)
+  _, content = utils.GetHttpClient().Get(url % chromium_revision, timeout=60)
+  return content
 
 
 def GetChromiumComponents(chromium_revision,
@@ -202,9 +203,3 @@ def GetChromiumComponentRange(old_revision,
     }
 
   return components
-
-
-if __name__ == '__main__':
-  import json
-  print json.dumps(GetChromiumComponents(
-      'b4b1aea80b25a3b2f7952c9d95585e880421ef2b'), sort_keys=True, indent=2)
diff --git a/tools/findit/chromium_deps_unittest.py b/tools/findit/chromium_deps_unittest.py
index b051f46..a26ac9c 100644
--- a/tools/findit/chromium_deps_unittest.py
+++ b/tools/findit/chromium_deps_unittest.py
@@ -84,13 +84,13 @@ deps_os = {
     self.assertEqual(expected_results, components)
 
   def testGetChromiumComponentRange(self):
-    chromium_revision1 = '283296'
-    webkit_revision1 = '178200'
+    chromium_revision1 = '283200'
+    webkit_revision1 = '178084'
     breakpad_revision1 = '1345'
     liblouis_commit_hashcode1 = '3c2daee56250162e5a75830871601d74328d39f5'
 
-    chromium_revision2 = '283200'
-    webkit_revision2 = '178084'
+    chromium_revision2 = '283296'
+    webkit_revision2 = '178200'
     breakpad_revision2 = '1345'
     liblouis_commit_hashcode2 = '3c2daee56250162e5a75830871601d74328d39f5'
 
@@ -106,39 +106,39 @@ deps_os = {
 
     expected_results = {
         'src/breakpad/src/': {
-            'old_revision': breakpad_revision2,
+            'old_revision': breakpad_revision1,
             'name': 'breakpad',
             'repository': 'http://google-breakpad.googlecode.com/svn/trunk/src',
             'rolled': False,
-            'new_revision': breakpad_revision1,
+            'new_revision': breakpad_revision2,
             'path': 'src/breakpad/src/',
             'repository_type': 'svn'
         },
         'src/third_party/liblouis/src/': {
-            'old_revision': liblouis_commit_hashcode2,
+            'old_revision': liblouis_commit_hashcode1,
             'name': 'liblouis',
             'repository':
                 'https://chromium.googlesource.com/external/liblouis.git',
             'rolled': False,
-            'new_revision': liblouis_commit_hashcode1,
+            'new_revision': liblouis_commit_hashcode2,
             'path': 'src/third_party/liblouis/src/',
             'repository_type': 'git'
         },
         'src/': {
-            'old_revision': chromium_revision2,
+            'old_revision': chromium_revision1,
             'name': 'chromium',
             'repository': 'https://src.chromium.org/chrome/trunk',
             'rolled': True,
-            'new_revision': chromium_revision1,
+            'new_revision': chromium_revision2,
             'path': 'src/',
             'repository_type': 'svn'
         },
         'src/third_party/WebKit/': {
-            'old_revision': webkit_revision2,
+            'old_revision': webkit_revision1,
             'name': 'blink',
             'repository': 'http://src.chromium.org/blink/trunk',
             'rolled': True,
-            'new_revision': webkit_revision1,
+            'new_revision': webkit_revision2,
             'path': 'src/third_party/WebKit/',
             'repository_type': 'svn'
         }
@@ -148,3 +148,7 @@ deps_os = {
         chromium_revision1, chromium_revision2,
         deps_file_downloader=_GetContentOfDEPS)
     self.assertEqual(expected_results, components)
+
+  def testGetSvnRevision(self):
+    deps = chromium_deps.GetChromiumComponents(284750)
+    self.assertTrue(isinstance(deps, dict))
diff --git a/tools/findit/common/__init__.py b/tools/findit/common/__init__.py
new file mode 100644
index 0000000..31f0497
--- /dev/null
+++ b/tools/findit/common/__init__.py
@@ -0,0 +1,4 @@
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
diff --git a/tools/findit/cacert.pem b/tools/findit/common/cacert.pem
index a4df056..a4df056 100644
--- a/tools/findit/cacert.pem
+++ b/tools/findit/common/cacert.pem
diff --git a/tools/findit/common/http_client.py b/tools/findit/common/http_client.py
new file mode 100644
index 0000000..82c358de
--- /dev/null
+++ b/tools/findit/common/http_client.py
@@ -0,0 +1,21 @@
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+class HttpClient(object):
+  """Represent a http client for sending request to a http[s] server.
+
+  If cookies need to be sent, they should be in a file pointed to by
+  COOKIE_FILE in the environment.
+  """
+
+  @staticmethod
+  def Get(url, params={}, timeout=None):
+    """Send a GET request to the given url with the given parameters.
+
+    Returns:
+      (status_code, data)
+      state_code: the http status code in the response.
+      data: the body of the response.
+    """
+    raise NotImplemented()
diff --git a/tools/findit/https.py b/tools/findit/common/http_client_local.py
index c382f0c..bf386f9 100644
--- a/tools/findit/https.py
+++ b/tools/findit/common/http_client_local.py
@@ -1,8 +1,9 @@
 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """
-Provides a utility function for https connections with certificate verification.
+A http client with support for https connections with certificate verification.
 
 The verification is based on http://tools.ietf.org/html/rfc6125#section-6.4.3
 and the code is from Lib/ssl.py in python3:
@@ -14,13 +15,17 @@ One use case is to download Chromium DEPS file in a secure way:
 Notice: python 2.7 or newer is required.
 """
 
+import cookielib
 import httplib
 import os
 import re
 import socket
 import ssl
+import urllib
 import urllib2
 
+import http_client
+
 
 _SCRIPT_DIR = os.path.dirname(__file__)
 _TRUSTED_ROOT_CERTS = os.path.join(_SCRIPT_DIR, 'cacert.pem')
@@ -171,25 +176,47 @@ class HTTPSHandler(urllib2.HTTPSHandler):
     return HTTPSConnection(host, **params)
 
 
-def SendRequest(https_url):
+def _SendRequest(url, timeout=None):
   """Send request to the given https url, and return the server response.
 
   Args:
-    https_url: The https url to send request to.
+    url: The https url to send request to.
 
   Returns:
-    A string that is the response from the server.
+    An integer: http code of the response.
+    A string: content of the response.
 
   Raises:
-    ValueError: Unexpected value is received during certificate verification.
     CertificateError: Certificate verification fails.
   """
-  if not https_url or not https_url.startswith('https://'):
-    raise ValueError('Not a https request for url %s.' % str(https_url))
+  if not url:
+    return None, None
+
+  handlers = []
+  if url.startswith('https://'):
+    # HTTPSHandler has to go first, because we don't want to send secure cookies
+    # to a man in the middle.
+    handlers.append(HTTPSHandler())
+
+
+  cookie_file = os.environ.get('COOKIE_FILE')
+  if cookie_file and os.path.exists(cookie_file):
+    handlers.append(
+        urllib2.HTTPCookieProcessor(cookielib.MozillaCookieJar(cookie_file)))
+
+  url_opener = urllib2.build_opener(*handlers)
+  if timeout is not None:
+    response = url_opener.open(url, timeout=timeout)
+  else:
+    response = url_opener.open(url)
+  return response.code, response.read()
 
-  url_opener = urllib2.build_opener(HTTPSHandler)
-  return url_opener.open(https_url).read()
 
+class HttpClientLocal(http_client.HttpClient):
+  """This http client is used locally in a workstation, GCE VMs, etc."""
 
-if __name__ == '__main__':
-  print SendRequest('https://src.chromium.org/chrome/trunk/src/DEPS')
+  @staticmethod
+  def Get(url, params={}, timeout=None):
+    if params:
+      url = '%s?%s' % (url, urllib.urlencode(params))
+    return _SendRequest(url, timeout=timeout)
diff --git a/tools/findit/common/http_client_local_unittest.py b/tools/findit/common/http_client_local_unittest.py
new file mode 100644
index 0000000..40d41fa
--- /dev/null
+++ b/tools/findit/common/http_client_local_unittest.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+from http_client_local import HttpClientLocal
+
+class HttpClientLocalTest(unittest.TestCase):
+
+  def testGetWithoutParameters(self):
+    code, deps = HttpClientLocal.Get(
+        'https://src.chromium.org/chrome/trunk/src/DEPS')
+    self.assertEqual(200, code)
+    self.assertTrue(isinstance(deps, str))
diff --git a/tools/findit/utils.py b/tools/findit/common/utils.py
index 4e8a0ab..363048c 100644
--- a/tools/findit/utils.py
+++ b/tools/findit/common/utils.py
@@ -5,6 +5,8 @@
 import re
 import sys
 
+from http_client_local import HttpClientLocal
+
 
 GIT_HASH_PATTERN = re.compile(r'^[0-9a-fA-F]{40}$')
 
@@ -22,3 +24,8 @@ def GetOSName(platform_name=sys.platform):
 
 def IsGitHash(revision):
   return GIT_HASH_PATTERN.match(str(revision))
+
+
+def GetHttpClient():
+  # TODO(stgao): return implementation for appengine when running on appengine.
+  return HttpClientLocal
diff --git a/tools/findit/crash_utils.py b/tools/findit/crash_utils.py
index c3cc1a5e..42a17ce 100644
--- a/tools/findit/crash_utils.py
+++ b/tools/findit/crash_utils.py
@@ -10,6 +10,7 @@ import os
 import time
 import urllib2
 
+from common import utils
 from result import Result
 
 
@@ -195,23 +196,19 @@ def GetDataFromURL(url, retries=10, sleep_time=0.1, timeout=5):
   Returns:
     None if the data retrieval fails, or the raw data.
   """
-  data = None
-  for i in range(retries):
+  count = 0
+  while True:
+    count += 1
     # Retrieves data from URL.
     try:
-      data = urllib2.urlopen(url, timeout=timeout)
-
-      # If retrieval is successful, return the data.
-      if data:
-        return data.read()
-
-    # If retrieval fails, try after sleep_time second.
-    except urllib2.URLError:
-      time.sleep(sleep_time)
-      continue
+      _, data = utils.GetHttpClient().Get(url)
+      return data
     except IOError:
-      time.sleep(sleep_time)
-      continue
+      if count < retries:
+        # If retrieval fails, try after sleep_time second.
+        time.sleep(sleep_time)
+      else:
+        break
 
   # Return None if it fails to read data from URL 'retries' times.
   return None
diff --git a/tools/findit/run_all_tests.py b/tools/findit/run_all_tests.py
index b38408e..230537a 100755
--- a/tools/findit/run_all_tests.py
+++ b/tools/findit/run_all_tests.py
@@ -7,6 +7,7 @@ import sys
 import unittest
 
 from chromium_deps_unittest import ChromiumDEPSTest
+from common.http_client_local_unittest import HttpClientLocalTest
 
 
 if __name__ == '__main__':