Extension docs server: APIDataSource

Implemented and tested the APIDataSource class to get info from the JSON APIs.

This will probably use some tools from the JSON schema compiler in the future.

Also made some changes to build_server.py.

BUG=131095

Review URL: https://chromiumcodereview.appspot.com/10546078

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@142802 0039d316-1c4b-4281-b951-d872f2087c98

12 files changed, 222 insertions, 77 deletions

diff --git a/chrome/common/extensions/docs/server2/PRESUBMIT.py b/chrome/common/extensions/docs/server2/PRESUBMIT.py
index b98649b..31cd75a 100644
--- a/chrome/common/extensions/docs/server2/PRESUBMIT.py
+++ b/chrome/common/extensions/docs/server2/PRESUBMIT.py
@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Presubmit script for changes affecting tools/json_schema_compiler/
+"""Presubmit script for changes affecting extensions docs server
 
 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
 for more details about the presubmit API built into gcl.
diff --git a/chrome/common/extensions/docs/server2/api_data_source.py b/chrome/common/extensions/docs/server2/api_data_source.py
new file mode 100644
index 0000000..ae55b2c
--- /dev/null
+++ b/chrome/common/extensions/docs/server2/api_data_source.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+
+import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
+import third_party.json_schema_compiler.model as model
+
+class APIDataSource(object):
+  """This class fetches and loads JSON APIs with the fetcher passed in with
+  |cache_builder|, so the APIs can be plugged into templates.
+  """
+  def __init__(self, cache_builder, base_paths):
+    self._cache = cache_builder.build(self._LoadAPI)
+    self._base_paths = base_paths
+
+  def _LoadAPI(self, api):
+    return json.loads(json_comment_eater.Nom(api))[0]
+
+  def __getitem__(self, key):
+    return self.get(key)
+
+  def get(self, key):
+    path, ext = os.path.splitext(key)
+    unix_name = model.UnixName(path)
+    json_path = unix_name + '.json'
+    for base_path in self._base_paths:
+      try:
+        return self._cache.get(base_path + '/' + json_path)
+      except:
+        pass
+    return None
diff --git a/chrome/common/extensions/docs/server2/api_data_source_test.py b/chrome/common/extensions/docs/server2/api_data_source_test.py
new file mode 100755
index 0000000..3724140
--- /dev/null
+++ b/chrome/common/extensions/docs/server2/api_data_source_test.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+import unittest
+
+from fetcher_cache import FetcherCache
+from local_fetcher import LocalFetcher
+from api_data_source import APIDataSource
+
+class APIDataSourceTest(unittest.TestCase):
+  def setUp(self):
+    self._base_path = os.path.join('test_data', 'api_data_source')
+
+  def _ReadLocalFile(self, filename):
+    with open(os.path.join(self._base_path, filename), 'r') as f:
+      return f.read()
+
+  def testSimple(self):
+    self._base_path = os.path.join(self._base_path, 'simple')
+    fetcher = LocalFetcher(self._base_path)
+    cache_builder = FetcherCache.Builder(fetcher, 0)
+    data_source = APIDataSource(cache_builder, ['./'])
+
+    # Take the dict out of the list.
+    expected = json.loads(self._ReadLocalFile('test_file.json'))[0]
+    self.assertEqual(expected, data_source['test_file'])
+    self.assertEqual(expected, data_source['testFile'])
+    self.assertEqual(expected, data_source['testFile.html'])
+
+    self.assertEqual(None, data_source['junk'])
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/chrome/common/extensions/docs/server2/build_server.py b/chrome/common/extensions/docs/server2/build_server.py
index db9d869..c9a0c53 100755
--- a/chrome/common/extensions/docs/server2/build_server.py
+++ b/chrome/common/extensions/docs/server2/build_server.py
@@ -12,20 +12,45 @@ import sys
 THIRD_PARTY_DIR = os.path.join(sys.path[0], os.pardir, os.pardir, os.pardir,
      os.pardir, os.pardir, 'third_party')
 LOCAL_THIRD_PARTY_DIR = os.path.join(sys.path[0], 'third_party')
+TOOLS_DIR = os.path.join(sys.path[0], os.pardir, os.pardir, os.pardir,
+     os.pardir, os.pardir, 'tools')
+
+SCHEMA_COMPILER_FILES = ['model.py']
+
+def MakeInit(path):
+  path = os.path.join(path, '__init__.py')
+  with open(os.path.join(path), 'w') as f:
+    os.utime(os.path.join(path), None)
+
+def CopyThirdParty(src, dest, files=None):
+  dest_path = os.path.join(LOCAL_THIRD_PARTY_DIR, dest)
+  if not files:
+    shutil.copytree(src, dest_path)
+    MakeInit(dest_path)
+    return
+  try:
+    os.makedirs(dest_path)
+  except:
+    pass
+  MakeInit(dest_path)
+  for filename in files:
+    shutil.copy(os.path.join(src, filename), os.path.join(dest_path, filename))
 
 def main():
   shutil.rmtree(LOCAL_THIRD_PARTY_DIR, True)
-  shutil.copytree(os.path.join(THIRD_PARTY_DIR, 'handlebar'),
-      os.path.join(LOCAL_THIRD_PARTY_DIR, 'handlebar'))
 
-  with open(os.path.join(LOCAL_THIRD_PARTY_DIR, '__init__.py'), 'w') as f:
-    os.utime(os.path.join(LOCAL_THIRD_PARTY_DIR, '__init__.py'), None)
+  CopyThirdParty(os.path.join(THIRD_PARTY_DIR, 'handlebar'), 'handlebar')
+  CopyThirdParty(os.path.join(TOOLS_DIR, 'json_schema_compiler'),
+                 'json_schema_compiler',
+                 SCHEMA_COMPILER_FILES)
+  CopyThirdParty(TOOLS_DIR, 'json_schema_compiler', ['json_comment_eater.py'])
+  MakeInit(LOCAL_THIRD_PARTY_DIR)
 
-  shutil.copy(os.path.join(LOCAL_THIRD_PARTY_DIR, '__init__.py'),
-      os.path.join(LOCAL_THIRD_PARTY_DIR, 'handlebar'))
-  with open(
-      os.path.join(LOCAL_THIRD_PARTY_DIR, 'handlebar/__init__.py'), 'a') as f:
-    f.write('\nfrom handlebar import Handlebar\n')
+  # To be able to use the Handlebar class we need this import in __init__.py.
+  with open(os.path.join(LOCAL_THIRD_PARTY_DIR,
+                         'handlebar',
+                         '__init__.py'), 'a') as f:
+    f.write('from handlebar import Handlebar\n')
 
 if __name__ == '__main__':
   main()
diff --git a/chrome/common/extensions/docs/server2/echo_handler.py b/chrome/common/extensions/docs/server2/echo_handler.py
index 78195cb..6ee0801 100755
--- a/chrome/common/extensions/docs/server2/echo_handler.py
+++ b/chrome/common/extensions/docs/server2/echo_handler.py
@@ -8,7 +8,7 @@ import os
 
 # Add the original server location to sys.path so we are able to import
 # modules from there.
-SERVER_PATH = 'chrome/common/extensions/docs/server2/'
+SERVER_PATH = 'chrome/common/extensions/docs/server2'
 if os.path.abspath(SERVER_PATH) not in sys.path:
   sys.path.append(os.path.abspath(SERVER_PATH))
 
@@ -19,15 +19,18 @@ import urlfetch
 from google.appengine.ext import webapp
 from google.appengine.ext.webapp.util import run_wsgi_app
 
+from api_data_source import APIDataSource
+from fetcher_cache import FetcherCache
 from local_fetcher import LocalFetcher
 from server_instance import ServerInstance
 from subversion_fetcher import SubversionFetcher
 from template_data_source import TemplateDataSource
 
-EXTENSIONS_PATH = 'chrome/common/extensions/'
-DOCS_PATH = 'docs/'
-PUBLIC_TEMPLATE_PATH = DOCS_PATH + 'template2/public/'
-PRIVATE_TEMPLATE_PATH = DOCS_PATH + 'template2/private/'
+EXTENSIONS_PATH = 'chrome/common/extensions'
+DOCS_PATH = 'docs'
+API_PATH = 'api'
+PUBLIC_TEMPLATE_PATH = DOCS_PATH + '/template2/public'
+PRIVATE_TEMPLATE_PATH = DOCS_PATH + '/template2/private'
 
 # Global cache of instances because the Server is recreated for every request.
 SERVER_INSTANCES = {}
@@ -43,11 +46,15 @@ class Server(webapp.RequestHandler):
     else:
       fetcher = SubversionFetcher(branch, EXTENSIONS_PATH, urlfetch)
       cache_timeout_seconds = 300
-    data_source = TemplateDataSource(
-        fetcher,
-        [PUBLIC_TEMPLATE_PATH, PRIVATE_TEMPLATE_PATH],
-        cache_timeout_seconds)
-    SERVER_INSTANCES[branch] = ServerInstance(data_source, fetcher)
+    cache_builder = FetcherCache.Builder(fetcher, cache_timeout_seconds)
+    template_data_source = TemplateDataSource(
+        cache_builder,
+        [PUBLIC_TEMPLATE_PATH, PRIVATE_TEMPLATE_PATH])
+    api_data_source = APIDataSource(cache_builder, [API_PATH])
+    SERVER_INSTANCES[branch] = ServerInstance(
+        api_data_source,
+        template_data_source,
+        fetcher)
     return SERVER_INSTANCES[branch]
 
   def _HandleRequest(self, path):
diff --git a/chrome/common/extensions/docs/server2/fetcher_cache.py b/chrome/common/extensions/docs/server2/fetcher_cache.py
new file mode 100644
index 0000000..dc3b12e
--- /dev/null
+++ b/chrome/common/extensions/docs/server2/fetcher_cache.py
@@ -0,0 +1,46 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import time
+
+class FetcherCache(object):
+  """A cache for fetcher objects.
+  """
+  class Builder(object):
+    """A class to build a fetcher cache.
+    """
+    def __init__(self, fetcher, timeout_seconds):
+      self._fetcher = fetcher
+      self._timeout_seconds = timeout_seconds
+
+    def build(self, populate_function):
+      return FetcherCache(self._fetcher,
+                          self._timeout_seconds,
+                          populate_function)
+
+  class _CacheEntry(object):
+    def __init__(self, cache_data, expiry):
+      self._cache_data = cache_data
+      self._expiry = expiry
+
+    def HasExpired(self):
+      return time.time() > self._expiry
+
+  def __init__(self, fetcher, timeout_seconds, populate_function):
+    self._fetcher = fetcher
+    self._timeout_seconds = timeout_seconds
+    self._populate_function = populate_function
+    self._cache = {}
+
+  def get(self, key):
+    if key in self._cache:
+      if self._cache[key].HasExpired():
+        self._cache.pop(key)
+      else:
+        return self._cache[key]._cache_data
+    cache_data = self._fetcher.FetchResource(key).content
+    self._cache[key] = self._CacheEntry(self._populate_function(cache_data),
+                                        time.time() + self._timeout_seconds)
+    return self._cache[key]._cache_data
diff --git a/chrome/common/extensions/docs/server2/server_instance.py b/chrome/common/extensions/docs/server2/server_instance.py
index 26e9624..e09fd01 100644
--- a/chrome/common/extensions/docs/server2/server_instance.py
+++ b/chrome/common/extensions/docs/server2/server_instance.py
@@ -11,14 +11,17 @@ class ServerInstance(object):
   """This class is used to hold a data source and fetcher for an instance of a
   server. Each new branch will get its own ServerInstance.
   """
-  def __init__(self, data_source, fetcher):
-    self._data_source = data_source
+  def __init__(self, api_data_source, template_data_source, fetcher):
+    self._api_data_source = api_data_source
+    self._template_data_source = template_data_source
     self._fetcher = fetcher
 
+
   def Run(self, path, request_handler):
     parts = path.split('/')
     filename = parts[-1]
-    content = self._data_source.Render(filename, '{"test": "Hello"}')
+    content = self._template_data_source.Render(filename,
+                                                self._api_data_source[filename])
     if not content:
       logging.info('Template not found for: ' + filename)
       try:
diff --git a/chrome/common/extensions/docs/server2/subversion_fetcher.py b/chrome/common/extensions/docs/server2/subversion_fetcher.py
index 9a6179d..84ffb04 100644
--- a/chrome/common/extensions/docs/server2/subversion_fetcher.py
+++ b/chrome/common/extensions/docs/server2/subversion_fetcher.py
@@ -2,21 +2,21 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-SUBVERSION_URL = 'http://src.chromium.org/viewvc/chrome/'
-TRUNK_URL = SUBVERSION_URL + 'trunk/'
-BRANCH_URL = SUBVERSION_URL + 'branches/'
+SUBVERSION_URL = 'http://src.chromium.org/viewvc/chrome'
+TRUNK_URL = SUBVERSION_URL + '/trunk'
+BRANCH_URL = SUBVERSION_URL + '/branches'
 
 class SubversionFetcher(object):
   """Class to fetch resources from src.chromium.org.
   """
   def __init__(self, branch, base_path, url_fetcher):
-    self._base_path = self._GetURLFromBranch(branch) + base_path
+    self._base_path = self._GetURLFromBranch(branch) + '/' + base_path
     self._url_fetcher = url_fetcher
 
   def _GetURLFromBranch(self, branch):
     if branch == 'trunk':
-      return TRUNK_URL + 'src/'
-    return BRANCH_URL + branch + '/src/'
+      return TRUNK_URL + '/src'
+    return BRANCH_URL + '/' + branch + '/src'
 
   def FetchResource(self, path):
-    return self._url_fetcher.fetch(self._base_path + path)
+    return self._url_fetcher.fetch(self._base_path + '/' + path)
diff --git a/chrome/common/extensions/docs/server2/template_data_source.py b/chrome/common/extensions/docs/server2/template_data_source.py
index 003cbf0..a2e1b69 100644
--- a/chrome/common/extensions/docs/server2/template_data_source.py
+++ b/chrome/common/extensions/docs/server2/template_data_source.py
@@ -2,40 +2,29 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import json
-import logging
-import os
-import time
-
 from third_party.handlebar import Handlebar
 
 class TemplateDataSource(object):
-  def __init__(self, fetcher, base_paths, cache_timeout_seconds):
-    logging.info('Template data source created: %s %d' %
-        (' '.join(base_paths), cache_timeout_seconds))
-    self._fetcher = fetcher
-    self._template_cache = {}
+  """This class fetches and compiles templates using the fetcher passed in with
+  |cache_builder|.
+  """
+  def __init__(self, cache_builder, base_paths):
+    self._cache = cache_builder.build(self._LoadTemplate)
     self._base_paths = base_paths
-    self._cache_timeout_seconds = cache_timeout_seconds
+
+  def _LoadTemplate(self, template):
+    return Handlebar(template)
 
   def Render(self, template_name, context):
     """This method will render a template named |template_name|, fetching all
-    the partial templates needed with |self._fetcher|. Partials are retrieved
+    the partial templates needed from |self._cache|. Partials are retrieved
     from the TemplateDataSource with the |get| method.
     """
     template = self.get(template_name)
     if not template:
       return ''
       # TODO error handling
-    return template.render(json.loads(context), {'templates': self}).text
-
-  class _CachedTemplate(object):
-    def __init__(self, template, expiry):
-      self.template = template
-      self._expiry = expiry
-
-    def HasExpired(self):
-      return time.time() > self._expiry
+    return template.render(context, {'templates': self}).text
 
   def __getitem__(self, key):
     return self.get(key)
@@ -44,22 +33,10 @@ class TemplateDataSource(object):
     index = key.rfind('.html')
     if index > 0:
       key = key[:index]
-    path = key + '.html'
-    if key in self._template_cache:
-      if self._template_cache[key].HasExpired():
-        self._template_cache.pop(key)
-    if key not in self._template_cache:
-      logging.info('Template cache miss for: ' + path)
-      compiled_template = None
-      for base_path in self._base_paths:
-        try:
-          template = self._fetcher.FetchResource(base_path + path).content
-          compiled_template = Handlebar(template)
-          self._template_cache[key] = self._CachedTemplate(
-              compiled_template,
-              time.time() + self._cache_timeout_seconds)
-          break
-        except:
-          pass
-
-    return compiled_template
+    real_path = key + '.html'
+    for base_path in self._base_paths:
+      try:
+        return self._cache.get(base_path + '/' + real_path)
+      except:
+        pass
+    return None
diff --git a/chrome/common/extensions/docs/server2/template_data_source_test.py b/chrome/common/extensions/docs/server2/template_data_source_test.py
index a96e4bd..c0239a1 100755
--- a/chrome/common/extensions/docs/server2/template_data_source_test.py
+++ b/chrome/common/extensions/docs/server2/template_data_source_test.py
@@ -7,6 +7,7 @@ import json
 import os
 import unittest
 
+from fetcher_cache import FetcherCache
 from local_fetcher import LocalFetcher
 from template_data_source import TemplateDataSource
 from third_party.handlebar import Handlebar
@@ -22,14 +23,16 @@ class TemplateDataSourceTest(unittest.TestCase):
   def _RenderTest(self, name, data_source):
     template_name = name + '_tmpl.html'
     template = Handlebar(self._ReadLocalFile(template_name))
-    self.assertEquals(self._ReadLocalFile(name + '_expected.html'),
-                      data_source.Render(template_name,
-                                         self._ReadLocalFile(name + '.json')))
+    context = json.loads(self._ReadLocalFile(name + '.json'))
+    self.assertEquals(
+        self._ReadLocalFile(name + '_expected.html'),
+        data_source.Render(template_name, context))
 
   def testSimple(self):
     self._base_path = os.path.join(self._base_path, 'simple')
     fetcher = LocalFetcher(self._base_path)
-    t_data_source = TemplateDataSource(fetcher, ['./'], 0)
+    cache_builder = FetcherCache.Builder(fetcher, 0)
+    t_data_source = TemplateDataSource(cache_builder, ['./'])
 
     template_a1 = Handlebar(self._ReadLocalFile('test1.html'))
     self.assertEqual(template_a1.render({}, {'templates': {}}).text,
@@ -44,7 +47,8 @@ class TemplateDataSourceTest(unittest.TestCase):
   def testPartials(self):
     self._base_path = os.path.join(self._base_path, 'partials')
     fetcher = LocalFetcher(self._base_path)
-    t_data_source = TemplateDataSource(fetcher, ['./'], 0)
+    cache_builder = FetcherCache.Builder(fetcher, 0)
+    t_data_source = TemplateDataSource(cache_builder, ['./'])
 
     self.assertEqual(self._ReadLocalFile('test.html'),
         t_data_source['test_tmpl'].render(
@@ -53,7 +57,8 @@ class TemplateDataSourceTest(unittest.TestCase):
   def testRender(self):
     self._base_path = os.path.join(self._base_path, 'render')
     fetcher = LocalFetcher(self._base_path)
-    t_data_source = TemplateDataSource(fetcher, ['./'], 0)
+    cache_builder = FetcherCache.Builder(fetcher, 0)
+    t_data_source = TemplateDataSource(cache_builder, ['./'])
     self._RenderTest('test1', t_data_source)
     self._RenderTest('test2', t_data_source)
 
diff --git a/chrome/common/extensions/docs/server2/test_data/api_data_source/simple/test_file.json b/chrome/common/extensions/docs/server2/test_data/api_data_source/simple/test_file.json
new file mode 100644
index 0000000..e4ade71
--- /dev/null
+++ b/chrome/common/extensions/docs/server2/test_data/api_data_source/simple/test_file.json
@@ -0,0 +1,11 @@
+[
+  {
+    "name": "Bob",
+    "occupation": "Dentist",
+    "hobbies": [
+      "Eating",
+      "Sleeping",
+      "Dentisting"
+    ]
+  }
+]
diff --git a/chrome/common/extensions/docs/template2/public/browserAction.html b/chrome/common/extensions/docs/template2/public/browserAction.html
index bec196f..4b2dabc 100644
--- a/chrome/common/extensions/docs/template2/public/browserAction.html
+++ b/chrome/common/extensions/docs/template2/public/browserAction.html
@@ -1 +1 @@
-{{test}} template
+{{namespace}} template