Docserver: Cleanup availability_finder.py

BUG=233982
NOTRY=True

Review URL: https://codereview.chromium.org/353883004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@279875 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 125 insertions, 63 deletions

diff --git a/chrome/common/extensions/docs/server2/app.yaml b/chrome/common/extensions/docs/server2/app.yaml
index 60bef88..f2af2da 100644
--- a/chrome/common/extensions/docs/server2/app.yaml
+++ b/chrome/common/extensions/docs/server2/app.yaml
@@ -1,5 +1,5 @@
 application: chrome-apps-doc
-version: 3-28-0
+version: 3-28-1
 runtime: python27
 api_version: 1
 threadsafe: false
diff --git a/chrome/common/extensions/docs/server2/availability_finder.py b/chrome/common/extensions/docs/server2/availability_finder.py
index 8cdb8cd..af46e4b 100644
--- a/chrome/common/extensions/docs/server2/availability_finder.py
+++ b/chrome/common/extensions/docs/server2/availability_finder.py
@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from collections import Mapping
 import posixpath
 
 from api_schema_graph import APISchemaGraph
@@ -10,13 +9,12 @@ from branch_utility import BranchUtility, ChannelInfo
 from extensions_paths import API_PATHS, JSON_TEMPLATES
 from features_bundle import FeaturesBundle
 from file_system import FileNotFoundError
-from platform_util import PlatformToExtensionType
 from third_party.json_schema_compiler.memoize import memoize
 from third_party.json_schema_compiler.model import UnixName
 
 
+_DEVTOOLS_API = 'devtools_api.json'
 _EXTENSION_API = 'extension_api.json'
-
 # The version where api_features.json is first available.
 _API_FEATURES_MIN_VERSION = 28
 # The version where permission_ and manifest_features.json are available and
@@ -28,6 +26,26 @@ _EXTENSION_API_MAX_VERSION = 17
 _SVN_MIN_VERSION = 5
 
 
+def _GetNamespaceFromFilename(api_name):
+  '''API names passed in from the templates follow a different naming
+  convention than the actual API namespace names. Convert |api_name|
+  to its proper namespace name.
+  '''
+  # Devtools APIs are located in a devtools/ directory
+  # (e.g. devtools/panels.json). The namespace will be devtools.panels.
+  if 'devtools/' in api_name:
+    api_name = api_name.replace('/', '.')
+  # Experimental API filenames have a 'experimental_' prefixed to them (e.g.
+  # devtools/experimental_audits.json). The namespace always has
+  # 'experimental.' prefixed to it (e.g. experimental.devtools.audits).
+  if 'experimental_' in api_name:
+    api_name = 'experimental.' + api_name.replace('experimental_', '')
+  # API filenames use '_'s as separators; the separator for namespaces is
+  # always a '.'.
+  api_name = api_name.replace('_', '.')
+  return api_name
+
+
 def _GetChannelFromFeatures(api_name, features):
   '''Finds API channel information for |api_name| from |features|.
   Returns None if channel information for the API cannot be located.
@@ -36,6 +54,51 @@ def _GetChannelFromFeatures(api_name, features):
   return feature.get('channel') if feature else None
 
 
+def _GetChannelFromAPIFeatures(api_name, features_bundle):
+  return _GetChannelFromFeatures(api_name, features_bundle.GetAPIFeatures())
+
+
+def _GetChannelFromManifestFeatures(api_name, features_bundle):
+  # _manifest_features.json uses unix_style API names.
+  api_name = UnixName(api_name)
+  return _GetChannelFromFeatures(api_name,
+                                 features_bundle.GetManifestFeatures())
+
+
+def _GetChannelFromPermissionFeatures(api_name, features_bundle):
+  return _GetChannelFromFeatures(api_name,
+                                 features_bundle.GetPermissionFeatures())
+
+
+def _GetAPISchemaFilename(api_name, file_system, version):
+  '''Gets the name of the file which may contain the schema for |api_name| in
+  |file_system|, or None if the API is not found. Note that this may be the
+  single _EXTENSION_API file which all APIs share in older versions of Chrome,
+  in which case it is unknown whether the API actually exists there.
+  '''
+  if version == 'trunk' or version > _ORIGINAL_FEATURES_MIN_VERSION:
+    # API schema filenames switch format to unix_hacker_style.
+    api_name = UnixName(api_name)
+
+  # Devtools API names have 'devtools.' prepended to them.
+  # The corresponding filenames do not.
+  if 'devtools_' in api_name:
+    api_name = api_name.replace('devtools_', '')
+
+  for api_path in API_PATHS:
+    try:
+      for base, _, filenames in file_system.Walk(api_path):
+        for ext in ('json', 'idl'):
+          filename = '%s.%s' % (api_name, ext)
+          if filename in filenames:
+            return posixpath.join(api_path, base, filename)
+          if _EXTENSION_API in filenames:
+            return posixpath.join(api_path, base, _EXTENSION_API)
+    except FileNotFoundError:
+      continue
+  return None
+
+
 class AvailabilityInfo(object):
   '''Represents availability data for an API. |scheduled| is a version number
   specifying when dev and beta APIs will become stable, or None if that data
@@ -99,34 +162,11 @@ class AvailabilityFinder(object):
     return AvailabilityInfo(
         self._branch_utility.GetChannelInfo(api_info['channel']))
 
-  def _GetAPISchemaFilename(self, api_name, file_system, version):
-    '''Gets the name of the file which may contain the schema for |api_name| in
-    |file_system|, or None if the API is not found. Note that this may be the
-    single _EXTENSION_API file which all APIs share in older versions of Chrome,
-    in which case it is unknown whether the API actually exists there.
-    '''
-    if version == 'trunk' or version > _ORIGINAL_FEATURES_MIN_VERSION:
-      # API schema filenames switch format to unix_hacker_style.
-      api_name = UnixName(api_name)
-
-    found_files = file_system.Read(API_PATHS, skip_not_found=True)
-    for path, filenames in found_files.Get().iteritems():
-      try:
-        for ext in ('json', 'idl'):
-          filename = '%s.%s' % (api_name, ext)
-          if filename in filenames:
-            return path + filename
-          if _EXTENSION_API in filenames:
-            return path + _EXTENSION_API
-      except FileNotFoundError:
-        pass
-    return None
-
   def _GetAPISchema(self, api_name, file_system, version):
     '''Searches |file_system| for |api_name|'s API schema data, and processes
     and returns it if found.
     '''
-    api_filename = self._GetAPISchemaFilename(api_name, file_system, version)
+    api_filename = _GetAPISchemaFilename(api_name, file_system, version)
     if api_filename is None:
       # No file for the API could be found in the given |file_system|.
       return None
@@ -141,13 +181,13 @@ class AvailabilityFinder(object):
     return matching_schemas or None
 
   def _HasAPISchema(self, api_name, file_system, version):
-    '''Whether or not an API schema for |api_name|exists in the given
+    '''Whether or not an API schema for |api_name| exists in the given
     |file_system|.
     '''
-    filename = self._GetAPISchemaFilename(api_name, file_system, version)
+    filename = _GetAPISchemaFilename(api_name, file_system, version)
     if filename is None:
       return False
-    if filename.endswith(_EXTENSION_API):
+    if filename.endswith(_EXTENSION_API) or filename.endswith(_DEVTOOLS_API):
       return self._GetAPISchema(api_name, file_system, version) is not None
     return True
 
@@ -164,7 +204,7 @@ class AvailabilityFinder(object):
     if version >= _API_FEATURES_MIN_VERSION:
       # The _api_features.json file first appears in version 28 and should be
       # the most reliable for finding API availability.
-      available_channel = self._GetChannelFromAPIFeatures(api_name,
+      available_channel = _GetChannelFromAPIFeatures(api_name,
                                                           features_bundle)
     if version >= _ORIGINAL_FEATURES_MIN_VERSION:
       # The _permission_features.json and _manifest_features.json files are
@@ -172,8 +212,8 @@ class AvailabilityFinder(object):
       # found using _api_features.json.
       available_channel = (
           available_channel or
-          self._GetChannelFromPermissionFeatures(api_name, features_bundle) or
-          self._GetChannelFromManifestFeatures(api_name, features_bundle))
+          _GetChannelFromPermissionFeatures(api_name, features_bundle) or
+          _GetChannelFromManifestFeatures(api_name, features_bundle))
       if available_channel is not None:
         return available_channel == 'stable'
     if version >= _SVN_MIN_VERSION:
@@ -189,9 +229,9 @@ class AvailabilityFinder(object):
     '''
     features_bundle = self._CreateFeaturesBundle(file_system)
     available_channel = (
-        self._GetChannelFromAPIFeatures(api_name, features_bundle) or
-        self._GetChannelFromPermissionFeatures(api_name, features_bundle) or
-        self._GetChannelFromManifestFeatures(api_name, features_bundle))
+        _GetChannelFromAPIFeatures(api_name, features_bundle) or
+        _GetChannelFromPermissionFeatures(api_name, features_bundle) or
+        _GetChannelFromManifestFeatures(api_name, features_bundle))
     if (available_channel is None and
         self._HasAPISchema(api_name, file_system, channel_info.version)):
       # If an API is not represented in any of the _features files, but exists
@@ -211,29 +251,18 @@ class AvailabilityFinder(object):
                           self._object_store_creator,
                           self._platform)
 
-  def _GetChannelFromAPIFeatures(self, api_name, features_bundle):
-    return _GetChannelFromFeatures(api_name, features_bundle.GetAPIFeatures())
-
-  def _GetChannelFromManifestFeatures(self, api_name, features_bundle):
-    # _manifest_features.json uses unix_style API names.
-    api_name = UnixName(api_name)
-    return _GetChannelFromFeatures(api_name,
-                                   features_bundle.GetManifestFeatures())
-
-  def _GetChannelFromPermissionFeatures(self, api_name, features_bundle):
-    return _GetChannelFromFeatures(api_name,
-                                   features_bundle.GetPermissionFeatures())
-
   def _CheckAPIAvailability(self, api_name, file_system, channel_info):
     '''Determines the availability for an API at a certain version of Chrome.
     Two branches of logic are used depending on whether or not the API is
     determined to be 'stable' at the given version.
     '''
     if channel_info.channel == 'stable':
-      return self._CheckStableAvailability(
-          api_name, file_system, channel_info.version)
-    return self._CheckChannelAvailability(
-        api_name, file_system, channel_info)
+      return self._CheckStableAvailability(api_name,
+                                           file_system,
+                                           channel_info.version)
+    return self._CheckChannelAvailability(api_name,
+                                          file_system,
+                                          channel_info)
 
   def _FindScheduled(self, api_name):
     '''Determines the earliest version of Chrome where the API is stable.
@@ -290,6 +319,7 @@ class AvailabilityFinder(object):
     '''Returns an APISchemaGraph annotated with each node's availability (the
     ChannelInfo at the oldest channel it's available in).
     '''
+    api_name = _GetNamespaceFromFilename(api_name)
     availability_graph = self._node_level_object_store.Get(api_name).Get()
     if availability_graph is not None:
       return availability_graph
@@ -299,17 +329,29 @@ class AvailabilityFinder(object):
       return value
 
     availability_graph = APISchemaGraph()
-
     host_fs = self._host_file_system
-    trunk_stat = assert_not_none(host_fs.Stat(self._GetAPISchemaFilename(
+    trunk_stat = assert_not_none(host_fs.Stat(_GetAPISchemaFilename(
         api_name, host_fs, 'trunk')))
 
     # Weird object thing here because nonlocal is Python 3.
     previous = type('previous', (object,), {'stat': None, 'graph': None})
 
     def update_availability_graph(file_system, channel_info):
-      version_filename = assert_not_none(self._GetAPISchemaFilename(
-          api_name, file_system, channel_info.version))
+      # If we can't find a filename, skip checking at this branch.
+      # For example, something could have a predetermined availability of 23,
+      # but it doesn't show up in the file system until 26.
+      # We know that the file will become available at some point.
+      #
+      # The problem with this is that at the first version where the API file
+      # exists, we'll get a huge chunk of new objects that don't match
+      # the predetermined API availability.
+      version_filename = _GetAPISchemaFilename(api_name,
+                                               file_system,
+                                               channel_info.version)
+      if version_filename is None:
+        # Continue the loop at the next version.
+        return True
+
       version_stat = assert_not_none(file_system.Stat(version_filename))
 
       # Important optimisation: only re-parse the graph if the file changed in
@@ -323,8 +365,9 @@ class AvailabilityFinder(object):
         #
         # Calling |availability_graph|.Lookup() on the nodes being updated
         # will return the |annotation| object -- the current |channel_info|.
-        version_graph = APISchemaGraph(self._GetAPISchema(
-            api_name, file_system, channel_info.version))
+        version_graph = APISchemaGraph(self._GetAPISchema(api_name,
+                                                          file_system,
+                                                          channel_info.version))
         availability_graph.Update(version_graph.Subtract(availability_graph),
                                   annotation=channel_info)
 
diff --git a/chrome/common/extensions/docs/server2/availability_finder_test.py b/chrome/common/extensions/docs/server2/availability_finder_test.py
index 3edf7ff..37fa6ec 100755
--- a/chrome/common/extensions/docs/server2/availability_finder_test.py
+++ b/chrome/common/extensions/docs/server2/availability_finder_test.py
@@ -7,7 +7,9 @@ import sys
 import unittest
 
 import api_schema_graph
-from availability_finder import AvailabilityFinder, AvailabilityInfo
+from availability_finder import (AvailabilityFinder,
+                                 AvailabilityInfo,
+                                 _GetNamespaceFromFilename)
 from branch_utility import BranchUtility, ChannelInfo
 from compiled_file_system import CompiledFileSystem
 from fake_host_file_system_provider import FakeHostFileSystemProvider
@@ -72,6 +74,23 @@ class AvailabilityFinderTest(unittest.TestCase):
         self._branch_utility.GetStableChannelInfo(13),
         stat_paths)
 
+  def testGetNamespaceFromFilename(self):
+    # Test simple name
+    self.assertEqual('storage', _GetNamespaceFromFilename('storage'))
+    # Test multi-word names
+    self.assertEqual('contextMenus',
+                     _GetNamespaceFromFilename('contextMenus'))
+    self.assertEqual('app.window', _GetNamespaceFromFilename('app_window'))
+    # Test devtools API
+    self.assertEqual('devtools.inspectedWindow',
+                     _GetNamespaceFromFilename('devtools/inspectedWindow'))
+    # Test experimental API
+    self.assertEqual('experimental.infobars',
+                     _GetNamespaceFromFilename('experimental_infobars'))
+    # Test experimental API in devtools
+    self.assertEqual('experimental.devtools.audits',
+                     _GetNamespaceFromFilename('devtools/experimental_audits'))
+
   def testGraphOptimization(self):
     for platform in GetPlatforms():
       # Keep track of how many times the APISchemaGraph constructor is called.
@@ -188,8 +207,8 @@ class AvailabilityFinderTest(unittest.TestCase):
               only_on='apps')
 
   def testGetAPINodeAvailability(self):
-    # Allow the LookupResult constructions below to take just one line.
     for platform in GetPlatforms():
+      # Allow the LookupResult constructions below to take just one line.
       lookup_result = api_schema_graph.LookupResult
       availability_graph = self._create_availability_finder(
           self._node_fs_creator,
diff --git a/chrome/common/extensions/docs/server2/cron.yaml b/chrome/common/extensions/docs/server2/cron.yaml
index 1b9b31a..23f580c 100644
--- a/chrome/common/extensions/docs/server2/cron.yaml
+++ b/chrome/common/extensions/docs/server2/cron.yaml
@@ -2,4 +2,4 @@ cron:
 - description: Repopulates all cached data.
   url: /_cron
   schedule: every 5 minutes
-  target: 3-28-0
+  target: 3-28-1