diff options
Diffstat (limited to 'chrome/common')
14 files changed, 96 insertions, 39 deletions
diff --git a/chrome/common/extensions/docs/server2/api_data_source.py b/chrome/common/extensions/docs/server2/api_data_source.py index f446bdc..a614a29 100644 --- a/chrome/common/extensions/docs/server2/api_data_source.py +++ b/chrome/common/extensions/docs/server2/api_data_source.py @@ -528,10 +528,8 @@ class APIDataSource(object): self._base_path = base_path self._availability_finder = availability_finder self._branch_utility = branch_utility - self._parse_cache = create_compiled_fs( - lambda _, json: json_parse.Parse(json), - 'intro-cache') + self._parse_cache = compiled_fs_factory.ForJson(file_system) self._template_cache = compiled_fs_factory.ForTemplates(file_system) # These must be set later via the SetFooDataSourceFactory methods. diff --git a/chrome/common/extensions/docs/server2/api_data_source_test.py b/chrome/common/extensions/docs/server2/api_data_source_test.py index c862377..611ef49 100755 --- a/chrome/common/extensions/docs/server2/api_data_source_test.py +++ b/chrome/common/extensions/docs/server2/api_data_source_test.py @@ -76,11 +76,8 @@ class APIDataSourceTest(unittest.TestCase): self._base_path = os.path.join(sys.path[0], 'test_data', 'test_json') self._compiled_fs_factory = CompiledFileSystem.Factory( ObjectStoreCreator.ForTest()) - self._json_cache = self._compiled_fs_factory.Create( - TestFileSystem(CANNED_TEST_FILE_SYSTEM_DATA), - lambda _, json: json_parse.Parse(json), - APIDataSourceTest, - 'test') + self._json_cache = self._compiled_fs_factory.ForJson( + TestFileSystem(CANNED_TEST_FILE_SYSTEM_DATA)) def _ReadLocalFile(self, filename): with open(os.path.join(self._base_path, filename), 'r') as f: diff --git a/chrome/common/extensions/docs/server2/api_models.py b/chrome/common/extensions/docs/server2/api_models.py index 3928c03..e19e37f 100644 --- a/chrome/common/extensions/docs/server2/api_models.py +++ b/chrome/common/extensions/docs/server2/api_models.py @@ -6,6 +6,7 @@ import logging import os import posixpath +from compiled_file_system import SingleFile from file_system import FileNotFoundError from future import Gettable, Future from schema_util import ProcessSchema @@ -13,6 +14,7 @@ from svn_constants import API_PATH from third_party.json_schema_compiler.model import Namespace, UnixName +@SingleFile def _CreateAPIModel(path, data): schema = ProcessSchema(path, data) if os.path.splitext(path)[1] == '.json': diff --git a/chrome/common/extensions/docs/server2/api_models_test.py b/chrome/common/extensions/docs/server2/api_models_test.py index ff191d0..45cb70e 100755 --- a/chrome/common/extensions/docs/server2/api_models_test.py +++ b/chrome/common/extensions/docs/server2/api_models_test.py @@ -11,6 +11,7 @@ from api_models import APIModels from compiled_file_system import CompiledFileSystem from features_bundle import FeaturesBundle from file_system import FileNotFoundError +from mock_file_system import MockFileSystem from object_store_creator import ObjectStoreCreator from test_file_system import TestFileSystem from test_util import ReadFile @@ -53,11 +54,11 @@ class APIModelsTest(unittest.TestCase): def setUp(self): object_store_creator = ObjectStoreCreator.ForTest() compiled_fs_factory = CompiledFileSystem.Factory(object_store_creator) - file_system = TestFileSystem(_TEST_DATA) + self._mock_file_system = MockFileSystem(TestFileSystem(_TEST_DATA)) features_bundle = FeaturesBundle( - file_system, compiled_fs_factory, object_store_creator) + self._mock_file_system, compiled_fs_factory, object_store_creator) self._api_models = APIModels( - features_bundle, compiled_fs_factory, file_system) + features_bundle, compiled_fs_factory, self._mock_file_system) def testGetNames(self): self.assertEqual( @@ -109,6 +110,27 @@ class APIModelsTest(unittest.TestCase): self.assertRaises(FileNotFoundError, self._api_models.GetModel('api/storage.idl').Get) + def testSingleFile(self): + # 2 stats (1 for JSON and 1 for IDL), 1 read (for IDL file which existed). + future = self._api_models.GetModel('alarms') + self.assertTrue(*self._mock_file_system.CheckAndReset( + read_count=1, stat_count=2)) + + # 1 read-resolve (for the IDL file). + # + # The important part here and above is that it's only doing a single read; + # any more would break the contract that only a single file is accessed - + # see the SingleFile annotation in api_models._CreateAPIModel. + future.Get() + self.assertTrue(*self._mock_file_system.CheckAndReset( + read_resolve_count=1)) + + # 2 stats (1 for JSON and 1 for IDL), no reads (still cached). + future = self._api_models.GetModel('alarms') + self.assertTrue(*self._mock_file_system.CheckAndReset(stat_count=2)) + future.Get() + self.assertTrue(*self._mock_file_system.CheckAndReset()) + if __name__ == '__main__': unittest.main() diff --git a/chrome/common/extensions/docs/server2/app.yaml b/chrome/common/extensions/docs/server2/app.yaml index c31cc06..e4d26f1 100644 --- a/chrome/common/extensions/docs/server2/app.yaml +++ b/chrome/common/extensions/docs/server2/app.yaml @@ -1,5 +1,5 @@ application: chrome-apps-doc -version: 2-33-0 +version: 2-33-1 runtime: python27 api_version: 1 threadsafe: false diff --git a/chrome/common/extensions/docs/server2/compiled_file_system.py b/chrome/common/extensions/docs/server2/compiled_file_system.py index 3304290..fd52ae3 100644 --- a/chrome/common/extensions/docs/server2/compiled_file_system.py +++ b/chrome/common/extensions/docs/server2/compiled_file_system.py @@ -12,6 +12,19 @@ from third_party.json_schema_compiler import json_parse from third_party.json_schema_compiler.memoize import memoize +_SINGLE_FILE_FUNCTIONS = set() + + +def SingleFile(fn): + '''A decorator which can be optionally applied to the compilation function + passed to CompiledFileSystem.Create, indicating that the function only + needs access to the file which is given in the function's callback. When + this is the case some optimisations can be done. + ''' + _SINGLE_FILE_FUNCTIONS.add(fn) + return fn + + class _CacheEntry(object): def __init__(self, cache_data, version): @@ -30,10 +43,10 @@ class CompiledFileSystem(object): def __init__(self, object_store_creator): self._object_store_creator = object_store_creator - def Create(self, file_system, populate_function, cls, category=None): + def Create(self, file_system, compilation_function, cls, category=None): '''Creates a CompiledFileSystem view over |file_system| that populates - its cache by calling |populate_function| with (path, data), where |data| - is the data that was fetched from |path| in |file_system|. + its cache by calling |compilation_function| with (path, data), where + |data| is the data that was fetched from |path| in |file_system|. The namespace for the compiled file system is derived similar to ObjectStoreCreator: from |cls| along with an optional |category|. @@ -44,10 +57,18 @@ class CompiledFileSystem(object): if category is not None: full_name.append(category) def create_object_store(my_category): + # The read caches can start populated (start_empty=False) because file + # updates are picked up by the stat - but only if the compilation + # function is affected by a single file. If the compilation function is + # affected by other files (e.g. compiling a list of APIs available to + # extensions may be affected by both a features file and the list of + # files in the API directory) then this optimisation won't work. return self._object_store_creator.Create( - CompiledFileSystem, category='/'.join(full_name + [my_category])) + CompiledFileSystem, + category='/'.join(full_name + [my_category]), + start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS) return CompiledFileSystem(file_system, - populate_function, + compilation_function, create_object_store('file'), create_object_store('list')) @@ -57,7 +78,7 @@ class CompiledFileSystem(object): These are memoized over file systems tied to different branches. ''' return self.Create(file_system, - lambda _, data: json_parse.Parse(data), + SingleFile(lambda _, data: json_parse.Parse(data)), CompiledFileSystem, category='json') @@ -68,7 +89,7 @@ class CompiledFileSystem(object): as Model and APISchemaGraph. ''' return self.Create(file_system, - schema_util.ProcessSchema, + SingleFile(schema_util.ProcessSchema), CompiledFileSystem, category='api-schema') @@ -76,17 +97,18 @@ class CompiledFileSystem(object): def ForTemplates(self, file_system): '''Creates a CompiledFileSystem for parsing templates. ''' - return self.Create(file_system, - lambda path, text: Handlebar(text, name=path), - CompiledFileSystem) + return self.Create( + file_system, + SingleFile(lambda path, text: Handlebar(text, name=path)), + CompiledFileSystem) def __init__(self, file_system, - populate_function, + compilation_function, file_object_store, list_object_store): self._file_system = file_system - self._populate_function = populate_function + self._compilation_function = compilation_function self._file_object_store = file_object_store self._list_object_store = list_object_store @@ -146,7 +168,7 @@ class CompiledFileSystem(object): return Future(delegate=Gettable(resolve)) def GetFromFile(self, path, binary=False): - '''Calls |populate_function| on the contents of the file at |path|. If + '''Calls |compilation_function| on the contents of the file at |path|. If |binary| is True then the file will be read as binary - but this will only apply for the first time the file is fetched; if already cached, |binary| will be ignored. @@ -162,13 +184,13 @@ class CompiledFileSystem(object): future_files = self._file_system.ReadSingle(path, binary=binary) def resolve(): - cache_data = self._populate_function(path, future_files.Get()) + cache_data = self._compilation_function(path, future_files.Get()) self._file_object_store.Set(path, _CacheEntry(cache_data, version)) return cache_data return Future(delegate=Gettable(resolve)) def GetFromFileListing(self, path): - '''Calls |populate_function| on the listing of the files at |path|. + '''Calls |compilation_function| on the listing of the files at |path|. Assumes that the path given is to a directory. ''' if not path.endswith('/'): @@ -185,7 +207,7 @@ class CompiledFileSystem(object): recursive_list_future = self._RecursiveList(path) def resolve(): - cache_data = self._populate_function(path, recursive_list_future.Get()) + cache_data = self._compilation_function(path, recursive_list_future.Get()) self._list_object_store.Set(path, _CacheEntry(cache_data, version)) return cache_data return Future(delegate=Gettable(resolve)) diff --git a/chrome/common/extensions/docs/server2/cron.yaml b/chrome/common/extensions/docs/server2/cron.yaml index c77e536..3392aeb 100644 --- a/chrome/common/extensions/docs/server2/cron.yaml +++ b/chrome/common/extensions/docs/server2/cron.yaml @@ -2,4 +2,4 @@ cron: - description: Repopulates all cached data. url: /_cron schedule: every 5 minutes - target: 2-33-0 + target: 2-33-1 diff --git a/chrome/common/extensions/docs/server2/directory_zipper.py b/chrome/common/extensions/docs/server2/directory_zipper.py index daa96e7..482a1d7a 100644 --- a/chrome/common/extensions/docs/server2/directory_zipper.py +++ b/chrome/common/extensions/docs/server2/directory_zipper.py @@ -6,6 +6,8 @@ from io import BytesIO import posixpath from zipfile import ZipFile +from compiled_file_system import SingleFile + class DirectoryZipper(object): '''Creates zip files of whole directories. @@ -17,6 +19,10 @@ class DirectoryZipper(object): self._MakeZipFile, DirectoryZipper) + # NOTE: It's ok to specify SingleFile here even though this method reads + # multiple files. All files are underneath |base_dir|. If any file changes its + # stat will change, so the stat of |base_dir| will also change. + @SingleFile def _MakeZipFile(self, base_dir, files): base_dir = base_dir.strip('/') zip_bytes = BytesIO() diff --git a/chrome/common/extensions/docs/server2/intro_data_source.py b/chrome/common/extensions/docs/server2/intro_data_source.py index 071f56f..ee244e8 100644 --- a/chrome/common/extensions/docs/server2/intro_data_source.py +++ b/chrome/common/extensions/docs/server2/intro_data_source.py @@ -79,6 +79,10 @@ class IntroDataSource(object): api_name = os.path.splitext(intro_path.split('/')[-1])[0] intro_with_links = self._ref_resolver.ResolveAllLinks(intro, namespace=api_name) + # TODO(kalman): Do $ref replacement after rendering the template, not + # before, so that (a) $ref links can contain template annotations, and (b) + # we can use CompiledFileSystem.ForTemplates to create the templates and + # save ourselves some effort. apps_parser = _IntroParser() apps_parser.feed(Handlebar(intro_with_links).render( { 'is_apps': True }).text) diff --git a/chrome/common/extensions/docs/server2/path_canonicalizer.py b/chrome/common/extensions/docs/server2/path_canonicalizer.py index b2ed8d2..d4f4b49 100644 --- a/chrome/common/extensions/docs/server2/path_canonicalizer.py +++ b/chrome/common/extensions/docs/server2/path_canonicalizer.py @@ -8,6 +8,7 @@ import posixpath import traceback from branch_utility import BranchUtility +from compiled_file_system import CompiledFileSystem, SingleFile from file_system import FileNotFoundError from third_party.json_schema_compiler.model import UnixName import svn_constants @@ -26,6 +27,7 @@ class PathCanonicalizer(object): ''' def __init__(self, compiled_fs_factory, file_system): # Map of simplified API names (for typo detection) to their real paths. + @SingleFile def make_public_apis(_, file_names): return dict((_SimplifyFileName(name), name) for name in file_names) self._public_apis = compiled_fs_factory.Create(file_system, diff --git a/chrome/common/extensions/docs/server2/redirector.py b/chrome/common/extensions/docs/server2/redirector.py index 9c8136e..0a38efd 100644 --- a/chrome/common/extensions/docs/server2/redirector.py +++ b/chrome/common/extensions/docs/server2/redirector.py @@ -6,14 +6,12 @@ import posixpath from urlparse import urlsplit from file_system import FileNotFoundError -from third_party.json_schema_compiler.json_parse import Parse class Redirector(object): def __init__(self, compiled_fs_factory, file_system, root_path): self._root_path = root_path self._file_system = file_system - self._cache = compiled_fs_factory.Create( - file_system, lambda _, rules: Parse(rules), Redirector) + self._cache = compiled_fs_factory.ForJson(file_system) def Redirect(self, host, path): ''' Check if a path should be redirected, first according to host diff --git a/chrome/common/extensions/docs/server2/sidenav_data_source.py b/chrome/common/extensions/docs/server2/sidenav_data_source.py index 6beab25..9364d72 100644 --- a/chrome/common/extensions/docs/server2/sidenav_data_source.py +++ b/chrome/common/extensions/docs/server2/sidenav_data_source.py @@ -5,6 +5,7 @@ import copy import logging +from compiled_file_system import SingleFile from data_source import DataSource from future import Gettable, Future from third_party.json_schema_compiler.json_parse import Parse @@ -50,6 +51,7 @@ class SidenavDataSource(DataSource): self._server_instance = server_instance self._request = request + @SingleFile def _CreateSidenavDict(self, _, content): items = Parse(content) # Start at level 2, the top <ul> element is level 1. diff --git a/chrome/common/extensions/docs/server2/sidenav_data_source_test.py b/chrome/common/extensions/docs/server2/sidenav_data_source_test.py index 322752f..00104d7 100755 --- a/chrome/common/extensions/docs/server2/sidenav_data_source_test.py +++ b/chrome/common/extensions/docs/server2/sidenav_data_source_test.py @@ -7,6 +7,7 @@ import json import unittest from compiled_file_system import CompiledFileSystem +from mock_file_system import MockFileSystem from object_store_creator import ObjectStoreCreator from server_instance import ServerInstance from servlet import Request @@ -101,7 +102,7 @@ class SamplesDataSourceTest(unittest.TestCase): self.assertEqual(2, len(log_output)) def testSidenavDataSource(self): - file_system = TestFileSystem({ + file_system = MockFileSystem(TestFileSystem({ 'apps_sidenav.json': json.dumps([{ 'title': 'H1', 'href': 'H1.html', @@ -110,7 +111,7 @@ class SamplesDataSourceTest(unittest.TestCase): 'href': '/H2.html' }] }]) - }, relative_to='docs/templates/json') + }, relative_to='docs/templates/json')) expected = [{ 'level': 2, @@ -127,6 +128,7 @@ class SamplesDataSourceTest(unittest.TestCase): sidenav_data_source = SidenavDataSource( ServerInstance.ForTest(file_system), Request.ForTest('/H2.html')) + self.assertTrue(*file_system.CheckAndReset()) log_output = CaptureLogging( lambda: self.assertEqual(expected, sidenav_data_source.get('apps'))) @@ -135,6 +137,11 @@ class SamplesDataSourceTest(unittest.TestCase): self.assertTrue( log_output[0].msg.startswith('Paths in sidenav must be qualified.')) + # Test that only a single file is read when creating the sidenav, so that + # we can be confident in the compiled_file_system.SingleFile annotation. + self.assertTrue(*file_system.CheckAndReset( + read_count=1, stat_count=1, read_resolve_count=1)) + def testCron(self): file_system = TestFileSystem({ 'apps_sidenav.json': '[{ "title": "H1" }]' , diff --git a/chrome/common/extensions/docs/server2/strings_data_source.py b/chrome/common/extensions/docs/server2/strings_data_source.py index b4042bf..d182add 100644 --- a/chrome/common/extensions/docs/server2/strings_data_source.py +++ b/chrome/common/extensions/docs/server2/strings_data_source.py @@ -3,17 +3,14 @@ # found in the LICENSE file. from data_source import DataSource -from third_party.json_schema_compiler.json_parse import Parse class StringsDataSource(DataSource): '''Provides templates with access to a key to string mapping defined in a JSON configuration file. ''' def __init__(self, server_instance, _): - self._cache = server_instance.compiled_fs_factory.Create( - server_instance.host_file_system_provider.GetTrunk(), - lambda _, strings_json: Parse(strings_json), - StringsDataSource) + self._cache = server_instance.compiled_fs_factory.ForJson( + server_instance.host_file_system_provider.GetTrunk()) self._strings_json_path = server_instance.strings_json_path def Cron(self): |