Fix some low hanging inefficiencies in the docs server. Two of the most

expensive operations (by profiling), apart from the template rendering itself,
are calling UnixName (model.py) and removing comments from JSON files
(json_comment_eater.py). This rewrites both and memoizes the former.

BUG=227490

Review URL: https://chromiumcodereview.appspot.com/13599004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@193334 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 112 insertions, 0 deletions

diff --git a/tools/json_comment_eater/everything.json b/tools/json_comment_eater/everything.json
new file mode 100644
index 0000000..31db503
--- /dev/null
+++ b/tools/json_comment_eater/everything.json
@@ -0,0 +1,13 @@
+// Copyright 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Test API.
+{ "namespace": "test",
+  "comments": "yo",  // Comments all have a // in them.
+  "strings": "yes",  // Comment with "strings" and " character
+  "escaped\"": "string\"isescaped",
+  "more//": "\"more",
+  "so\\many": "\\\\escapes\\\\\"whoa",
+  "comment//inmiddle": "of string"
+}
diff --git a/tools/json_comment_eater/everything_expected.json b/tools/json_comment_eater/everything_expected.json
new file mode 100644
index 0000000..3fa02c1
--- /dev/null
+++ b/tools/json_comment_eater/everything_expected.json
@@ -0,0 +1,13 @@
+
+
+
+
+
+{ "namespace": "test",
+  "comments": "yo",  
+  "strings": "yes",  
+  "escaped\"": "string\"isescaped",
+  "more//": "\"more",
+  "so\\many": "\\\\escapes\\\\\"whoa",
+  "comment//inmiddle": "of string"
+}
diff --git a/tools/json_comment_eater/json_comment_eater.py b/tools/json_comment_eater/json_comment_eater.py
new file mode 100644
index 0000000..8587464
--- /dev/null
+++ b/tools/json_comment_eater/json_comment_eater.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''Utility to remove comments from JSON files so that they can be parsed by
+json.loads.
+'''
+
+def _Rcount(string, chars):
+  '''Returns the number of consecutive characters from |chars| that occur at the
+  end of |string|.
+  '''
+  return len(string) - len(string.rstrip(chars))
+
+def _FindNextToken(string, tokens, start):
+  '''Finds the next token in |tokens| that occurs in |string| from |start|.
+  Returns a tuple (index, token key).
+  '''
+  min_index, min_key = (-1, None)
+  for k in tokens:
+    index = string.find(k, start)
+    if index != -1 and (min_index == -1 or index < min_index):
+      min_index, min_key = (index, k)
+  return (min_index, min_key)
+
+def _ReadString(input, start, output):
+  output.append('"')
+  start_range, end_range = (start, input.find('"', start))
+  # \" escapes the ", \\" doesn't, \\\" does, etc.
+  while (end_range != -1 and
+         _Rcount(input[start_range:end_range], '\\') % 2 == 1):
+    start_range, end_range = (end_range, input.find('"', end_range + 1))
+  if end_range == -1:
+    return start_range + 1
+  output.append(input[start:end_range + 1])
+  return end_range + 1
+
+def _ReadComment(input, start, output):
+  eol_tokens = ('\n', '\r')
+  eol_token_index, eol_token = _FindNextToken(input, eol_tokens, start)
+  if eol_token is None:
+    return len(input)
+  output.append(eol_token)
+  return eol_token_index + len(eol_token)
+
+def Nom(input):
+  token_actions = {
+    '"': _ReadString,
+    '//': _ReadComment,
+  }
+  output = []
+  pos = 0
+  while pos < len(input):
+    token_index, token = _FindNextToken(input, token_actions.keys(), pos)
+    if token is None:
+      output.append(input[pos:])
+      break
+    output.append(input[pos:token_index])
+    pos = token_actions[token](input, token_index + len(token), output)
+  return ''.join(output)
diff --git a/tools/json_comment_eater/json_comment_eater_test.py b/tools/json_comment_eater/json_comment_eater_test.py
new file mode 100755
index 0000000..5a230eb
--- /dev/null
+++ b/tools/json_comment_eater/json_comment_eater_test.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from json_comment_eater import Nom
+import unittest
+
+class JsonCommentEaterTest(unittest.TestCase):
+  def _Load(self, test_name):
+    '''Loads the input and expected output for |test_name| as given by reading
+    in |test_name|.json and |test_name|_expected.json, and returns the string
+    contents as a tuple in that order.
+    '''
+    def read(file_name):
+      with open(file_name, 'r') as f:
+        return f.read()
+    return [read(pattern % test_name)
+            for pattern in ('%s.json', '%s_expected.json')]
+
+  def testEverything(self):
+    json, expected_json = self._Load('everything')
+    self.assertEqual(expected_json, Nom(json))
+
+if __name__ == '__main__':
+  unittest.main()