diff options
| -rwxr-xr-x | chrome/common/extensions/docs/build/directory.py | 9 | ||||
| -rw-r--r-- | third_party/json_minify/README.chromium | 3 | ||||
| -rw-r--r-- | third_party/json_minify/__init__.py | 5 | ||||
| -rw-r--r-- | third_party/json_minify/minify_json.py | 113 | ||||
| -rwxr-xr-x | tools/json_comment_eater.py | 40 | ||||
| -rw-r--r-- | tools/json_schema_compiler/json_schema.py | 50 |
6 files changed, 55 insertions, 165 deletions
diff --git a/chrome/common/extensions/docs/build/directory.py b/chrome/common/extensions/docs/build/directory.py index 64f701c..d4fbf04 100755 --- a/chrome/common/extensions/docs/build/directory.py +++ b/chrome/common/extensions/docs/build/directory.py @@ -12,7 +12,12 @@ import re import hashlib import zipfile import simplejson as json -import json_minify as minify +import sys + +_script_path = os.path.realpath(__file__) +sys.path.insert(0, os.path.normpath(_script_path + + "/../../../../../../tools")) +import json_comment_eater # Make sure we get consistent string sorting behavior by explicitly using the # default C locale. @@ -58,7 +63,7 @@ def parse_json_file(path, encoding="utf-8"): try: json_str = json_file.read() - json_obj = json.loads(minify.json_minify(json_str), encoding) + json_obj = json.loads(json_comment_eater.Nom(json_str), encoding) except ValueError, msg: raise Exception("Failed to parse JSON out of file %s: %s" % (path, msg)) finally: diff --git a/third_party/json_minify/README.chromium b/third_party/json_minify/README.chromium index 0f5f630..019c326 100644 --- a/third_party/json_minify/README.chromium +++ b/third_party/json_minify/README.chromium @@ -11,7 +11,6 @@ Description: A set of scripts that remove comments and whitespace from JSON files. Local Modifications: -- Added the __init__.py file for easier imports - Got the (much faster) json-minify-sans-regexp.js file from the second URL listed -- Small change to minify_json.py to pass license tests +- Removed minify_json.py, it is crazy slow. See commit for replacement. diff --git a/third_party/json_minify/__init__.py b/third_party/json_minify/__init__.py deleted file mode 100644 index 45bce20..0000000 --- a/third_party/json_minify/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -from minify_json import json_minify diff --git a/third_party/json_minify/minify_json.py b/third_party/json_minify/minify_json.py deleted file mode 100644 index 69aac6e..0000000 --- a/third_party/json_minify/minify_json.py +++ /dev/null @@ -1,113 +0,0 @@ -# Use of this source code is governed by a BSD-style license (MIT) -''' -Created on 20/01/2011 - -v0.1 (C) Gerald Storer -MIT License - -Based on JSON.minify.js: -https://github.com/getify/JSON.minify -''' - -import re - -def json_minify(json,strip_space=True): - tokenizer=re.compile('"|(/\*)|(\*/)|(//)|\n|\r') - in_string = False - in_multiline_comment = False - in_singleline_comment = False - - new_str = [] - from_index = 0 # from is a keyword in Python - - for match in re.finditer(tokenizer,json): - - if not in_multiline_comment and not in_singleline_comment: - tmp2 = json[from_index:match.start()] - if not in_string and strip_space: - tmp2 = re.sub('[ \t\n\r]*','',tmp2) # replace only white space defined in standard - new_str.append(tmp2) - - from_index = match.end() - - if match.group() == '"' and not in_multiline_comment and not in_singleline_comment: - escaped = re.search('(\\\\)*$',json[:match.start()]) - if not in_string or escaped is None or len(escaped.group()) % 2 == 0: - # start of string with ", or unescaped " character found to end string - in_string = not in_string - from_index -= 1 # include " character in next catch - - elif match.group() == '/*' and not in_string and not in_multiline_comment and not in_singleline_comment: - in_multiline_comment = True - elif match.group() == '*/' and not in_string and in_multiline_comment and not in_singleline_comment: - in_multiline_comment = False - elif match.group() == '//' and not in_string and not in_multiline_comment and not in_singleline_comment: - in_singleline_comment = True - elif (match.group() == '\n' or match.group() == '\r') and not in_string and not in_multiline_comment and in_singleline_comment: - in_singleline_comment = False - elif not in_multiline_comment and not in_singleline_comment and ( - match.group() not in ['\n','\r',' ','\t'] or not strip_space): - new_str.append(match.group()) - - new_str.append(json[from_index:]) - return ''.join(new_str) - -if __name__ == '__main__': - import json # requires Python 2.6+ to run tests - - def test_json(s): - return json.loads(json_minify(s)) - - test1 = '''// this is a JSON file with comments -{ - "foo": "bar", // this is cool - "bar": [ - "baz", "bum", "zam" - ], -/* the rest of this document is just fluff - in case you are interested. */ - "something": 10, - "else": 20 -} - -/* NOTE: You can easily strip the whitespace and comments - from such a file with the JSON.minify() project hosted - here on github at http://github.com/getify/JSON.minify -*/ -''' - - test1_res = '''{"foo":"bar","bar":["baz","bum","zam"],"something":10,"else":20}''' - - test2 = ''' -{"/*":"*/","//":"",/*"//"*/"/*/":// -"//"} - -''' - test2_res = '''{"/*":"*/","//":"","/*/":"//"}''' - - test3 = r'''/* -this is a -multi line comment */{ - -"foo" -: - "bar/*"// something - , "b\"az":/* -something else */"blah" - -} -''' - test3_res = r'''{"foo":"bar/*","b\"az":"blah"}''' - - test4 = r'''{"foo": "ba\"r//", "bar\\": "b\\\"a/*z", - "baz\\\\": /* yay */ "fo\\\\\"*/o" -} -''' - test4_res = r'''{"foo":"ba\"r//","bar\\":"b\\\"a/*z","baz\\\\":"fo\\\\\"*/o"}''' - - assert test_json(test1) == json.loads(test1_res),'Failed test 1' - assert test_json(test2) == json.loads(test2_res),'Failed test 2' - assert test_json(test3) == json.loads(test3_res),'Failed test 3' - assert test_json(test4) == json.loads(test4_res),'Failed test 4' - if __debug__: # Don't print passed message if the asserts didn't run - print 'Passed all tests' diff --git a/tools/json_comment_eater.py b/tools/json_comment_eater.py new file mode 100755 index 0000000..1b1e07a84 --- /dev/null +++ b/tools/json_comment_eater.py @@ -0,0 +1,40 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" Utility to remove comments from JSON files so that they can be parsed by +json.loads.""" + +def _ReadString(input, start, output): + output.append('"') + in_escape = False + for pos in xrange(start, len(input)): + output.append(input[pos]) + if in_escape: + in_escape = False + else: + if input[pos] == '\\': + in_escape = True + elif input[pos] == '"': + return pos + 1 + return pos + +def _ReadComment(input, start, output): + for pos in xrange(start, len(input)): + if input[pos] in ['\r', '\n']: + output.append(input[pos]) + return pos + 1 + return pos + +def Nom(input): + output = [] + pos = 0 + while pos < len(input): + if input[pos] == '"': + pos = _ReadString(input, pos + 1, output) + elif input[pos:pos+2] == '//': + pos = _ReadComment(input, pos + 2, output) + else: + output.append(input[pos]) + pos += 1 + return ''.join(output) diff --git a/tools/json_schema_compiler/json_schema.py b/tools/json_schema_compiler/json_schema.py index 240a168..f51cece 100644 --- a/tools/json_schema_compiler/json_schema.py +++ b/tools/json_schema_compiler/json_schema.py @@ -4,49 +4,12 @@ import copy import json +import os.path +import sys -def StripJSONComments(stream): - """Strips //-style comments from a stream of JSON. Allows un-escaped // - inside string values. - """ - # Previously we used json_minify to strip comments, but it seems to be pretty - # slow and does more than we need. This implementation does a lot less work - - # it just strips comments from the beginning of the '//' delimiter until end - # of line, but only if we're not inside a string. For example: - # - # {"url": "http://www.example.com"} - # - # will work properly, as will: - # - # { - # "url": "http://www.example.com" // some comment - # } - result = "" - last_char = None - inside_string = False - inside_comment = False - buf = "" - for char in stream: - if inside_comment: - if char == '\n': - inside_comment = False - else: - continue - else: - if char == '/' and not inside_string: - if last_char == '/': - inside_comment = True - last_char = char - continue - else: - if last_char == '/' and not inside_string: - result += '/' - if char == '"': - inside_string = not inside_string - last_char = char - result += char - - return result +_script_path = os.path.realpath(__file__) +sys.path.insert(0, os.path.normpath(_script_path + "/../../")) +import json_comment_eater def DeleteNocompileNodes(item): def HasNocompile(thing): @@ -69,7 +32,8 @@ def DeleteNocompileNodes(item): def Load(filename): with open(filename, 'r') as handle: - return DeleteNocompileNodes(json.loads(StripJSONComments(handle.read()))) + return DeleteNocompileNodes( + json.loads(json_comment_eater.Nom(handle.read()))) # A dictionary mapping |filename| to the object resulting from loading the JSON |