diff options
| -rwxr-xr-x | third_party/closure_compiler/build/inputs.py | 2 | ||||
| -rwxr-xr-x | third_party/closure_compiler/checker.py | 84 | ||||
| -rw-r--r-- | third_party/closure_compiler/processor.py | 86 | ||||
| -rwxr-xr-x | third_party/closure_compiler/processor_test.py | 10 |
4 files changed, 145 insertions, 37 deletions
diff --git a/third_party/closure_compiler/build/inputs.py b/third_party/closure_compiler/build/inputs.py index 1075995..356c771 100755 --- a/third_party/closure_compiler/build/inputs.py +++ b/third_party/closure_compiler/build/inputs.py @@ -24,7 +24,7 @@ def GetInputs(args): files = set() for file in opts.sources + opts.depends + opts.externs: files.add(file) - files.update(processor.Processor(file).included_files()) + files.update(processor.Processor(file).included_files) return files diff --git a/third_party/closure_compiler/checker.py b/third_party/closure_compiler/checker.py index e5d776f..d45cba1 100755 --- a/third_party/closure_compiler/checker.py +++ b/third_party/closure_compiler/checker.py @@ -3,17 +3,22 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Runs Closure compiler on a JavaScript file to check for errors.""" + import argparse import os -import processor import re import subprocess import sys import tempfile +import processor class Checker(object): - _common_closure_args = [ + """Runs the Closure compiler on a given source file and returns the + success/errors.""" + + _COMMON_CLOSURE_ARGS = [ "--accept_const_keyword", "--language_in=ECMASCRIPT5", "--summary_detail_level=3", @@ -42,9 +47,7 @@ class Checker(object): "--jscomp_off=duplicate", ] - _found_java = False - - _jar_command = [ + _JAR_COMMAND = [ "java", "-jar", "-Xms1024m", @@ -52,6 +55,8 @@ class Checker(object): "-XX:+TieredCompilation" ] + _found_java = False + def __init__(self, verbose=False): current_dir = os.path.join(os.path.dirname(__file__)) self._compiler_jar = os.path.join(current_dir, "lib", "compiler.jar") @@ -77,6 +82,14 @@ class Checker(object): self._clean_up() def _run_command(self, cmd): + """Runs a shell command. + + Args: + cmd: A list of tokens to be joined into a shell command. + + Return: + True if the exit code was 0, else False. + """ cmd_str = " ".join(cmd) self._debug("Running command: " + cmd_str) @@ -85,6 +98,7 @@ class Checker(object): cmd_str, stdout=devnull, stderr=subprocess.PIPE, shell=True) def _check_java_path(self): + """Checks that `java` is on the system path.""" if not self._found_java: proc = self._run_command(["which", "java"]) proc.communicate() @@ -95,24 +109,42 @@ class Checker(object): return self._found_java - def _run_jar(self, jar, args=[]): + def _run_jar(self, jar, args=None): + args = args or [] self._check_java_path() - return self._run_command(self._jar_command + [jar] + args) + return self._run_command(self._JAR_COMMAND + [jar] + args) def _fix_line_number(self, match): + """Changes a line number from /tmp/file:300 to /orig/file:100. + + Args: + match: A re.MatchObject from matching against a line number regex. + + Returns: + The fixed up /file and :line number. + """ real_file = self._processor.get_file_from_line(match.group(1)) return "%s:%d" % (os.path.abspath(real_file.file), real_file.line_number) def _fix_up_error(self, error): + """Filter out irrelevant errors or fix line numbers. + + Args: + error: A Closure compiler error (2 line string with error and source). + + Return: + The fixed up erorr string (blank if it should be ignored). + """ if " first declared in " in error: # Ignore "Variable x first declared in /same/file". return "" - file = self._expanded_file - fixed = re.sub("%s:(\d+)" % file, self._fix_line_number, error) - return fixed.replace(file, os.path.abspath(self._file_arg)) + expanded_file = self._expanded_file + fixed = re.sub("%s:(\d+)" % expanded_file, self._fix_line_number, error) + return fixed.replace(expanded_file, os.path.abspath(self._file_arg)) def _format_errors(self, errors): + """Formats Closure compiler errors to easily spot compiler output.""" errors = filter(None, errors) contents = ("\n" + "## ").join("\n\n".join(errors).splitlines()) return "## " + contents if contents else "" @@ -123,22 +155,38 @@ class Checker(object): tmp_file.write(contents) return tmp_file.name - def check(self, file, depends=[], externs=[]): + def check(self, source_file, depends=None, externs=None): + """Closure compile a file and check for errors. + + Args: + source_file: A file to check. + depends: Other files that would be included with a <script> earlier in + the page. + externs: @extern files that inform the compiler about custom globals. + + Returns: + (exitcode, output) The exit code of the Closure compiler (as a number) + and its output (as a string). + """ + depends = depends or [] + externs = externs or [] + if not self._check_java_path(): return 1, "" - self._debug("FILE: " + file) + self._debug("FILE: " + source_file) - if file.endswith("_externs.js"): - self._debug("Skipping externs: " + file) + if source_file.endswith("_externs.js"): + self._debug("Skipping externs: " + source_file) return - self._file_arg = file + self._file_arg = source_file tmp_dir = tempfile.gettempdir() rel_path = lambda f: os.path.join(os.path.relpath(os.getcwd(), tmp_dir), f) - contents = ['<include src="%s">' % rel_path(f) for f in depends + [file]] + includes = [rel_path(f) for f in depends + [source_file]] + contents = ['<include src="%s">' % i for i in includes] meta_file = self._create_temp_file("\n".join(contents)) self._debug("Meta file: " + meta_file) @@ -147,7 +195,7 @@ class Checker(object): self._debug("Expanded file: " + self._expanded_file) args = ["--js=" + self._expanded_file] + ["--externs=" + e for e in externs] - args_file_content = " " + " ".join(self._common_closure_args + args) + args_file_content = " " + " ".join(self._COMMON_CLOSURE_ARGS + args) self._debug("Args: " + args_file_content.strip()) args_file = self._create_temp_file(args_file_content) @@ -162,7 +210,7 @@ class Checker(object): output = self._format_errors(map(self._fix_up_error, errors)) if runner_cmd.returncode: - self._error("Error in: " + file + ("\n" + output if output else "")) + self._error("Error in: " + source_file + ("\n" + output if output else "")) elif output: self._debug("Output: " + output) diff --git a/third_party/closure_compiler/processor.py b/third_party/closure_compiler/processor.py index 682dce4..be6b797 100644 --- a/third_party/closure_compiler/processor.py +++ b/third_party/closure_compiler/processor.py @@ -2,38 +2,83 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Process Chrome resources (HTML/CSS/JS) to handle <include> and <if> tags.""" + from collections import defaultdict import re import os class LineNumber(object): - def __init__(self, file, line_number): - self.file = file + """A simple wrapper to hold line information (e.g. file.js:32). + + Args: + source_file: A file path. + line_number: The line in |file|. + """ + def __init__(self, source_file, line_number): + self.file = source_file self.line_number = int(line_number) class FileCache(object): + """An in-memory cache to speed up reading the same files over and over. + + Usage: + FileCache.read(path_to_file) + """ + _cache = defaultdict(str) - def _read(self, file): - file = os.path.abspath(file) - self._cache[file] = self._cache[file] or open(file, "r").read() - return self._cache[file] + @classmethod + def read(self, source_file): + """Read a file and return it as a string. + + Args: + source_file: a file to read and return the contents of. - @staticmethod - def read(file): - return FileCache()._read(file) + Returns: + |file| as a string. + """ + abs_file = os.path.abspath(source_file) + self._cache[abs_file] = self._cache[abs_file] or open(abs_file, "r").read() + return self._cache[abs_file] class Processor(object): + """Processes resource files, inlining the contents of <include> tags, removing + <if> tags, and retaining original line info. + + For example + + 1: /* blah.js */ + 2: <if expr="is_win"> + 3: <include src="win.js"> + 4: </if> + + would be turned into: + + 1: /* blah.js */ + 2: + 3: /* win.js */ + 4: alert('Ew; Windows.'); + 5: + + Args: + source_file: A file to process. + + Attributes: + contents: Expanded contents after inlining <include>s and stripping <if>s. + included_files: A list of files that were inlined via <include>. + """ + _IF_TAGS_REG = "</?if[^>]*?>" _INCLUDE_REG = "<include[^>]+src=['\"]([^>]*)['\"]>" - def __init__(self, file): + def __init__(self, source_file): self._included_files = set() self._index = 0 - self._lines = self._get_file(file) + self._lines = self._get_file(source_file) while self._index < len(self._lines): current_line = self._lines[self._index] @@ -50,18 +95,25 @@ class Processor(object): self.contents = "\n".join(l[2] for l in self._lines) # Returns a list of tuples in the format: (file, line number, line contents). - def _get_file(self, file): - lines = FileCache.read(file).splitlines() - return [(file, lnum + 1, line) for lnum, line in enumerate(lines)] + def _get_file(self, source_file): + lines = FileCache.read(source_file).splitlines() + return [(source_file, lnum + 1, line) for lnum, line in enumerate(lines)] - def _include_file(self, file): - self._included_files.add(file) - f = self._get_file(file) + def _include_file(self, source_file): + self._included_files.add(source_file) + f = self._get_file(source_file) self._lines = self._lines[:self._index] + f + self._lines[self._index + 1:] def get_file_from_line(self, line_number): + """Get the original file and line number for an expanded file's line number. + + Args: + line_number: A processed file's line number. + """ line_number = int(line_number) - 1 return LineNumber(self._lines[line_number][0], self._lines[line_number][1]) + @property def included_files(self): + """A list of files that were inlined via <include>.""" return self._included_files diff --git a/third_party/closure_compiler/processor_test.py b/third_party/closure_compiler/processor_test.py index 7d9765e..942d472 100755 --- a/third_party/closure_compiler/processor_test.py +++ b/third_party/closure_compiler/processor_test.py @@ -3,11 +3,15 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +"""Test resources processing, i.e. <if> and <include> tag handling.""" + import unittest from processor import FileCache, Processor, LineNumber class ProcessorTest(unittest.TestCase): + """Test <include> tag processing logic.""" + def __init__(self, *args, **kwargs): unittest.TestCase.__init__(self, *args, **kwargs) self.maxDiff = None @@ -58,6 +62,7 @@ debug(global); self.assertEqual(expected_line.line_number, actual_line.line_number) def testGetFileFromLine(self): + """Verify that inlined files retain their original line info.""" self.assertLineNumber(1, LineNumber("/checked.js", 1)) self.assertLineNumber(5, LineNumber("/checked.js", 5)) self.assertLineNumber(6, LineNumber("/global.js", 1)) @@ -68,11 +73,14 @@ debug(global); self.assertLineNumber(11, LineNumber("/checked.js", 8)) def testIncludedFiles(self): + """Verify that files are tracked correctly as they're inlined.""" self.assertEquals(set(["/global.js", "/debug.js"]), - self._processor.included_files()) + self._processor.included_files) class IfStrippingTest(unittest.TestCase): + """Test that the contents of XML <if> blocks are stripped.""" + def __init__(self, *args, **kwargs): unittest.TestCase.__init__(self, *args, **kwargs) self.maxDiff = None |