Roll typ to v0.9.4.

This picks up several bug fixes in typ, and will most notably add
a new debugging option (-vvv) that will help with debugging some
telemetry issues.

It is also unfortunate that we have two copies of typ checked in
at this point; we should switch to pulling this in via DEPS, but
this is not the time and place for that change.

R=nednguyen@chromium.org
TBR=nednguyen@google.com
BUG=457446

Review URL: https://codereview.chromium.org/1328053002

Cr-Commit-Position: refs/heads/master@{#347539}

17 files changed, 98 insertions, 80 deletions

diff --git a/third_party/typ/README.chromium b/third_party/typ/README.chromium
index b7d6bfc..9ee815d 100644
--- a/third_party/typ/README.chromium
+++ b/third_party/typ/README.chromium
@@ -1,7 +1,7 @@
 Name: typ
 URL: https://github.com/dpranke/typ.git
-Version: 0.8.9
-Revision: 68b64ff805d266c7a249294abc45cc12076aeb9f
+Version: 0.9.4
+Revision: d4caf6b40658b9c43bbfc5794d37ae34d6ff0486
 Security Critical: no
 License: Apache 2.0
 License File: NOT_SHIPPED
diff --git a/third_party/typ/pylintrc b/third_party/typ/pylintrc
index 4abb6c60..35c2a10 100644
--- a/third_party/typ/pylintrc
+++ b/third_party/typ/pylintrc
@@ -65,9 +65,6 @@ disable=C0111,I0011,R0201,R0801,W0141,W0142,W0511,W0703
 # (visual studio) and html
 output-format=text
 
-# Include message's id in output
-include-ids=yes
-
 # Put messages in a separate file for each module / package specified on the
 # command line instead of printing them on stdout. Reports (if any) will be
 # written in a file name "pylint_global.[txt|html]".
@@ -218,7 +215,7 @@ max-locals=32
 max-returns=32
 
 # Maximum number of branch for function / method body
-max-branchs=32
+max-branches=32
 
 # Maximum number of statements in function / method body
 max-statements=65
diff --git a/third_party/typ/run b/third_party/typ/run
index 62558a6..9bea31f 100755
--- a/third_party/typ/run
+++ b/third_party/typ/run
@@ -27,15 +27,12 @@ def call(*args, **kwargs):
 
 
 def main(argv):
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(prog='run')
     parser.add_argument('--no3', action='store_true',
                         help='Do not run the tests under Python 3.')
     parser.add_argument('-v', '--verbose', action='store_true')
     subps = parser.add_subparsers()
 
-    subp = subps.add_parser('build', help='build the package')
-    subp.set_defaults(func=run_build)
-
     subp = subps.add_parser('clean', help='Remove any local files.')
     subp.set_defaults(func=run_clean)
 
@@ -44,30 +41,12 @@ def main(argv):
     subp.set_defaults(func=run_coverage)
     cov.add_arguments(subp)
 
-    subp = subps.add_parser('develop',
-                             help='Install a symlinked package locally.')
-    subp.set_defaults(func=run_develop)
-    subp.add_argument('--system', action='store_true',
-                      help=('Install to the system site-package dir '
-                            'rather than the user\'s (requires root).'))
-
-    subp = subps.add_parser('format',
-                            help='Reformat the source code.')
-    subp.set_defaults(func=run_format)
-
     subp = subps.add_parser('help',
                             help='Get help on a subcommand.')
     subp.add_argument(nargs='?', action='store', dest='subcommand',
                       help='The command to get help for.')
     subp.set_defaults(func=run_help)
 
-    subp = subps.add_parser('install',
-                            help='build the package and install locally.')
-    subp.set_defaults(func=run_install)
-    subp.add_argument('--system', action='store_true',
-                      help=('Install to the system site-package dir '
-                            'rather than the user\'s (requires root).'))
-
     subp = subps.add_parser('lint',
                             help='run lint over the source')
     subp.set_defaults(func=run_lint)
@@ -91,10 +70,6 @@ def main(argv):
     args.func(args)
 
 
-def run_build(args):
-    call([sys.executable, 'setup.py', 'build', '--quiet'])
-
-
 def run_clean(args):
     call(['git', 'clean', '-fxd'])
 
@@ -106,50 +81,37 @@ def run_coverage(args):
         args.source = [os.path.join(repo_dir, 'typ')]
     argv = cov.argv_from_args(args)
     cov_args = [path_to_runner, '-j', '1']
+    print('Running coverage of unit tests for Python 2.7.')
     call(['python', path_to_cov] + argv + cov_args)
     if has_python34:
+        print('Running coverage of unit tests for Python 3.4.')
         call(['python3', path_to_cov] + argv + cov_args)
 
 
-def run_develop(args):
-    call([sys.executable, 'setup.py', 'develop'])
-
-
-def run_format(args):
-    call('autopep8 --in-place *.py */*.py */*/*.py', shell=True)
-
-
 def run_help(args):
     if args.subcommand:
         main([args.subcommand, '--help'])
     main(['--help'])
 
 
-def run_install(args):
-    if args.system:
-        argv = []
-    else:
-        argv = ['--user']
-    call([sys.executable, 'setup.py', 'install'] + argv)
-
-
 def run_lint(args):
     call('pylint --rcfile=pylintrc */*.py */*/*.py', shell=True)
-    call('pep8 *.py */*.py */*/*.py', shell=True)
 
 
 def run_tests(args):
-    # Test that we can run the module directly if typ is in sys.path.
+    print('Testing running the typ module directly if it is in sys.path.')
     call(['python', '-m', 'typ', 'typ.tests.main_test.TestMain.test_basic'])
 
-    # Test that we can run the command line directly if typ is not in sys.path.
+    print('Testing running the runner directly if nothing is in sys.path.')
     home_dir = os.environ['HOME']
     call(['python', path_to_runner, 'typ.tests.main_test.TestMain.test_basic'],
          cwd=home_dir)
 
     # Now run all the tests under Python2 and Python3.
+    print('Running the unit tests under Python 2.')
     call(['python', path_to_runner])
     if has_python34:
+        print('Running the unit tests under Python 3.4.')
         call(['python3', path_to_runner])
 
 
diff --git a/third_party/typ/setup.py b/third_party/typ/setup.py
index 2c32a68..8a684ae 100644
--- a/third_party/typ/setup.py
+++ b/third_party/typ/setup.py
@@ -49,7 +49,7 @@ setup(
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',
-        'License :: OSI Approved :: Apache',
+        'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
diff --git a/third_party/typ/tools/cov.py b/third_party/typ/tools/cov.py
index 8e78fc4..5313c15 100755
--- a/third_party/typ/tools/cov.py
+++ b/third_party/typ/tools/cov.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 import argparse
 import sys
 import textwrap
diff --git a/third_party/typ/typ/arg_parser.py b/third_party/typ/typ/arg_parser.py
index b55dee8..cb51711 100644
--- a/third_party/typ/typ/arg_parser.py
+++ b/third_party/typ/typ/arg_parser.py
@@ -91,6 +91,8 @@ class ArgumentParser(argparse.ArgumentParser):
                               help=('Globs to omit when reporting coverage '
                                     '(defaults to %s).' %
                                     DEFAULT_COVERAGE_OMIT))
+            self.add_argument('--coverage-annotate', action='store_true',
+                              help=('Produce an annotate source report.'))
             self.add_argument('--coverage-show-missing', action='store_true',
                               help=('Show missing line ranges in coverage '
                                     'report.'))
diff --git a/third_party/typ/typ/fakes/host_fake.py b/third_party/typ/typ/fakes/host_fake.py
index 5b9e46d..89c0075 100644
--- a/third_party/typ/typ/fakes/host_fake.py
+++ b/third_party/typ/typ/fakes/host_fake.py
@@ -23,7 +23,7 @@ from typ.host import _TeedStream
 is_python3 = bool(sys.version_info.major == 3)
 
 if is_python3:  # pragma: python3
-    # redefining built-in 'unicode' pylint: disable=W0622
+    # pylint: disable=redefined-builtin,invalid-name
     unicode = str
 
 
diff --git a/third_party/typ/typ/fakes/test_result_server_fake.py b/third_party/typ/typ/fakes/test_result_server_fake.py
index e7b81e9..4caeaf2 100644
--- a/third_party/typ/typ/fakes/test_result_server_fake.py
+++ b/third_party/typ/typ/fakes/test_result_server_fake.py
@@ -24,7 +24,8 @@ if sys.version_info.major == 2:  # pragma: python2
     from SocketServer import TCPServer
 else:  # pragma: python3
     assert sys.version_info.major == 3
-    unicode = str  # pylint: disable=W0622
+    # pylint: disable=redefined-builtin
+    unicode = str
     from http.server import BaseHTTPRequestHandler  # pylint: disable=F0401
     HTTPRequestHandler = BaseHTTPRequestHandler
     from socketserver import TCPServer  # pylint: disable=F0401
diff --git a/third_party/typ/typ/json_results.py b/third_party/typ/typ/json_results.py
index 5e95ed5..807c754 100644
--- a/third_party/typ/typ/json_results.py
+++ b/third_party/typ/typ/json_results.py
@@ -127,7 +127,7 @@ def failed_test_names(results):
     for r in results.results:
         if r.actual == ResultType.Failure:
             names.add(r.name)
-        elif (r.actual == ResultType.Pass and r.name in names):
+        elif r.actual == ResultType.Pass and r.name in names:
             names.remove(r.name)
     return names
 
diff --git a/third_party/typ/typ/pool.py b/third_party/typ/typ/pool.py
index 1afcd1c..6200a8a 100644
--- a/third_party/typ/typ/pool.py
+++ b/third_party/typ/typ/pool.py
@@ -15,6 +15,7 @@
 import copy
 import multiprocessing
 import pickle
+import traceback
 
 from typ.host import Host
 
@@ -98,6 +99,7 @@ class _ProcessPool(object):
         # Instead, we have to hack the innards of multiprocessing. It
         # seems likely that there's a bug somewhere, either in this module or
         # in multiprocessing.
+        # pylint: disable=protected-access
         if self.host.is_python3:  # pragma: python3
             multiprocessing.queues.is_exiting = lambda: True
         else:  # pragma: python2
@@ -142,9 +144,10 @@ class _ProcessPool(object):
         return final_responses
 
     def _handle_error(self, msg):
-        worker_num, ex_str = msg
+        worker_num, tb = msg
         self.erred = True
-        raise Exception("error from worker %d: %s" % (worker_num, ex_str))
+        raise Exception("Error from worker %d (traceback follows):\n%s" %
+                        (worker_num, tb))
 
 
 # 'Too many arguments' pylint: disable=R0913
@@ -168,7 +171,8 @@ def _loop(requests, responses, host, worker_num,
     except KeyboardInterrupt as e:
         responses.put((_MessageType.Interrupt, (worker_num, str(e))))
     except Exception as e:
-        responses.put((_MessageType.Error, (worker_num, str(e))))
+        responses.put((_MessageType.Error,
+                       (worker_num, traceback.format_exc(e))))
 
 
 class _AsyncPool(object):
diff --git a/third_party/typ/typ/runner.py b/third_party/typ/typ/runner.py
index 9872a60..38c0e1c 100644
--- a/third_party/typ/typ/runner.py
+++ b/third_party/typ/typ/runner.py
@@ -20,6 +20,7 @@ import os
 import pdb
 import sys
 import unittest
+import traceback
 
 from collections import OrderedDict
 
@@ -294,8 +295,9 @@ class Runner(object):
                 # TODO: figure out what to do if multiple files are
                 # specified and they don't all have the same correct
                 # top level dir.
-                if h.exists(h.dirname(args.tests[0]), '__init__.py'):
-                    top_dir = h.dirname(args.tests[0])
+                d = h.realpath(h.dirname(args.tests[0]))
+                if h.exists(d, '__init__.py'):
+                    top_dir = d
                 else:
                     top_dir = args.tests[0]
             else:
@@ -503,9 +505,22 @@ class Runner(object):
             self.final_responses.extend(pool.join())
 
     def _print_test_started(self, stats, test_input):
-        if not self.args.quiet and self.args.overwrite:
-            self.update(stats.format() + test_input.name,
-                        elide=(not self.args.verbose))
+        if self.args.quiet:
+            # Print nothing when --quiet was passed.
+            return
+
+        # If -vvv was passed, print when the test is queued to be run.
+        # We don't actually know when the test picked up to run, because
+        # that is handled by the child process (where we can't easily
+        # print things). Otherwise, only print when the test is started
+        # if we know we can overwrite the line, so that we do not
+        # get multiple lines of output as noise (in -vvv, we actually want
+        # the noise).
+        test_start_msg = stats.format() + test_input.name
+        if self.args.verbose > 2:
+            self.update(test_start_msg + ' queued', elide=False)
+        if self.args.overwrite:
+            self.update(test_start_msg, elide=(not self.args.verbose))
 
     def _print_test_finished(self, stats, result):
         stats.add_time()
@@ -615,6 +630,8 @@ class Runner(object):
             cov.combine()
             cov.report(show_missing=self.args.coverage_show_missing,
                        omit=self.args.coverage_omit)
+            if self.args.coverage_annotate:
+                cov.annotate(omit=self.args.coverage_omit)
 
     def _add_trace_event(self, trace, name, start, end):
         event = {
@@ -725,6 +742,7 @@ class _Child(object):
 def _setup_process(host, worker_num, child):
     child.host = host
     child.worker_num = worker_num
+    # pylint: disable=protected-access
 
     if child.coverage:  # pragma: no cover
         import coverage
@@ -745,9 +763,9 @@ def _teardown_process(child):
     e = None
     if child.teardown_fn:
         try:
-          res = child.teardown_fn(child, child.context_after_setup)
+            res = child.teardown_fn(child, child.context_after_setup)
         except Exception as e:
-          pass
+            pass
 
     if child.cov:  # pragma: no cover
         child.cov.stop()
@@ -772,23 +790,33 @@ def _run_one_test(child, test_input):
     # but could come up when testing non-typ code as well.
     h.capture_output(divert=not child.passthrough)
 
+    tb_str = ''
     try:
         orig_skip = unittest.skip
         orig_skip_if = unittest.skipIf
         if child.all:
             unittest.skip = lambda reason: lambda x: x
             unittest.skipIf = lambda condition, reason: lambda x: x
+
         try:
             suite = child.loader.loadTestsFromName(test_name)
-        except Exception:
-            suite = _load_via_load_tests(child, test_name)
+        except Exception as e:
+            try:
+                suite = _load_via_load_tests(child, test_name)
+            except Exception as e:  # pragma: untested
+                suite = []
+                tb_str = traceback.format_exc(e)
     finally:
         unittest.skip = orig_skip
         unittest.skipIf = orig_skip_if
 
     tests = list(suite)
     if len(tests) != 1:
-        err = 'failed to load %s' % test_name
+        err = 'Failed to load %s'
+        if tb_str:  # pragma: untested
+            err += (' (traceback follows):\n  %s' %
+                    '  \n'.join(tb_str.splitlines()))
+
         h.restore_output()
         return Result(test_name, ResultType.Failure, start, 0,
                       child.worker_num, unexpected=True, code=1,
diff --git a/third_party/typ/typ/test_case.py b/third_party/typ/typ/test_case.py
index 5be83ae..d709a57 100644
--- a/third_party/typ/typ/test_case.py
+++ b/third_party/typ/typ/test_case.py
@@ -59,7 +59,7 @@ class MainTestCase(TestCase):
     def make_host(self):
         # If we are ever called by unittest directly, and not through typ,
         # this will probably fail.
-        assert(self.child)
+        assert self.child
         return self.child.host
 
     def call(self, host, argv, stdin, env):
diff --git a/third_party/typ/typ/tests/host_test.py b/third_party/typ/typ/tests/host_test.py
index a06085a..5100a2a2 100644
--- a/third_party/typ/typ/tests/host_test.py
+++ b/third_party/typ/typ/tests/host_test.py
@@ -130,9 +130,9 @@ class TestHost(unittest.TestCase):
         self.assertEqual(h.basename('foo.txt'), 'foo.txt')
         self.assertEqual(h.basename('foo/bar.txt'), 'bar.txt')
 
-    def test_mktempfile(self, delete=False):
+    def test_mktempfile(self, delete=False): # pylint: disable=unused-argument
         h = self.host()
-        f= h.mktempfile()
+        f = h.mktempfile()
         f.close()
         self.assertNotEqual(f.name, None)
 
diff --git a/third_party/typ/typ/tests/main_test.py b/third_party/typ/typ/tests/main_test.py
index cc690d5..7a2d179 100644
--- a/third_party/typ/typ/tests/main_test.py
+++ b/third_party/typ/typ/tests/main_test.py
@@ -29,7 +29,7 @@ from typ.fakes import test_result_server_fake
 is_python3 = bool(sys.version_info.major == 3)
 
 if is_python3:  # pragma: python3
-    # redefining built-in 'unicode' pylint: disable=W0622
+    # pylint: disable=redefined-builtin,invalid-name
     unicode = str
 
 d = textwrap.dedent
@@ -265,8 +265,9 @@ class TestCli(test_case.MainTestCase):
         self.assertNotIn('Retrying failed tests (attempt #2 of 3)', out)
         self.assertIn('1 test run, 0 failures.\n', out)
         results = json.loads(files['full_results.json'])
-        self.assertEqual(results['tests'][
-            'fail_then_pass_test']['FPTest']['test_count']['actual'],
+        self.assertEqual(
+            results['tests'][
+                'fail_then_pass_test']['FPTest']['test_count']['actual'],
             'FAIL PASS')
 
     def test_failures_are_not_elided(self):
@@ -540,9 +541,9 @@ class TestCli(test_case.MainTestCase):
 
     def test_timing(self):
         self.check(['-t'], files=PASS_TEST_FILES, ret=0, err='',
-                   rout=('\[1/1\] pass_test.PassingTest.test_pass passed '
-                         '\d+.\d+s\n'
-                         '1 test run in \d+.\d+s, 0 failures.'))
+                   rout=(r'\[1/1\] pass_test.PassingTest.test_pass passed '
+                         r'\d+.\d+s\n'
+                         r'1 test run in \d+.\d+s, 0 failures.'))
 
     def test_test_results_server(self):
         server = test_result_server_fake.start()
@@ -596,11 +597,11 @@ class TestCli(test_case.MainTestCase):
                     '--test-type', 'typ_tests',
                     '--metadata', 'foo=bar'],
                    files=PASS_TEST_FILES, ret=1, err='',
-                   rout=('\[1/1\] pass_test.PassingTest.test_pass passed\n'
+                   rout=(r'\[1/1\] pass_test.PassingTest.test_pass passed\n'
                          '1 test run, 0 failures.\n'
                          'Uploading the JSON results raised .*\n'))
 
-    def test_verbose(self):
+    def test_verbose_2(self):
         self.check(['-vv', '-j', '1', 'output_test.PassTest'],
                    files=OUTPUT_TEST_FILES, ret=0,
                    out=d("""\
@@ -611,6 +612,19 @@ class TestCli(test_case.MainTestCase):
                          2 tests run, 0 failures.
                          """), err='')
 
+    def test_verbose_3(self):
+        self.check(['-vvv', '-j', '1', 'output_test.PassTest'],
+                   files=OUTPUT_TEST_FILES, ret=0,
+                   out=d("""\
+                         [0/2] output_test.PassTest.test_err queued
+                         [1/2] output_test.PassTest.test_err passed:
+                           hello on stderr
+                         [1/2] output_test.PassTest.test_out queued
+                         [2/2] output_test.PassTest.test_out passed:
+                           hello on stdout
+                         2 tests run, 0 failures.
+                         """), err='')
+
     def test_version(self):
         self.check('--version', ret=0, out=(VERSION + '\n'))
 
diff --git a/third_party/typ/typ/tests/pool_test.py b/third_party/typ/typ/tests/pool_test.py
index fdb3f39..e4b8bd7 100644
--- a/third_party/typ/typ/tests/pool_test.py
+++ b/third_party/typ/typ/tests/pool_test.py
@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
+import unittest
+
 from typ import test_case
 from typ.host import Host
 from typ.pool import make_pool, _MessageType, _ProcessPool, _loop
@@ -96,6 +99,7 @@ class TestPool(test_case.TestCase):
         pool.join()
         self.assertEqual(len(pool.discarded_responses), 1)
 
+    @unittest.skipIf(sys.version_info.major == 3, 'fails under python3')
     def test_join_gets_an_error(self):
         host = Host()
         pool = make_pool(host, 2, _error, None, _stub, _stub)
@@ -132,6 +136,7 @@ class TestPool(test_case.TestCase):
         self.run_through_loop(pool=pool)
         pool.join()
 
+    @unittest.skipIf(sys.version_info.major == 3, 'fails under python3')
     def test_loop_get_raises_error(self):
         pool = self.run_through_loop(_error)
         self.assertRaises(Exception, pool.get)
diff --git a/third_party/typ/typ/tests/runner_test.py b/third_party/typ/typ/tests/runner_test.py
index e03d611..9583149 100644
--- a/third_party/typ/typ/tests/runner_test.py
+++ b/third_party/typ/typ/tests/runner_test.py
@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import tempfile
+import unittest
 
 from textwrap import dedent as d
 
@@ -43,6 +45,7 @@ class RunnerTests(TestCase):
         ret, _, _ = r.run()
         self.assertEqual(ret, 0)
 
+    @unittest.skipIf(sys.version_info.major == 3, 'fails under python3')
     def test_exception_in_teardown(self):
         r = Runner()
         r.args.tests = ['typ.tests.runner_test.ContextTests']
diff --git a/third_party/typ/typ/version.py b/third_party/typ/typ/version.py
index 073f6fe..edf14a0 100644
--- a/third_party/typ/typ/version.py
+++ b/third_party/typ/typ/version.py
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-VERSION = '0.8.9'
+VERSION = '0.9.4'