- Collect stderr separately from stdout and append it to the layout test output.

- Write #EOF to stderr at the end of each test so it doesn't block.
- Disable "DidStopLoading called while not loading" and "DidStartLoading called
while loading" errors temporarily so the ~165 tests that
give those errors can still be run while we fix the underlying problem.
- Update Windows test list accordingly.  stderr was always being included on
Mac, so these tests shouldn't have changed state there.

BUG=3937
TEST=covered by layout tests
Review URL: http://codereview.chromium.org/8778

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@5183 0039d316-1c4b-4281-b951-d872f2087c98

5 files changed, 243 insertions, 8 deletions

diff --git a/webkit/tools/layout_tests/layout_package/test_shell_thread.py b/webkit/tools/layout_tests/layout_package/test_shell_thread.py
index c21072d..f8864f0 100644
--- a/webkit/tools/layout_tests/layout_package/test_shell_thread.py
+++ b/webkit/tools/layout_tests/layout_package/test_shell_thread.py
@@ -78,6 +78,19 @@ def ProcessOutput(proc, filename, test_uri, test_types, test_args):
       outlines.append(line)
     line = proc.stdout.readline()
 
+  # If we had any stderr output, append that. This is not ideal, but at least
+  # it'll catch errors.
+  line = proc.stderr.readline()
+  while line.rstrip() != "#EOF":
+    # TODO(pamg): We suppress this stderr message temporarily so we can run
+    # the tests deterministically until someone has a chance to fix the
+    # underlying problem.
+    # See http://code.google.com/p/chromium/issues/detail?id=4285
+    if not line.endswith('alias ISO-8859-8-I maps to ISO-8859-8-I already, but'
+                         ' someone is trying to make it map to ISO-8859-8'):
+      outlines.append(line)
+    line = proc.stderr.readline()
+
   # Check the output and save the results.
   for test_type in test_types:
     new_failures = test_type.CompareOutput(filename, proc,
@@ -94,10 +107,13 @@ def ProcessOutput(proc, filename, test_uri, test_types, test_args):
 def StartTestShell(binary, args):
   """Returns the process for a new test_shell started in layout-tests mode."""
   cmd = [binary, '--layout-tests'] + args
+  # We'd really like to combine stderr into stdout here by setting stderr to
+  # subprocess.STDOUT, but on Windows that's just dropping stderr output on
+  # the floor, at least in Python 2.4.1.
   return subprocess.Popen(cmd,
                           stdin=subprocess.PIPE,
                           stdout=subprocess.PIPE,
-                          stderr=subprocess.STDOUT)
+                          stderr=subprocess.PIPE)
 
 
 class SingleTestThread(threading.Thread):
diff --git a/webkit/tools/layout_tests/test_lists/win/tests_fixable.txt b/webkit/tools/layout_tests/test_lists/win/tests_fixable.txt
index 95cc756..effe958 100644
--- a/webkit/tools/layout_tests/test_lists/win/tests_fixable.txt
+++ b/webkit/tools/layout_tests/test_lists/win/tests_fixable.txt
@@ -41,6 +41,192 @@ V8 # LayoutTests/http/tests/xmlhttprequest/upload-onprogress-event.html = FAIL
 V8 # LayoutTests/http/tests/xmlhttprequest/upload-progress-events.html = FAIL
 
 // -----------------------------------------------------------------
+// DidStartLoading AND DidStopLoading
+// -----------------------------------------------------------------
+
+// These tests would fail if the error messages "DidStartLoading called while
+// loading" and "DidStopLoading called while not loading" in
+// test_webview_delegate.cc were switched back from DLOG to LOG. With those
+// messages disabled they are expected to pass, but they're listed here since
+// the underlying bug needs fixing (at which point the messages should be
+// re-enabled).
+// See http://code.google.com/p/chromium/issues/detail?id=3937
+
+V8 # LayoutTests/dom/html/level2/html/HTMLFormElement10.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement01.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement02.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement03.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement04.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement05.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement06.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement07.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement08.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameElement09.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameSetElement01.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLFrameSetElement02.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement01.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement02.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement03.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement04.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement05.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement06.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement07.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement08.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement09.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement10.html = PASS
+V8 # LayoutTests/dom/html/level2/html/HTMLIFrameElement11.html = PASS
+V8 # LayoutTests/fast/css/acid2-pixel.html = PASS
+V8 # LayoutTests/fast/css/acid2.html = PASS
+V8 # LayoutTests/fast/css/target-fragment-match.html = PASS
+V8 # LayoutTests/fast/dom/Document/early-document-access.html = PASS
+V8 # LayoutTests/fast/dom/Document/open-with-pending-load.html = PASS
+V8 # LayoutTests/fast/dom/Element/offsetLeft-offsetTop-body-quirk.html = PASS
+V8 # LayoutTests/fast/dom/HTMLObjectElement/object-as-frame.html = PASS
+V8 # LayoutTests/fast/dom/Range/acid3-surround-contents.html = PASS
+V8 # LayoutTests/fast/dom/SelectorAPI/resig-SelectorsAPI-test.xhtml = PASS
+V8 # LayoutTests/fast/dom/Window/window-appendages-cleared.html = PASS
+V8 # LayoutTests/fast/dom/Window/window-early-properties.html = PASS
+V8 # LayoutTests/fast/dom/css-cached-import-rule.html = PASS
+V8 # LayoutTests/fast/dom/exception-no-frame-timeout-crash.html = PASS
+V8 # LayoutTests/fast/dom/location-hash.html = PASS
+V8 # LayoutTests/fast/dom/null-document-location-put-crash.html = PASS
+V8 # LayoutTests/fast/dom/null-document-window-open-crash.html = PASS
+V8 # LayoutTests/fast/dom/null-document-xmlhttprequest-open.html = PASS
+V8 # LayoutTests/fast/dom/replaceChild.html = PASS
+V8 # LayoutTests/fast/dom/wrapper-classes.html = PASS
+V8 # LayoutTests/fast/encoding/percent-escaping.html = PASS
+V8 # LayoutTests/fast/events/anchor-image-scrolled-x-y.html = PASS
+V8 # LayoutTests/fast/events/drag-in-frames.html = PASS
+V8 # LayoutTests/fast/events/focusingUnloadedFrame.html = PASS
+V8 # LayoutTests/fast/events/input-image-scrolled-x-y.html = PASS
+V8 # LayoutTests/fast/events/mouseout-dead-subframe.html = PASS
+V8 # LayoutTests/fast/events/mouseover-mouseout2.html = PASS
+V8 # LayoutTests/fast/events/onload-webkit-before-webcore.html = PASS
+V8 # LayoutTests/fast/events/onloadFrameCrash.html = PASS
+V8 # LayoutTests/fast/events/onunload-window-property.html = PASS
+V8 # LayoutTests/fast/events/onunload.html = PASS
+V8 # LayoutTests/fast/events/open-window-from-another-frame.html = PASS
+V8 # LayoutTests/fast/events/simulated-key-state.html = PASS
+V8 # LayoutTests/fast/forms/access-key.html = PASS
+V8 # LayoutTests/fast/forms/button-state-restore.html = PASS
+V8 # LayoutTests/fast/forms/document-write.html = PASS
+V8 # LayoutTests/fast/forms/empty-get.html = PASS
+V8 # LayoutTests/fast/forms/form-and-frame-interaction-retains-values.html = PASS
+V8 # LayoutTests/fast/forms/form-data-encoding-2.html = PASS
+V8 # LayoutTests/fast/forms/form-data-encoding-normalization-overrun.html = PASS
+V8 # LayoutTests/fast/forms/form-data-encoding.html = PASS
+V8 # LayoutTests/fast/forms/form-get-multipart.html = PASS
+V8 # LayoutTests/fast/forms/form-get-multipart2.html = PASS
+V8 # LayoutTests/fast/forms/form-get-multipart3.html = PASS
+V8 # LayoutTests/fast/forms/form-post-urlencoded.html = PASS
+V8 # LayoutTests/fast/forms/missing-action.html = PASS
+V8 # LayoutTests/fast/forms/search-display-none-cancel-button.html = PASS
+V8 # LayoutTests/fast/forms/submit-nil-value-field-assert.html = PASS
+V8 # LayoutTests/fast/forms/submit-with-base.html = PASS
+V8 # LayoutTests/fast/forms/textarea-appearance-wrap.html = PASS
+V8 # LayoutTests/fast/forms/textarea-hard-linewrap-empty.html = PASS
+V8 # LayoutTests/fast/forms/textarea-hard-linewrap.html = PASS
+V8 # LayoutTests/fast/forms/textarea-linewrap-dynamic.html = PASS
+V8 # LayoutTests/fast/forms/textarea-setvalue-submit.html = PASS
+V8 # LayoutTests/fast/frames/crash-removed-iframe.html = PASS
+V8 # LayoutTests/fast/frames/empty-frame-document.html = PASS
+V8 # LayoutTests/fast/frames/frame-display-none-focus.html = PASS
+V8 # LayoutTests/fast/frames/frame-limit.html = PASS
+V8 # LayoutTests/fast/frames/iframe-option-crash.xhtml = PASS
+V8 # LayoutTests/fast/frames/onload-remove-iframe-crash.html = PASS
+V8 # LayoutTests/fast/frames/viewsource-plain-text-tags.html = PASS
+V8 # LayoutTests/fast/html/empty-fragment-id-goto-top.html = PASS
+V8 # LayoutTests/fast/leaks/002.html = PASS
+V8 # LayoutTests/fast/loader/cancel-load-during-port-block-timer.html = PASS
+V8 # LayoutTests/fast/loader/data-url-encoding-html.html = PASS
+V8 # LayoutTests/fast/loader/data-url-encoding-svg.html = PASS
+V8 # LayoutTests/fast/loader/early-load-cancel.html = PASS
+V8 # LayoutTests/fast/loader/empty-ref-versus-no-ref.html = PASS
+V8 # LayoutTests/fast/loader/onunload-form-submit-crash-2.html = PASS
+V8 # LayoutTests/fast/loader/onunload-form-submit-crash.html = PASS
+V8 # LayoutTests/fast/overflow/scroll-vertical-not-horizontal.html = PASS
+V8 # LayoutTests/fast/table/giantCellspacing.html = PASS
+V8 # LayoutTests/fast/text/reset-drag-on-mouse-down.html = PASS
+V8 # LayoutTests/fast/xsl/transform-xhr-doc.xhtml = PASS
+V8 # LayoutTests/http/tests/loading/bad-scheme-subframe.html = PASS
+V8 # LayoutTests/http/tests/loading/bad-server-subframe.html = PASS
+V8 # LayoutTests/http/tests/loading/text-content-type-with-binary-extension.html = PASS
+V8 # LayoutTests/http/tests/misc/acid2-pixel.html = PASS
+V8 # LayoutTests/http/tests/misc/acid2.html = PASS
+V8 # LayoutTests/http/tests/misc/isindex-formdata.html = PASS
+V8 # LayoutTests/http/tests/misc/object-image-error-with-onload.html = PASS
+V8 # LayoutTests/http/tests/misc/onload-remove-iframe-crash-2.html = PASS
+V8 # LayoutTests/http/tests/misc/post-submit-button.html = PASS
+V8 # LayoutTests/http/tests/misc/redirect-to-about-blank.html = PASS
+V8 # LayoutTests/http/tests/misc/window-dot-stop.html = PASS
+V8 # LayoutTests/http/tests/misc/window-open-then-write.html = PASS
+V8 # LayoutTests/http/tests/multipart/invalid-image-data-standalone.html = PASS
+V8 # LayoutTests/http/tests/navigation/anchor-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/anchor-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/anchor-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/anchor-subframeload.html = PASS
+V8 # LayoutTests/http/tests/navigation/document-location-click-timeout.html = PASS
+V8 # LayoutTests/http/tests/navigation/error404-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/error404-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/error404-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/error404-subframeload.html = PASS
+V8 # LayoutTests/http/tests/navigation/fallback-anchor-reload.html = PASS
+V8 # LayoutTests/http/tests/navigation/javascriptlink-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/javascriptlink-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/javascriptlink-subframeload.html = PASS
+V8 # LayoutTests/http/tests/navigation/metaredirect-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/metaredirect-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/metaredirect-subframeload.html = PASS
+V8 # LayoutTests/http/tests/navigation/multiple-back-forward-entries.html = PASS
+V8 # LayoutTests/http/tests/navigation/redirect-cycle.html = PASS
+V8 # LayoutTests/http/tests/navigation/redirect302-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/relativeanchor-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/relativeanchor-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/reload-subframe-frame.html = PASS
+V8 # LayoutTests/http/tests/navigation/reload-subframe-iframe.html = PASS
+V8 # LayoutTests/http/tests/navigation/reload-subframe-object.html = PASS
+V8 # LayoutTests/http/tests/navigation/restore-form-state-https.html = PASS
+V8 # LayoutTests/http/tests/navigation/slowmetaredirect-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/slowtimerredirect-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-frames-loadsame.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-loadsame.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-reload.html = PASS
+V8 # LayoutTests/http/tests/navigation/success200-subframeload.html = PASS
+V8 # LayoutTests/http/tests/navigation/target-frame-from-window.html = PASS
+V8 # LayoutTests/http/tests/navigation/timerredirect-basic.html = PASS
+V8 # LayoutTests/http/tests/navigation/timerredirect-frames.html = PASS
+V8 # LayoutTests/http/tests/navigation/timerredirect-goback.html = PASS
+V8 # LayoutTests/http/tests/navigation/timerredirect-subframeload.html = PASS
+V8 # LayoutTests/http/tests/security/cross-frame-access-custom.html = PASS
+V8 # LayoutTests/http/tests/security/cross-frame-access-frames.html = PASS
+V8 # LayoutTests/http/tests/security/cross-frame-access-name-getter.html = PASS
+V8 # LayoutTests/http/tests/xmlhttprequest/event-target.html = PASS
+V8 # LayoutTests/http/tests/xmlhttprequest/frame-load-cancelled-abort.html = PASS
+V8 # LayoutTests/plugins/geturl-replace-query.html = PASS
+V8 # LayoutTests/plugins/open-and-close-window-with-plugin.html = PASS
+V8 # LayoutTests/svg/custom/frame-getSVGDocument.html = PASS
+V8 # LayoutTests/svg/dynamic-updates/SVGAElement-dom-href-attr.html = PASS
+V8 # LayoutTests/svg/dynamic-updates/SVGAElement-dom-target-attr.html = PASS
+V8 # LayoutTests/svg/dynamic-updates/SVGAElement-svgdom-href-prop.html = PASS
+V8 # LayoutTests/svg/dynamic-updates/SVGAElement-svgdom-target-prop.html = PASS
+V8 # LayoutTests/tables/mozilla/bugs/bug137388-1.html = PASS
+V8 # LayoutTests/tables/mozilla/bugs/bug137388-2.html = PASS
+V8 # chrome/fast/dom/navigator-cookieEnabled-no-crash.html = PASS
+V8 # chrome/fast/dom/window-collection-length-no-crash.html = PASS
+V8 # pending/css/overflow_hidden.html = PASS
+V8 # pending/dom/html/level2/html/HTMLFrameElement09.html = PASS
+V8 # pending/dom/html/level2/html/HTMLIFrameElement11.html = PASS
+V8 # pending/fast/events/tabindex-focus-blur-all.html = PASS
+V8 # pending/fast/frames/crash-remove-onload-from-src.html = PASS
+V8 # pending/fast/history/history_reload.html = PASS
+V8 # pending/fast/js/const.html = PASS
+V8 # pending/fast/parser/bad-xml-entity.xml = PASS
+V8 # pending/http/tests/security/cross-frame-access-frames.html = PASS
+
+// -----------------------------------------------------------------
 // TEXT
 // -----------------------------------------------------------------
 
@@ -158,6 +344,10 @@ V8 # pending/fast/forms/textarea-scrollbar-height.html = FAIL
 // Other
 // -----------------------------------------------------------------
 
+// Bug http://code.google.com/p/chromium/issues/detail?id=3942
+// Stderr output reports error moving/renaming cache folder
+V8 # LayoutTests/http/tests/security/cross-frame-access-callback-explicit-domain-ALLOW.html = FAIL
+
 // Bug 865472, this should just need proper pixel test rebaselining.
 // Also skipped by Apple on Windows (rdar://5723191).
 V8 # LayoutTests/http/tests/navigation/javascriptlink-frames.html = FAIL
diff --git a/webkit/tools/test_shell/mac/test_webview_delegate.mm b/webkit/tools/test_shell/mac/test_webview_delegate.mm
index febbc7c..909c156 100644
--- a/webkit/tools/test_shell/mac/test_webview_delegate.mm
+++ b/webkit/tools/test_shell/mac/test_webview_delegate.mm
@@ -85,7 +85,13 @@ void TestWebViewDelegate::OpenURL(WebView* webview, const GURL& url,
 
 void TestWebViewDelegate::DidStartLoading(WebView* webview) {
   if (page_is_loading_) {
-    LOG(ERROR) << "DidStartLoading called while loading";
+    // When we started including stderr in layout-test output, a number of
+    // number of tests began failing. The vast majority of the new failures
+    // were due to either this error message or the one in DidStopLoading.
+    // This is being disabled temporarily so we can keep running the tests.
+    // See http://code.google.com/p/chromium/issues/detail?id=3937
+    // TODO(pamg): Remove this when the underlying bug is fixed.
+    //LOG(ERROR) << "DidStartLoading called while loading";
     return;
   }
   page_is_loading_ = true;
@@ -93,7 +99,13 @@ void TestWebViewDelegate::DidStartLoading(WebView* webview) {
 
 void TestWebViewDelegate::DidStopLoading(WebView* webview) {
   if (!page_is_loading_) {
-    LOG(ERROR) << "DidStopLoading called while not loading";
+    // When we started including stderr in layout-test output, a number of
+    // number of tests began failing. The vast majority of the new failures
+    // were due to either this error message or the one in DidStartLoading.
+    // This is being disabled temporarily so we can keep running the tests.
+    // See http://code.google.com/p/chromium/issues/detail?id=3937
+    // TODO(pamg): Remove this when the underlying bug is fixed.
+    //LOG(ERROR) << "DidStopLoading called while not loading";
     return;
   }
   page_is_loading_ = false;
diff --git a/webkit/tools/test_shell/test_shell_win.cc b/webkit/tools/test_shell/test_shell_win.cc
index 0ca297ab..0e88328 100644
--- a/webkit/tools/test_shell/test_shell_win.cc
+++ b/webkit/tools/test_shell/test_shell_win.cc
@@ -250,9 +250,12 @@ bool TestShell::RunFileTest(const char *filename, const TestParams& params) {
       std::string md5sum = DumpImage(webFrame, params.pixel_file_name);
       printf("#MD5:%s\n", md5sum.c_str());
     }
-    if (dumped_anything)
+    if (dumped_anything) {
       printf("#EOF\n");
+      fprintf(stderr, "#EOF\n");
+    }
     fflush(stdout);
+    fflush(stderr);
   }
 
   return true;
@@ -384,9 +387,11 @@ unsigned int __stdcall WatchDogThread(void *arg) {
     // Print a warning to be caught by the layout-test script.
     // Note: the layout test driver may or may not recognize
     // this as a timeout.
-    puts("#TEST_TIMED_OUT\n");
-    puts("#EOF\n");
+    printf("#TEST_TIMED_OUT\n");
+    printf("#EOF\n");
+    fprintf(stderr, "#EOF\n");
     fflush(stdout);
+    fflush(stderr);
     TerminateProcess(GetCurrentProcess(), 0);
   }
   // Finished normally.
diff --git a/webkit/tools/test_shell/test_webview_delegate.cc b/webkit/tools/test_shell/test_webview_delegate.cc
index 4eff32f..a7ee633 100644
--- a/webkit/tools/test_shell/test_webview_delegate.cc
+++ b/webkit/tools/test_shell/test_webview_delegate.cc
@@ -82,7 +82,13 @@ void TestWebViewDelegate::OpenURL(WebView* webview, const GURL& url,
 
 void TestWebViewDelegate::DidStartLoading(WebView* webview) {
   if (page_is_loading_) {
-    LOG(ERROR) << "DidStartLoading called while loading";
+    // When we started including stderr in layout-test output, a number of
+    // number of tests began failing. The vast majority of the new failures
+    // were due to either this error message or the one in DidStopLoading.
+    // This is being disabled temporarily so we can keep running the tests.
+    // See http://code.google.com/p/chromium/issues/detail?id=3937
+    // TODO(pamg): Remove this when the underlying bug is fixed.
+    //LOG(ERROR) << "DidStartLoading called while loading";
     return;
   }
   page_is_loading_ = true;
@@ -90,7 +96,13 @@ void TestWebViewDelegate::DidStartLoading(WebView* webview) {
 
 void TestWebViewDelegate::DidStopLoading(WebView* webview) {
   if (!page_is_loading_) {
-    LOG(ERROR) << "DidStopLoading called while not loading";
+    // When we started including stderr in layout-test output, a number of
+    // number of tests began failing. The vast majority of the new failures
+    // were due to either this error message or the one in DidStartLoading.
+    // This is being disabled temporarily so we can keep running the tests.
+    // See http://code.google.com/p/chromium/issues/detail?id=3937
+    // TODO(pamg): Remove this when the underlying bug is fixed.
+    //LOG(ERROR) << "DidStopLoading called while not loading";
     return;
   }
   page_is_loading_ = false;