Use apache on the Mac on V8-Latest builders as an experiment

to see if http test flakiness goes away.
Review URL: http://codereview.chromium.org/437063

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@33130 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 167 insertions, 3 deletions

diff --git a/webkit/tools/layout_tests/layout_package/apache_http_server.py b/webkit/tools/layout_tests/layout_package/apache_http_server.py
new file mode 100644
index 0000000..1950c6b
--- /dev/null
+++ b/webkit/tools/layout_tests/layout_package/apache_http_server.py
@@ -0,0 +1,144 @@
+# Copyright (c) 2009 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A class to start/stop the apache http server used by layout tests."""
+
+import logging
+import optparse
+import os
+import subprocess
+import sys
+import time
+
+import google.httpd_utils
+
+import path_utils
+import platform_utils
+
+class LayoutTestApacheHttpd(object):
+  _PORTS = [
+    {'port': 8000},
+    {'port': 8080},
+    {'port': 8443, 'is_ssl': True}
+  ]
+
+  def __init__(self, output_dir):
+    """Args:
+      output_dir: the absolute path to the layout test result directory
+    """
+    self._output_dir = output_dir
+    self._httpd_proc = None
+    path_utils.MaybeMakeDirectory(output_dir)
+
+    # The upstream .conf file assumed the existence of /tmp/WebKit for placing
+    # apache files like the lock file there.
+    path_utils.MaybeMakeDirectory(os.path.join("/tmp", "WebKit"))
+
+    test_dir = path_utils.PathFromBase('third_party', 'WebKit',
+        'LayoutTests')
+    js_test_resources_dir = os.path.join(test_dir, "fast", "js", "resources")
+    mime_types_path = os.path.join(test_dir, "http", "conf", "mime.types")
+    cert_file = os.path.join(test_dir, "http", "conf", "webkit-httpd.pem")
+
+    cmd = [os.path.join("/usr", "sbin", "httpd"),
+        '-f', self._GetApacheConfigFilePath(test_dir, output_dir),
+        '-C', "DocumentRoot %s" % os.path.join(test_dir, "http", "tests"),
+        '-c', "Alias /js-test-resources %s" % js_test_resources_dir,
+        '-C', "Listen %s" % "127.0.0.1:8000",
+        '-C', "Listen %s" % "127.0.0.1:8081",
+        '-c', "TypesConfig \"%s\"" % mime_types_path,
+        '-c', "CustomLog \"%s/access_log.txt\" common" % output_dir,
+        '-c', "ErrorLog \"%s/error_log.txt\"" % output_dir,
+        '-C', "User \"%s\"" % os.environ.get("USERNAME",
+                                             os.environ.get("USER", "")),
+        '-c', "SSLCertificateFile %s" % cert_file,
+         ]
+
+    self._start_cmd = cmd
+
+  def _GetApacheConfigFilePath(self, test_dir, output_dir):
+    """Returns the path to the apache config file to use.
+    Args:
+      test_dir: absolute path to the LayoutTests directory.
+      output_dir: absolute path to the layout test results directory.
+    """
+    conf_file_name = "apache2-httpd.conf"
+    httpd_config = os.path.join(test_dir, "http", "conf", conf_file_name)
+    httpd_config_copy = os.path.join(output_dir, conf_file_name)
+    main_document_root = os.path.join(path_utils.LayoutTestsDir(),
+        "LayoutTests", "http", "tests")
+    chrome_document_root = path_utils.PathFromBase('webkit', 'data',
+        'layout_tests')
+    httpd_conf = open(httpd_config).read()
+    # TODO(ojan): Instead of writing an extra file, checkin a conf file
+    # upstream. Or, even better, upstream/delete all our chrome http tests so we
+    # don't need this special-cased DocumentRoot and then just use the upstream
+    # conf file.
+    httpd_conf = (httpd_conf +
+        self._GetVirtualHostConfig(chrome_document_root, 8081))
+
+    f = open(httpd_config_copy, 'wb')
+    f.write(httpd_conf)
+    f.close()
+
+    return httpd_config_copy
+
+  def _GetVirtualHostConfig(self, document_root, port, ssl=False):
+    """Returns a <VirtualHost> directive block for an httpd.conf file.  It will
+    listen to 127.0.0.1 on each of the given port.
+    """
+    return '\n'.join(('<VirtualHost 127.0.0.1:%s>' % port,
+                      'DocumentRoot %s' % document_root,
+                      ssl and 'SSLEngine On' or '',
+                      '</VirtualHost>', ''))
+
+  def _IsServerRunningOnAllPorts(self):
+    """Returns whether the server is running on all the desired ports."""
+    for mapping in self._PORTS:
+      url = 'http%s://127.0.0.1:%d/' % ('is_ssl' in mapping and 's' or '',
+                                        mapping['port'])
+      if not google.httpd_utils.UrlIsAlive(url):
+        return False
+
+    return True
+
+  def _StartHttpdProcess(self):
+    """Starts the httpd process and returns whether there were errors."""
+    self._httpd_proc = subprocess.Popen(self._start_cmd, stderr=subprocess.PIPE)
+    if len(self._httpd_proc.stderr.read()):
+      return False
+    return True
+
+  def _WaitForAction(self, action):
+    """Repeat the action for 20 seconds or until it succeeds. Returns whether
+    it succeeded."""
+    succeeded = False
+
+    start_time = time.time()
+    # Give apache up to 20 seconds to start up.
+    while time.time() - start_time < 20 and not succeeded:
+      time.sleep(0.1)
+      succeeded = action()
+
+    return succeeded
+
+  def Start(self):
+    """Starts the apache http server."""
+    # Stop any currently running servers.
+    self.Stop()
+
+    logging.debug("Starting apache http server")
+    server_started = self._WaitForAction(self._StartHttpdProcess)
+    if server_started:
+      server_started = self._WaitForAction(self._IsServerRunningOnAllPorts)
+
+    if server_started:
+      logging.debug("Server successfully started")
+    else:
+      raise google.httpd_utils.HttpdNotStarted('Failed to start httpd')
+
+  def Stop(self):
+    """Stops the apache http server."""
+    logging.debug("Shutting down any running http servers")
+    path_utils.ShutDownHTTPServer(self._httpd_proc)
diff --git a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
index dd9b190..4da1731 100644
--- a/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
+++ b/webkit/tools/layout_tests/layout_package/platform_utils_mac.py
@@ -104,6 +104,7 @@ def ShutDownHTTPServer(server_process):
     #   killall: illegal option -- T
     # Use of the earlier -TERM placement is just fine on 10.5.
     subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), 'lighttpd'])
+    subprocess.call(['killall', '-TERM', '-u', os.getenv('USER'), 'httpd'])
   else:
     os.kill(server_process.pid, signal.SIGTERM)
 
diff --git a/webkit/tools/layout_tests/run_webkit_tests.py b/webkit/tools/layout_tests/run_webkit_tests.py
index 106b1c4..95d0fc6 100755
--- a/webkit/tools/layout_tests/run_webkit_tests.py
+++ b/webkit/tools/layout_tests/run_webkit_tests.py
@@ -35,6 +35,7 @@ import sys
 import time
 import traceback
 
+from layout_package import apache_http_server
 from layout_package import compare_failures
 from layout_package import test_expectations
 from layout_package import http_server
@@ -130,7 +131,12 @@ class TestRunner:
     """
     self._options = options
 
-    self._http_server = http_server.Lighttpd(options.results_directory)
+    if options.use_apache:
+      self._http_server = apache_http_server.LayoutTestApacheHttpd(
+          options.results_directory)
+    else:
+      self._http_server = http_server.Lighttpd(options.results_directory)
+
     self._websocket_server = websocket_server.PyWebSocket(
         options.results_directory)
     # disable wss server. need to install pyOpenSSL on buildbots.
@@ -1072,6 +1078,10 @@ def main(options, args):
     else:
       options.target = "Release"
 
+  if not options.use_apache:
+    options.use_apache = (sys.platform == 'darwin' and options.builder_name and
+        options.builder_name.find("(V8-Latest)") != -1)
+
   if options.results_directory.startswith("/"):
     # Assume it's an absolute path and normalize.
     options.results_directory = path_utils.GetAbsolutePath(
@@ -1247,6 +1257,9 @@ if '__main__' == __name__:
                                 "test list")
   option_parser.add_option("", "--num-test-shells",
                            help="Number of testshells to run in parallel.")
+  option_parser.add_option("", "--use-apache", action="store_true",
+                           default=False,
+                           help="Whether to use apache instead of lighttpd.")
   option_parser.add_option("", "--time-out-ms", default=None,
                            help="Set the timeout for each test")
   option_parser.add_option("", "--run-singly", action="store_true",
diff --git a/webkit/tools/layout_tests/test_expectations.txt b/webkit/tools/layout_tests/test_expectations.txt
index 4b9c88e..653fa36 100644
--- a/webkit/tools/layout_tests/test_expectations.txt
+++ b/webkit/tools/layout_tests/test_expectations.txt
@@ -506,8 +506,8 @@ BUG10270 : LayoutTests/fast/dom/frame-loading-via-document-write.html = FAIL
 BUG18978 MAC : LayoutTests/http/tests/xmlhttprequest/cache-override.html = FAIL
 // LightTPD doesn't accept unknown HTTP methods
 BUG18978 SKIP : LayoutTests/http/tests/xmlhttprequest/methods-lower-case.html = TIMEOUT CRASH
-BUG18978 DEBUG : LayoutTests/http/tests/xmlhttprequest/methods-async.html = TIMEOUT
-BUG18978 LINUX MAC RELEASE : LayoutTests/http/tests/xmlhttprequest/methods-async.html = TIMEOUT
+BUG18978 DEBUG : LayoutTests/http/tests/xmlhttprequest/methods-async.html = TIMEOUT TEXT
+BUG18978 LINUX MAC RELEASE : LayoutTests/http/tests/xmlhttprequest/methods-async.html = TIMEOUT TEXT
 BUG18978 WIN RELEASE : LayoutTests/http/tests/xmlhttprequest/methods-async.html = TIMEOUT TEXT
 BUG18978 WIN SLOW : LayoutTests/http/tests/xmlhttprequest/access-control-basic-allow-preflight-cache-invalidation-by-method.html = FAIL
 BUG18978 MAC : LayoutTests/http/tests/xmlhttprequest/access-control-basic-allow-preflight-cache-invalidation-by-method.html = FAIL TIMEOUT
@@ -3027,6 +3027,12 @@ BUG28301 : LayoutTests/media/media-captions.html = TIMEOUT
 BUG28317 : LayoutTests/fast/css/color-correction-on-background-image.html = FAIL
 BUG28317 : LayoutTests/fast/css/color-correction-untagged-images.html = FAIL
 
+// Started failing with move from Lighttpd to Apache on Mac.
+// Since we have some bots on Lighttpd and some on Apache, mark it as flaky for
+// now. Once we decide on one or the other, then we can address these tests.
+BUG_OJAN MAC : LayoutTests/http/tests/loading/text-content-type-with-binary-extension.html = TEXT PASS
+BUG_OJAN MAC : LayoutTests/http/tests/xmlhttprequest/web-apps/012.html = TEXT PASS
+
 // Failures in TestShell that resulted from Mac plugin/focus fixes in r32631.
 BUG28380 MAC : LayoutTests/fast/events/tabindex-focus-blur-all.html = CRASH FAIL