Add a setting to CF to remove 'chromeframe' from the UserAgent on a per-pattern basis.

Useful for testing and dealing with sites with broken UA parsing.

BUG=117157
TEST=chrome_frame_tests,chrome_frame_unittests,add a ExcludeUAFromDomain key to the CF settings, add some pattern values, observe that the UA string does not contain CF.


Review URL: http://codereview.chromium.org/9720001

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@129985 0039d316-1c4b-4281-b951-d872f2087c98

11 files changed, 484 insertions, 166 deletions

diff --git a/chrome_frame/chrome_frame.gyp b/chrome_frame/chrome_frame.gyp
index e5e7577..60c4b4c 100644
--- a/chrome_frame/chrome_frame.gyp
+++ b/chrome_frame/chrome_frame.gyp
@@ -732,6 +732,8 @@
         'ready_mode/ready_mode.cc',
         'ready_mode/ready_mode.h',
         'register_bho.rgs',
+        'registry_list_preferences_holder.cc',
+        'registry_list_preferences_holder.h',
         'stream_impl.cc',
         'stream_impl.h',
         'urlmon_bind_status_callback.h',
diff --git a/chrome_frame/html_utils.cc b/chrome_frame/html_utils.cc
index 3121a75..6892ede 100644
--- a/chrome_frame/html_utils.cc
+++ b/chrome_frame/html_utils.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -343,19 +343,16 @@ const char* GetChromeFrameUserAgent() {
 
 std::string AddChromeFrameToUserAgentValue(const std::string& value) {
   if (value.empty()) {
-    DLOG(WARNING) << "empty user agent value";
-    return "";
+    return value;
   }
 
-  DCHECK_EQ(false, StartsWithASCII(value, "User-Agent:", true));
-
   if (value.find(kChromeFrameUserAgent) != std::string::npos) {
     // Our user agent has already been added.
     return value;
   }
 
   std::string ret(value);
-  std::string::size_type insert_position = ret.find(')');
+  size_t insert_position = ret.find(')');
   if (insert_position != std::string::npos) {
     if (insert_position > 1 && isalnum(ret[insert_position - 1]))
       ret.insert(insert_position++, ";");
@@ -369,6 +366,38 @@ std::string AddChromeFrameToUserAgentValue(const std::string& value) {
   return ret;
 }
 
+std::string RemoveChromeFrameFromUserAgentValue(const std::string& value) {
+  size_t cf_start = value.find(kChromeFrameUserAgent);
+  if (cf_start == std::string::npos) {
+    // The user agent is not present.
+    return value;
+  }
+
+  size_t offset = 0;
+  // If we prepended a '; ' or a ' ' then remove that in the output.
+  if (cf_start > 1 && value[cf_start - 1] == ' ')
+    ++offset;
+  if (cf_start > 3 &&
+      value[cf_start - 2] == ';' &&
+      isalnum(value[cf_start - 3])) {
+    ++offset;
+  }
+
+  std::string ret(value, 0, std::max(cf_start - offset, 0U));
+  cf_start += strlen(kChromeFrameUserAgent);
+  while (cf_start < value.length() &&
+         ((value[cf_start] >= '0' && value[cf_start] <= '9') ||
+          value[cf_start] == '.' ||
+          value[cf_start] == '/')) {
+    ++cf_start;
+  }
+
+  if (cf_start < value.length())
+    ret.append(value, cf_start, std::string::npos);
+
+  return ret;
+}
+
 std::string GetDefaultUserAgentHeaderWithCFTag() {
   std::string ua(GetDefaultUserAgent());
   return "User-Agent: " + AddChromeFrameToUserAgentValue(ua);
diff --git a/chrome_frame/html_utils.h b/chrome_frame/html_utils.h
index 285bc63..54175b2 100644
--- a/chrome_frame/html_utils.h
+++ b/chrome_frame/html_utils.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -127,11 +127,16 @@ class HTMLScanner {
 
 namespace http_utils {
 
-// Adds "chromeframe/x.y" to the end of the User-Agent string (x.y is the
-// version).  If the cf tag has already been added to the string,
-// the original string is returned.
+// Adds "chromeframe/a.b.c.d" to the User-Agent string (a.b.c.d is the version).
+// If the cf tag has already been added to the string, the original string is
+// returned.
 std::string AddChromeFrameToUserAgentValue(const std::string& value);
 
+// Removes "chromeframe/a.b.c.d" from the User-Agent string (a.b.c.d is the
+// version).  If the cf tag is not present in the string, the original string is
+// returned.
+std::string RemoveChromeFrameFromUserAgentValue(const std::string& value);
+
 // Fetches the user agent from urlmon and adds chrome frame to the
 // comment section.
 // NOTE: The returned string includes the "User-Agent: " header name.
diff --git a/chrome_frame/http_negotiate.cc b/chrome_frame/http_negotiate.cc
index 7ca3cea..d77bd92 100644
--- a/chrome_frame/http_negotiate.cc
+++ b/chrome_frame/http_negotiate.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -81,9 +81,11 @@ class SimpleBindStatusCallback : public CComObjectRootEx<CComSingleThreadModel>,
     return E_NOTIMPL;
   }
 };
-}  // end namespace
 
-std::string AppendCFUserAgentString(LPCWSTR headers,
+// Returns the full user agent header from the HTTP header strings passed to
+// IHttpNegotiate::BeginningTransaction. Looks first in |additional_headers|
+// and if it can't be found there looks in |headers|.
+std::string GetUserAgentFromHeaders(LPCWSTR headers,
                                     LPCWSTR additional_headers) {
   using net::HttpUtil;
 
@@ -107,52 +109,85 @@ std::string AppendCFUserAgentString(LPCWSTR headers,
       user_agent_value = original_it.values();
   }
 
-  // Use the default "User-Agent" if none was provided.
-  if (user_agent_value.empty())
-    user_agent_value = http_utils::GetDefaultUserAgent();
-
-  // Now add chromeframe to it.
-  user_agent_value = http_utils::AddChromeFrameToUserAgentValue(
-      user_agent_value);
+  return user_agent_value;
+}
 
-  // Build new headers, skip the existing user agent value from
-  // existing headers.
+// Removes the named header |field| from a set of headers. |field| must be
+// lower-case.
+std::string ExcludeFieldFromHeaders(const std::string& old_headers,
+                                    const char* field) {
+  using net::HttpUtil;
   std::string new_headers;
-  headers_iterator.Reset();
+  new_headers.reserve(old_headers.size());
+  HttpUtil::HeadersIterator headers_iterator(old_headers.begin(),
+                                             old_headers.end(), "\r\n");
   while (headers_iterator.GetNext()) {
-    std::string name(headers_iterator.name());
-    if (!LowerCaseEqualsASCII(name, kLowerCaseUserAgent)) {
-      new_headers += name + ": " + headers_iterator.values() + "\r\n";
+    if (!LowerCaseEqualsASCII(headers_iterator.name_begin(),
+                              headers_iterator.name_end(),
+                              field)) {
+      new_headers.append(headers_iterator.name_begin(),
+                         headers_iterator.name_end());
+      new_headers += ": ";
+      new_headers.append(headers_iterator.values_begin(),
+                         headers_iterator.values_end());
+      new_headers += "\r\n";
     }
   }
 
-  new_headers += "User-Agent: " + user_agent_value;
-  new_headers += "\r\n";
   return new_headers;
 }
 
+std::string MutateCFUserAgentString(LPCWSTR headers,
+                                    LPCWSTR additional_headers,
+                                    bool add_user_agent) {
+  std::string user_agent_value(GetUserAgentFromHeaders(headers,
+                                                       additional_headers));
+
+  // Use the default "User-Agent" if none was provided.
+  if (user_agent_value.empty())
+    user_agent_value = http_utils::GetDefaultUserAgent();
+
+  // Now add chromeframe to it.
+  user_agent_value = add_user_agent ?
+      http_utils::AddChromeFrameToUserAgentValue(user_agent_value) :
+      http_utils::RemoveChromeFrameFromUserAgentValue(user_agent_value);
+
+  // Build a new set of additional headers, skipping the existing user agent
+  // value if present.
+  return ReplaceOrAddUserAgent(additional_headers, user_agent_value);
+}
+
+}  // end namespace
+
+
+std::string AppendCFUserAgentString(LPCWSTR headers,
+                                    LPCWSTR additional_headers) {
+  return MutateCFUserAgentString(headers, additional_headers, true);
+}
+
+
+// Looks for a user agent header found in |headers| or |additional_headers|
+// then returns |additional_headers| with a modified user agent header that does
+// not include the chromeframe token.
+std::string RemoveCFUserAgentString(LPCWSTR headers,
+                                    LPCWSTR additional_headers) {
+  return MutateCFUserAgentString(headers, additional_headers, false);
+}
+
+
+// Unconditionally adds the specified |user_agent_value| to the given set of
+// |headers|, removing any that were already there.
 std::string ReplaceOrAddUserAgent(LPCWSTR headers,
                                   const std::string& user_agent_value) {
-  using net::HttpUtil;
-
   std::string new_headers;
   if (headers) {
     std::string ascii_headers(WideToASCII(headers));
-
-    // Extract "User-Agent" from the headers.
-    HttpUtil::HeadersIterator headers_iterator(ascii_headers.begin(),
-                                               ascii_headers.end(), "\r\n");
-
     // Build new headers, skip the existing user agent value from
     // existing headers.
-    while (headers_iterator.GetNext()) {
-      std::string name(headers_iterator.name());
-      if (!LowerCaseEqualsASCII(name, kLowerCaseUserAgent)) {
-        new_headers += name + ": " + headers_iterator.values() + "\r\n";
-      }
-    }
+    new_headers = ExcludeFieldFromHeaders(ascii_headers, kLowerCaseUserAgent);
   }
-  new_headers += "User-Agent: " + user_agent_value;
+  new_headers += "User-Agent: ";
+  new_headers += user_agent_value;
   new_headers += "\r\n";
   return new_headers;
 }
@@ -231,14 +266,18 @@ HRESULT HttpNegotiatePatch::BeginningTransaction(
   }
   if (modify_user_agent_) {
     std::string updated_headers;
+
     if (IsGcfDefaultRenderer() &&
         RendererTypeForUrl(url) == RENDERER_TYPE_CHROME_DEFAULT_RENDERER) {
       // Replace the user-agent header with Chrome's.
       updated_headers = ReplaceOrAddUserAgent(*additional_headers,
                                               http_utils::GetChromeUserAgent());
+    } else if (ShouldRemoveUAForUrl(url)) {
+      updated_headers = RemoveCFUserAgentString(headers, *additional_headers);
     } else {
       updated_headers = AppendCFUserAgentString(headers, *additional_headers);
     }
+
     *additional_headers = reinterpret_cast<wchar_t*>(::CoTaskMemRealloc(
         *additional_headers,
         (updated_headers.length() + 1) * sizeof(wchar_t)));
diff --git a/chrome_frame/registry_list_preferences_holder.cc b/chrome_frame/registry_list_preferences_holder.cc
new file mode 100644
index 0000000..581aed6
--- /dev/null
+++ b/chrome_frame/registry_list_preferences_holder.cc
@@ -0,0 +1,45 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome_frame/registry_list_preferences_holder.h"
+
+#include "base/string_util.h"
+#include "base/win/registry.h"
+
+RegistryListPreferencesHolder::RegistryListPreferencesHolder() : valid_(false) {
+}
+
+void RegistryListPreferencesHolder::Init(HKEY hive,
+                                         const wchar_t* registry_path,
+                                         const wchar_t* list_name) {
+  string16 list_path(registry_path);
+  list_path += L"\\";
+  list_path += list_name;
+  base::win::RegistryValueIterator string_list(hive, list_path.c_str());
+  for (; string_list.Valid(); ++string_list)
+    values_.push_back(string_list.Name());
+
+  valid_ = true;
+}
+
+bool RegistryListPreferencesHolder::ListMatches(const string16& string) const {
+  DCHECK(Valid());
+  std::vector<string16>::const_iterator iter(values_.begin());
+  for (; iter != values_.end(); ++iter) {
+    if (MatchPattern(string, *iter))
+      return true;
+  }
+
+  return false;
+}
+
+void RegistryListPreferencesHolder::AddStringForTesting(
+    const string16& string) {
+  values_.push_back(string);
+}
+
+void RegistryListPreferencesHolder::ResetForTesting() {
+  values_.clear();
+  valid_ = false;
+}
diff --git a/chrome_frame/registry_list_preferences_holder.h b/chrome_frame/registry_list_preferences_holder.h
new file mode 100644
index 0000000..3c4ee46
--- /dev/null
+++ b/chrome_frame/registry_list_preferences_holder.h
@@ -0,0 +1,51 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// A utility class for accessing and caching registry preferences that take
+// the form of lists of strings.
+// TODO(robertshield): Use the RegistryWatcher stuff to keep this list up to
+// date.
+
+#ifndef CHROME_FRAME_REGISTRY_LIST_PREFERENCES_HOLDER_H_
+#define CHROME_FRAME_REGISTRY_LIST_PREFERENCES_HOLDER_H_
+#pragma once
+
+#include <windows.h>
+
+#include <vector>
+
+#include "base/string16.h"
+
+class RegistryListPreferencesHolder {
+ public:
+  RegistryListPreferencesHolder();
+
+  // Initializes the RegistryListPreferencesHolder using the values stored
+  // under the key |list_name| stored at |registry_path| under |hive|.
+  void Init(HKEY hive,
+            const wchar_t* registry_path,
+            const wchar_t* list_name);
+
+  bool Valid() const { return valid_; }
+
+  // Returns true iff |string| matches any of the strings in values_, using the
+  // matching logic in base's MatchPattern().
+  bool ListMatches(const string16& string) const;
+
+  // Manually add a string to values_ for testing purposes.
+  void AddStringForTesting(const string16& string);
+
+  // Reset the holder causing it to be re-initialized, for testing purposes
+  // only.
+  // TODO(robertshield): Remove this once the RegistryWatcher stuff is wired up.
+  void ResetForTesting();
+
+ private:
+  std::vector<string16> values_;
+  bool valid_;
+
+  DISALLOW_COPY_AND_ASSIGN(RegistryListPreferencesHolder);
+};
+
+#endif  // CHROME_FRAME_REGISTRY_LIST_PREFERENCES_HOLDER_H_
diff --git a/chrome_frame/test/html_util_unittests.cc b/chrome_frame/test/html_util_unittests.cc
index 95a408f..827dfc7 100644
--- a/chrome_frame/test/html_util_unittests.cc
+++ b/chrome_frame/test/html_util_unittests.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -319,79 +319,96 @@ TEST_F(HtmlUtilUnittest, IEConditionalCommentNonTerminatedTest) {
   ASSERT_EQ(2, boo_tag_list.size());
 }
 
-TEST_F(HtmlUtilUnittest, AddChromeFrameToUserAgentValue) {
-  struct TestCase {
-    std::string input_;
-    std::string expected_;
-  } test_cases[] = {
-    {
-      "", ""
-    }, {
-      "Mozilla/4.7 [en] (WinNT; U)",
-      "Mozilla/4.7 [en] (WinNT; U; chromeframe/0.0.0.0)"
-    }, {
-      "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)",
-      "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT; chromeframe/0.0.0.0)"
-    }, {
-      "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461; "
-          ".NET CLR 1.1.4322)",
-      "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461; "
-          ".NET CLR 1.1.4322; chromeframe/0.0.0.0)"
-    }, {
-      "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 4.0) Opera 5.11 [en]",
-      "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 4.0; chromeframe/0.0.0.0) "
-          "Opera 5.11 [en]"
-    }, {
-      "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
-      "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; "
-          "chromeframe/0.0.0.0)"
-    }, {
-      "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.0.2) "
-          "Gecko/20030208 Netscape/7.02",
-      "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.0.2; "
-          "chromeframe/0.0.0.0) Gecko/20030208 Netscape/7.02"
-    }, {
-      "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040612 "
-          "Firefox/0.8",
-      "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6; chromeframe/0.0.0.0) "
-          "Gecko/20040612 Firefox/0.8"
-    }, {
-      "Mozilla/5.0 (compatible; Konqueror/3.2; Linux) (KHTML, like Gecko)",
-      "Mozilla/5.0 (compatible; Konqueror/3.2; Linux; chromeframe/0.0.0.0) "
-          "(KHTML, like Gecko)"
-    }, {
-      "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6h",
-      "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 "
-          "OpenSSL/0.9.6h chromeframe/0.0.0.0",
-    }, {
-      "Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.7.10) "
-          "Gecko/20050716 Firefox/1.0.6",
-      "Mozilla/5.0 (X11; U; Linux i686 (x86_64; chromeframe/0.0.0.0); en-US; "
-          "rv:1.7.10) Gecko/20050716 Firefox/1.0.6"
-    }, {
-      "Invalid/1.1 ((((((",
-      "Invalid/1.1 (((((( chromeframe/0.0.0.0",
-    }, {
-      "Invalid/1.1 ()))))",
-      "Invalid/1.1 ( chromeframe/0.0.0.0)))))",
-    }, {
-      "Strange/1.1 ()",
-      "Strange/1.1 ( chromeframe/0.0.0.0)",
-    }
-  };
+struct UserAgentTestCase {
+  std::string input_;
+  std::string expected_;
+} user_agent_test_cases[] = {
+  {
+    "", ""
+  }, {
+    "Mozilla/4.7 [en] (WinNT; U)",
+    "Mozilla/4.7 [en] (WinNT; U; chromeframe/0.0.0.0)"
+  }, {
+    "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)",
+    "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT; chromeframe/0.0.0.0)"
+  }, {
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461; "
+        ".NET CLR 1.1.4322)",
+    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461; "
+        ".NET CLR 1.1.4322; chromeframe/0.0.0.0)"
+  }, {
+    "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 4.0) Opera 5.11 [en]",
+    "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 4.0; chromeframe/0.0.0.0) "
+        "Opera 5.11 [en]"
+  }, {
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
+    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; "
+        "chromeframe/0.0.0.0)"
+  }, {
+    "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.0.2) "
+        "Gecko/20030208 Netscape/7.02",
+    "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.0.2; "
+        "chromeframe/0.0.0.0) Gecko/20030208 Netscape/7.02"
+  }, {
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040612 "
+        "Firefox/0.8",
+    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6; chromeframe/0.0.0.0) "
+        "Gecko/20040612 Firefox/0.8"
+  }, {
+    "Mozilla/5.0 (compatible; Konqueror/3.2; Linux) (KHTML, like Gecko)",
+    "Mozilla/5.0 (compatible; Konqueror/3.2; Linux; chromeframe/0.0.0.0) "
+        "(KHTML, like Gecko)"
+  }, {
+    "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6h",
+    "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 "
+        "OpenSSL/0.9.6h chromeframe/0.0.0.0",
+  }, {
+    "Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US; rv:1.7.10) "
+        "Gecko/20050716 Firefox/1.0.6",
+    "Mozilla/5.0 (X11; U; Linux i686 (x86_64; chromeframe/0.0.0.0); en-US; "
+        "rv:1.7.10) Gecko/20050716 Firefox/1.0.6"
+  }, {
+    "Invalid/1.1 ((((((",
+    "Invalid/1.1 (((((( chromeframe/0.0.0.0",
+  }, {
+    "Invalid/1.1 ()))))",
+    "Invalid/1.1 ( chromeframe/0.0.0.0)))))",
+  }, {
+    "Strange/1.1 ()",
+    "Strange/1.1 ( chromeframe/0.0.0.0)",
+  }
+};
 
-  for (int i = 0; i < arraysize(test_cases); ++i) {
+TEST_F(HtmlUtilUnittest, AddChromeFrameToUserAgentValue) {
+  for (int i = 0; i < arraysize(user_agent_test_cases); ++i) {
     std::string new_ua(
-        http_utils::AddChromeFrameToUserAgentValue(test_cases[i].input_));
-    EXPECT_EQ(test_cases[i].expected_, new_ua);
+        http_utils::AddChromeFrameToUserAgentValue(
+            user_agent_test_cases[i].input_));
+    EXPECT_EQ(user_agent_test_cases[i].expected_, new_ua);
   }
 
   // Now do the same test again, but test that we don't add the chromeframe
   // tag if we've already added it.
-  for (int i = 0; i < arraysize(test_cases); ++i) {
-    std::string ua(test_cases[i].expected_);
+  for (int i = 0; i < arraysize(user_agent_test_cases); ++i) {
+    std::string ua(user_agent_test_cases[i].expected_);
     std::string new_ua(http_utils::AddChromeFrameToUserAgentValue(ua));
-    EXPECT_EQ(test_cases[i].expected_, new_ua);
+    EXPECT_EQ(user_agent_test_cases[i].expected_, new_ua);
+  }
+}
+
+TEST_F(HtmlUtilUnittest, RemoveChromeFrameFromUserAgentValue) {
+  for (int i = 0; i < arraysize(user_agent_test_cases); ++i) {
+    std::string new_ua(
+        http_utils::RemoveChromeFrameFromUserAgentValue(
+            user_agent_test_cases[i].expected_));
+    EXPECT_EQ(user_agent_test_cases[i].input_, new_ua);
+  }
+
+  // Also test that we don't modify the UA if chromeframe is not present.
+  for (int i = 0; i < arraysize(user_agent_test_cases); ++i) {
+    std::string ua(user_agent_test_cases[i].input_);
+    std::string new_ua(http_utils::RemoveChromeFrameFromUserAgentValue(ua));
+    EXPECT_EQ(user_agent_test_cases[i].input_, new_ua);
   }
 }
 
diff --git a/chrome_frame/test/http_negotiate_unittest.cc b/chrome_frame/test/http_negotiate_unittest.cc
index d63b79b..177459c 100644
--- a/chrome_frame/test/http_negotiate_unittest.cc
+++ b/chrome_frame/test/http_negotiate_unittest.cc
@@ -1,16 +1,18 @@
-// Copyright (c) 2010 The Chromium Authors. All rights reserved.
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <atlbase.h>
 #include <atlcom.h>
 
+#include "base/string16.h"
 #include "base/string_util.h"
 #include "base/utf_string_conversions.h"
 #include "base/win/scoped_bstr.h"
 #include "base/win/scoped_comptr.h"
 #include "chrome_frame/http_negotiate.h"
 #include "chrome_frame/html_utils.h"
+#include "chrome_frame/registry_list_preferences_holder.h"
 #include "chrome_frame/test/chrome_frame_test_utils.h"
 #include "chrome_frame/utils.h"
 #include "gtest/gtest.h"
@@ -65,17 +67,17 @@ TEST_F(HttpNegotiateTest, BeginningTransaction) {
               static_cast<IHttpNegotiate*>(
                   &test_http)))[kBeginningTransactionIndex]);
 
-  std::wstring cf_ua(
+  string16 cf_ua(
       ASCIIToWide(http_utils::GetDefaultUserAgentHeaderWithCFTag()));
-  std::wstring cf_tag(
+  string16 cf_tag(
       ASCIIToWide(http_utils::GetChromeFrameUserAgent()));
 
-  EXPECT_NE(std::wstring::npos, cf_ua.find(L"chromeframe/"));
+  EXPECT_NE(string16::npos, cf_ua.find(L"chromeframe/"));
 
   struct TestCase {
-    const std::wstring original_headers_;
-    const std::wstring delegate_additional_;
-    const std::wstring expected_additional_;
+    const string16 original_headers_;
+    const string16 delegate_additional_;
+    const string16 expected_additional_;
     HRESULT delegate_return_value_;
   } test_cases[] = {
     { L"Accept: */*\r\n",
@@ -116,12 +118,82 @@ TEST_F(HttpNegotiateTest, BeginningTransaction) {
 
     if (additional) {
       // Check against the expected additional headers.
-      EXPECT_EQ(test.expected_additional_, std::wstring(additional));
+      EXPECT_EQ(test.expected_additional_, string16(additional));
       ::CoTaskMemFree(additional);
     }
   }
 }
 
+TEST_F(HttpNegotiateTest, BeginningTransactionUARemoval) {
+  static const int kBeginningTransactionIndex = 3;
+  CComObjectStackEx<TestHttpNegotiate> test_http;
+  IHttpNegotiate_BeginningTransaction_Fn original =
+      reinterpret_cast<IHttpNegotiate_BeginningTransaction_Fn>(
+          (*reinterpret_cast<void***>(
+              static_cast<IHttpNegotiate*>(
+                  &test_http)))[kBeginningTransactionIndex]);
+
+  string16 nocf_ua(
+      ASCIIToWide(http_utils::RemoveChromeFrameFromUserAgentValue(
+          http_utils::GetDefaultUserAgentHeaderWithCFTag())));
+  string16 cf_ua(
+      ASCIIToWide(http_utils::AddChromeFrameToUserAgentValue(
+          WideToASCII(nocf_ua))));
+
+  EXPECT_EQ(string16::npos, nocf_ua.find(L"chromeframe/"));
+  EXPECT_NE(string16::npos, cf_ua.find(L"chromeframe/"));
+
+  string16 ua_url(L"www.withua.com");
+  string16 no_ua_url(L"www.noua.com");
+
+  RegistryListPreferencesHolder& ua_holder =
+      GetUserAgentPreferencesHolderForTesting();
+  ua_holder.AddStringForTesting(no_ua_url);
+
+  struct TestCase {
+    const string16 url_;
+    const string16 original_headers_;
+    const string16 delegate_additional_;
+    const string16 expected_additional_;
+  } test_cases[] = {
+    { ua_url,
+      L"",
+      L"Accept: */*\r\n" + cf_ua + L"\r\n",
+      L"Accept: */*\r\n" + cf_ua + L"\r\n" },
+    { ua_url,
+      L"",
+      L"Accept: */*\r\n" + nocf_ua + L"\r\n",
+      L"Accept: */*\r\n" + cf_ua + L"\r\n" },
+    { no_ua_url,
+      L"",
+      L"Accept: */*\r\n" + cf_ua + L"\r\n",
+      L"Accept: */*\r\n" + nocf_ua + L"\r\n" },
+    { no_ua_url,
+      L"",
+      L"Accept: */*\r\n" + nocf_ua + L"\r\n",
+      L"Accept: */*\r\n" + nocf_ua + L"\r\n" },
+  };
+
+  for (int i = 0; i < arraysize(test_cases); ++i) {
+    TestCase& test = test_cases[i];
+    wchar_t* additional = NULL;
+    test_http.beginning_transaction_ret_ = S_OK;
+    test_http.additional_headers_ = test.delegate_additional_.c_str();
+    HttpNegotiatePatch::BeginningTransaction(original, &test_http,
+        test.url_.c_str(), test.original_headers_.c_str(), 0,
+        &additional);
+    EXPECT_TRUE(additional != NULL);
+
+    if (additional) {
+      // Check against the expected additional headers.
+      EXPECT_EQ(test.expected_additional_, string16(additional))
+          << "Iteration: " << i;
+      ::CoTaskMemFree(additional);
+    }
+  }
+}
+
+
 class TestInternetProtocolSink
     : public CComObjectRootEx<CComMultiThreadModel>,
       public IInternetProtocolSink {
@@ -175,13 +247,13 @@ END_COM_MAP()
     return status_;
   }
 
-  const std::wstring& last_status_text() const {
+  const string16& last_status_text() const {
     return status_text_;
   }
 
  protected:
   ULONG status_;
-  std::wstring status_text_;
+  string16 status_text_;
   base::win::ScopedComPtr<IWebBrowser2> browser_;
 };
 
diff --git a/chrome_frame/test/util_unittests.cc b/chrome_frame/test/util_unittests.cc
index 572ef13..9700144 100644
--- a/chrome_frame/test/util_unittests.cc
+++ b/chrome_frame/test/util_unittests.cc
@@ -1,4 +1,4 @@
-// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
@@ -10,6 +10,7 @@
 #include "base/win/registry.h"
 #include "chrome_frame/navigation_constraints.h"
 #include "chrome_frame/test/chrome_frame_test_utils.h"
+#include "chrome_frame/registry_list_preferences_holder.h"
 #include "chrome_frame/utils.h"
 
 #include "testing/gtest/include/gtest/gtest.h"
@@ -452,6 +453,11 @@ TEST_F(UtilTests, RendererTypeForUrlTest) {
   DWORD saved_default_renderer = 0;
   config_key.ReadValueDW(kEnableGCFRendererByDefault, &saved_default_renderer);
 
+  // We need to manually reset the holder between checks.
+  // TODO(robertshield): Remove this when the RegistryWatcher is wired up.
+  RegistryListPreferencesHolder& renderer_type_preferences_holder =
+      GetRendererTypePreferencesHolderForTesting();
+
   // Make sure the host is the default renderer.
   config_key.WriteValue(kEnableGCFRendererByDefault, static_cast<DWORD>(0));
   EXPECT_FALSE(IsGcfDefaultRenderer());
@@ -459,6 +465,7 @@ TEST_F(UtilTests, RendererTypeForUrlTest) {
   opt_for_gcf.DeleteValue(kTestFilter);  // Just in case this exists
   EXPECT_EQ(RENDERER_TYPE_UNDETERMINED, RendererTypeForUrl(kTestUrl));
   opt_for_gcf.WriteValue(kTestFilter, L"");
+  renderer_type_preferences_holder.ResetForTesting();
   EXPECT_EQ(RENDERER_TYPE_CHROME_OPT_IN_URL, RendererTypeForUrl(kTestUrl));
 
   // Now set GCF as the default renderer.
@@ -466,15 +473,19 @@ TEST_F(UtilTests, RendererTypeForUrlTest) {
   EXPECT_TRUE(IsGcfDefaultRenderer());
 
   opt_for_host.DeleteValue(kTestFilter);  // Just in case this exists
+  renderer_type_preferences_holder.ResetForTesting();
   EXPECT_EQ(RENDERER_TYPE_CHROME_DEFAULT_RENDERER,
             RendererTypeForUrl(kTestUrl));
   opt_for_host.WriteValue(kTestFilter, L"");
+  renderer_type_preferences_holder.ResetForTesting();
   EXPECT_EQ(RENDERER_TYPE_UNDETERMINED, RendererTypeForUrl(kTestUrl));
 
   // Cleanup.
   opt_for_gcf.DeleteValue(kTestFilter);
   opt_for_host.DeleteValue(kTestFilter);
   config_key.WriteValue(kEnableGCFRendererByDefault, saved_default_renderer);
+  renderer_type_preferences_holder.ResetForTesting();
+  RendererTypeForUrl(L"");
 }
 
 TEST_F(UtilTests, XUaCompatibleDirectiveTest) {
diff --git a/chrome_frame/utils.cc b/chrome_frame/utils.cc
index ba41167..8f98b31 100644
--- a/chrome_frame/utils.cc
+++ b/chrome_frame/utils.cc
@@ -35,6 +35,7 @@
 #include "chrome_frame/html_utils.h"
 #include "chrome_frame/navigation_constraints.h"
 #include "chrome_frame/policy_settings.h"
+#include "chrome_frame/registry_list_preferences_holder.h"
 #include "chrome_frame/simple_resource_loader.h"
 #include "googleurl/src/gurl.h"
 #include "googleurl/src/url_canon.h"
@@ -47,25 +48,27 @@ using base::win::RegKey;
 
 // Note that these values are all lower case and are compared to
 // lower-case-transformed values.
-const wchar_t kMetaTag[] = L"meta";
-const wchar_t kHttpEquivAttribName[] = L"http-equiv";
-const wchar_t kContentAttribName[] = L"content";
-const wchar_t kXUACompatValue[] = L"x-ua-compatible";
+const char kGCFProtocol[] = "gcf";
 const wchar_t kBodyTag[] = L"body";
+const wchar_t kContentAttribName[] = L"content";
 const wchar_t kChromeContentPrefix[] = L"chrome=";
-const char kGCFProtocol[] = "gcf";
-const wchar_t kChromeProtocolPrefix[] = L"gcf:";
 const wchar_t kChromeMimeType[] = L"application/chromepage";
-const wchar_t kPatchProtocols[] = L"PatchProtocols";
-const wchar_t kChromeFrameConfigKey[] = L"Software\\Google\\ChromeFrame";
-const wchar_t kRenderInGCFUrlList[] = L"RenderInGcfUrls";
-const wchar_t kRenderInHostUrlList[] = L"RenderInHostUrls";
-const wchar_t kEnableGCFRendererByDefault[] = L"IsDefaultRenderer";
+const wchar_t kChromeProtocolPrefix[] = L"gcf:";
+const wchar_t kHttpEquivAttribName[] = L"http-equiv";
 const wchar_t kIexploreProfileName[] = L"iexplore";
+const wchar_t kMetaTag[] = L"meta";
 const wchar_t kRundllProfileName[] = L"rundll32";
+const wchar_t kXUACompatValue[] = L"x-ua-compatible";
 
+// Registry key and value names related to Chrome Frame configuration options.
 const wchar_t kAllowUnsafeURLs[] = L"AllowUnsafeURLs";
+const wchar_t kChromeFrameConfigKey[] = L"Software\\Google\\ChromeFrame";
 const wchar_t kEnableBuggyBhoIntercept[] = L"EnableBuggyBhoIntercept";
+const wchar_t kEnableGCFRendererByDefault[] = L"IsDefaultRenderer";
+const wchar_t kExcludeUAFromDomainList[] = L"ExcludeUAFromDomain";
+const wchar_t kPatchProtocols[] = L"PatchProtocols";
+const wchar_t kRenderInGCFUrlList[] = L"RenderInGcfUrls";
+const wchar_t kRenderInHostUrlList[] = L"RenderInHostUrls";
 
 static const wchar_t kChromeFramePersistNPAPIReg[] = L"PersistNPAPIReg";
 
@@ -113,6 +116,14 @@ namespace {
 base::LazyInstance<base::ThreadLocalPointer<IBrowserService> >
     g_tls_browser_for_cf_navigation = LAZY_INSTANCE_INITIALIZER;
 
+// Holds the cached preferences for the per-url render type settings.
+base::LazyInstance<RegistryListPreferencesHolder>::Leaky
+    g_render_type_for_url_holder;
+
+// Holds the cached preferences for the per-url user agent filter.
+base::LazyInstance<RegistryListPreferencesHolder>::Leaky
+    g_user_agent_filter_holder;
+
 }  // end anonymous namespace
 
 HRESULT UtilRegisterTypeLib(HINSTANCE tlb_instance,
@@ -174,7 +185,7 @@ HRESULT UtilRegisterTypeLib(ITypeLib* typelib,
                                                     OLECHAR FAR* full_path,
                                                     OLECHAR FAR* help_dir);
   LPCSTR function_name =
-    for_current_user_only ? "RegisterTypeLibForUser" : "RegisterTypeLib";
+      for_current_user_only ? "RegisterTypeLibForUser" : "RegisterTypeLib";
   RegisterTypeLibPrototype reg_tlb =
       reinterpret_cast<RegisterTypeLibPrototype>(
           GetProcAddress(GetModuleHandle(_T("oleaut32.dll")),
@@ -716,36 +727,32 @@ RendererType RendererTypeForUrl(const std::wstring& url) {
         RENDERER_TYPE_CHROME_OPT_IN_URL : RENDERER_TYPE_UNDETERMINED;
   }
 
-  RegKey config_key;
-  if (config_key.Open(HKEY_CURRENT_USER, kChromeFrameConfigKey,
-                      KEY_READ) != ERROR_SUCCESS) {
-    return RENDERER_TYPE_UNDETERMINED;
-  }
-
-  RendererType renderer_type = RENDERER_TYPE_UNDETERMINED;
-
-  const wchar_t* url_list_name = NULL;
-  int render_in_cf_by_default = FALSE;
-  config_key.ReadValueDW(kEnableGCFRendererByDefault,
-                         reinterpret_cast<DWORD*>(&render_in_cf_by_default));
-  if (render_in_cf_by_default) {
-    url_list_name = kRenderInHostUrlList;
-    renderer_type = RENDERER_TYPE_CHROME_DEFAULT_RENDERER;
-  } else {
-    url_list_name = kRenderInGCFUrlList;
-  }
+  // TODO(robertshield): Move this into a holder-type class that listens
+  // for reg change events as well.
+  static int render_in_cf_by_default = FALSE;
 
-  bool match_found = false;
-  base::win::RegistryValueIterator url_list(config_key.Handle(), url_list_name);
-  while (!match_found && url_list.Valid()) {
-    if (MatchPattern(url, url_list.Name())) {
-      match_found = true;
+  RegistryListPreferencesHolder& render_type_for_url_holder =
+      g_render_type_for_url_holder.Get();
+  if (!render_type_for_url_holder.Valid()) {
+    const wchar_t* url_list_name = kRenderInGCFUrlList;
+    if (IsGcfDefaultRenderer()) {
+      url_list_name = kRenderInHostUrlList;
+      render_in_cf_by_default = TRUE;
     } else {
-      ++url_list;
+      render_in_cf_by_default = FALSE;
     }
+
+    render_type_for_url_holder.Init(HKEY_CURRENT_USER,
+                                    kChromeFrameConfigKey,
+                                    url_list_name);
   }
+  DCHECK(render_type_for_url_holder.Valid());
+
+  RendererType renderer_type =
+      render_in_cf_by_default ? RENDERER_TYPE_CHROME_DEFAULT_RENDERER :
+                                RENDERER_TYPE_UNDETERMINED;
 
-  if (match_found) {
+  if (render_type_for_url_holder.ListMatches(url)) {
     renderer_type = render_in_cf_by_default ?
       RENDERER_TYPE_UNDETERMINED :
       RENDERER_TYPE_CHROME_OPT_IN_URL;
@@ -754,6 +761,31 @@ RendererType RendererTypeForUrl(const std::wstring& url) {
   return renderer_type;
 }
 
+bool ShouldRemoveUAForUrl(const string16& url) {
+  // TODO(robertshield): Wire up the stuff in PolicySettings here so the value
+  // can be specified via group policy.
+  // TODO(robertshield): Add a default list of exclusions here for site with
+  // known bad UA parsing.
+  RegistryListPreferencesHolder& user_agent_filter_holder =
+      g_user_agent_filter_holder.Get();
+  if (!user_agent_filter_holder.Valid()) {
+    user_agent_filter_holder.Init(HKEY_CURRENT_USER,
+                                  kChromeFrameConfigKey,
+                                  kExcludeUAFromDomainList);
+  }
+  DCHECK(user_agent_filter_holder.Valid());
+
+  return user_agent_filter_holder.ListMatches(url);
+}
+
+RegistryListPreferencesHolder& GetRendererTypePreferencesHolderForTesting() {
+  return g_render_type_for_url_holder.Get();
+}
+
+RegistryListPreferencesHolder& GetUserAgentPreferencesHolderForTesting() {
+  return g_user_agent_filter_holder.Get();
+}
+
 HRESULT NavigateBrowserToMoniker(IUnknown* browser, IMoniker* moniker,
                                  const wchar_t* headers, IBindCtx* bind_ctx,
                                  const wchar_t* fragment, IStream* post_data,
diff --git a/chrome_frame/utils.h b/chrome_frame/utils.h
index b3899c3..d9f6580 100644
--- a/chrome_frame/utils.h
+++ b/chrome_frame/utils.h
@@ -15,33 +15,36 @@
 #include "base/basictypes.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
+#include "base/string16.h"
 #include "base/threading/thread.h"
 #include "base/win/scoped_comptr.h"
 #include "googleurl/src/gurl.h"
 #include "ui/gfx/rect.h"
 
 class FilePath;
+class RegistryListPreferencesHolder;
 interface IBrowserService;
 interface IWebBrowser2;
 struct ContextMenuModel;
 
 // utils.h : Various utility functions and classes
+extern const char kGCFProtocol[];
 
+extern const wchar_t kAllowUnsafeURLs[];
 extern const wchar_t kChromeContentPrefix[];
-extern const char kGCFProtocol[];
-extern const wchar_t kChromeProtocolPrefix[];
-extern const wchar_t kChromeFrameHeadlessMode[];
 extern const wchar_t kChromeFrameAccessibleMode[];
-extern const wchar_t kChromeFrameUnpinnedMode[];
-extern const wchar_t kAllowUnsafeURLs[];
-extern const wchar_t kEnableBuggyBhoIntercept[];
-extern const wchar_t kChromeMimeType[];
 extern const wchar_t kChromeFrameAttachTabPattern[];
 extern const wchar_t kChromeFrameConfigKey[];
-extern const wchar_t kRenderInGCFUrlList[];
-extern const wchar_t kRenderInHostUrlList[];
+extern const wchar_t kChromeFrameHeadlessMode[];
+extern const wchar_t kChromeFrameUnpinnedMode[];
+extern const wchar_t kChromeMimeType[];
+extern const wchar_t kChromeProtocolPrefix[];
+extern const wchar_t kEnableBuggyBhoIntercept[];
 extern const wchar_t kEnableGCFRendererByDefault[];
+extern const wchar_t kExcludeUAFromDomainList[];
 extern const wchar_t kIexploreProfileName[];
+extern const wchar_t kRenderInGCFUrlList[];
+extern const wchar_t kRenderInHostUrlList[];
 extern const wchar_t kRundllProfileName[];
 extern const wchar_t kUseBackgroundThreadForSubResources[];
 
@@ -272,6 +275,18 @@ bool IsGcfDefaultRenderer();
 // - RENDERER_TYPE_CHROME_OPT_IN_URL
 RendererType RendererTypeForUrl(const std::wstring& url);
 
+// Check if we should try to remove the CF user agent based on registry
+// settings.
+bool ShouldRemoveUAForUrl(const string16& url);
+
+// Testing methods that return the backing stores behind RendererTypeForUrl and
+// ShouldRemoveUAForUrl. Intended to allow unit testing code that calls the
+// above methods.
+// TODO(robertshield): All of the FooForUrl code should be removed from here
+// and further refactored.
+RegistryListPreferencesHolder& GetRendererTypePreferencesHolderForTesting();
+RegistryListPreferencesHolder& GetUserAgentPreferencesHolderForTesting();
+
 // A shortcut for QueryService
 template <typename T>
 HRESULT DoQueryService(const IID& service_id, IUnknown* unk, T** service) {