Port of WebKit's appcache manifest parser code.

Added unittests for manifest parser to test_shell_tests.

This is a clone of jennb's CL here.
http://codereview.chromium.org/160608

TBR=jennb
BUG=none
TEST=manifest_parser_unittest.cc

Review URL: http://codereview.chromium.org/165072

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@22673 0039d316-1c4b-4281-b951-d872f2087c98

1 files changed, 294 insertions, 0 deletions

diff --git a/webkit/appcache/manifest_parser_unittest.cc b/webkit/appcache/manifest_parser_unittest.cc
new file mode 100644
index 0000000..07721ab
--- /dev/null
+++ b/webkit/appcache/manifest_parser_unittest.cc
@@ -0,0 +1,294 @@
+// Copyright (c) 2009 The Chromium Authos. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+
+#include "googleurl/src/gurl.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webkit/appcache/manifest_parser.h"
+
+using appcache::FallbackUrlVector;
+using appcache::Manifest;
+using appcache::ParseManifest;
+
+namespace {
+
+class ManifestParserTest : public testing::Test {
+};
+
+}  // namespace
+
+TEST(ManifestParserTest, NoData) {
+  GURL url;
+  Manifest manifest;
+  EXPECT_FALSE(ParseManifest(url, "", 0, manifest));
+  EXPECT_FALSE(ParseManifest(url, "CACHE MANIFEST\r", 0, manifest));  // 0 len
+}
+
+TEST(ManifestParserTest, CheckSignature) {
+  GURL url;
+  Manifest manifest;
+
+  const std::string kBadSignatures[] = {
+    "foo",
+    "CACHE MANIFEST;V2\r",          // not followed by whitespace
+    "CACHE MANIFEST#bad\r",         // no whitespace before comment
+    "cache manifest ",              // wrong case
+    "#CACHE MANIFEST\r",            // comment
+    "xCACHE MANIFEST\n",            // bad first char
+    " CACHE MANIFEST\r",            // begins with whitespace
+    "\xEF\xBE\xBF" "CACHE MANIFEST\r",  // bad UTF-8 BOM value
+  };
+
+  for (size_t i = 0; i < arraysize(kBadSignatures); ++i) {
+    const std::string bad = kBadSignatures[i];
+    EXPECT_FALSE(ParseManifest(url, bad.c_str(), bad.length(), manifest));
+  }
+
+  const std::string kGoodSignatures[] = {
+    "CACHE MANIFEST",
+    "CACHE MANIFEST ",
+    "CACHE MANIFEST\r",
+    "CACHE MANIFEST\n",
+    "CACHE MANIFEST\r\n",
+    "CACHE MANIFEST\t# ignore me\r",
+    "CACHE MANIFEST ignore\r\n",
+    "\xEF\xBB\xBF" "CACHE MANIFEST \r\n",   // BOM present
+  };
+
+  for (size_t i = 0; i < arraysize(kGoodSignatures); ++i) {
+    const std::string good = kGoodSignatures[i];
+    EXPECT_TRUE(ParseManifest(url, good.c_str(), good.length(), manifest));
+  }
+}
+
+TEST(ManifestParserTest, NoManifestUrl) {
+  Manifest manifest;
+  const std::string kData("CACHE MANIFEST\r"
+    "relative/tobase.com\r"
+    "http://absolute.com/addme.com");
+  const GURL kUrl = GURL::EmptyGURL();
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+  EXPECT_TRUE(manifest.explicit_urls.empty());
+  EXPECT_TRUE(manifest.online_whitelisted_urls.empty());
+  EXPECT_TRUE(manifest.fallback_urls.empty());
+}
+
+TEST(ManifestParserTest, ExplicitUrls) {
+  Manifest manifest;
+  const GURL kUrl("http://www.foo.com");
+  const std::string kData("CACHE MANIFEST\r"
+    "relative/one\r"
+    "# some comment\r"
+    "http://www.foo.com/two#strip\r\n"
+    "NETWORK:\r"
+    "  \t CACHE:\r"
+    "HTTP://www.diff.com/three\r"
+    "FALLBACK:\r"
+    " \t # another comment with leading whitespace\n"
+    "IGNORE:\r"
+    "http://www.foo.com/ignore\r"
+    "CACHE: \r"
+    "garbage:#!@\r"
+    "https://www.foo.com/diffscheme \t \r"
+    "  \t relative/four#stripme\n\r");
+
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+  EXPECT_TRUE(manifest.online_whitelisted_urls.empty());
+  EXPECT_TRUE(manifest.fallback_urls.empty());
+
+  base::hash_set<std::string> urls = manifest.explicit_urls;
+  const size_t kExpected = 4;
+  ASSERT_EQ(kExpected, urls.size());
+  EXPECT_TRUE(urls.find("http://www.foo.com/relative/one") != urls.end());
+  EXPECT_TRUE(urls.find("http://www.foo.com/two") != urls.end());
+  EXPECT_TRUE(urls.find("http://www.diff.com/three") != urls.end());
+  EXPECT_TRUE(urls.find("http://www.foo.com/relative/four") != urls.end());
+}
+
+TEST(ManifestParserTest, WhitelistUrls) {
+  Manifest manifest;
+  const GURL kUrl("http://www.bar.com");
+  const std::string kData("CACHE MANIFEST\r"
+    "NETWORK:\r"
+    "relative/one\r"
+    "# a comment\r"
+    "http://www.bar.com/two\r"
+    "HTTP://www.diff.com/three#strip\n\r"
+    "FALLBACK:\r"
+    "garbage\r"
+    "UNKNOWN:\r"
+    "http://www.bar.com/ignore\r"
+    "CACHE:\r"
+    "NETWORK:\r"
+    "https://www.wrongscheme.com\n"
+    "relative/four#stripref \t \r"
+    "http://www.five.com\r\n");
+
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+  EXPECT_TRUE(manifest.explicit_urls.empty());
+  EXPECT_TRUE(manifest.fallback_urls.empty());
+
+  std::vector<GURL> online = manifest.online_whitelisted_urls;
+  const size_t kExpected = 5;
+  ASSERT_EQ(kExpected, online.size());
+  EXPECT_EQ(GURL("http://www.bar.com/relative/one"), online[0]);
+  EXPECT_EQ(GURL("http://www.bar.com/two"), online[1]);
+  EXPECT_EQ(GURL("http://www.diff.com/three"), online[2]);
+  EXPECT_EQ(GURL("http://www.bar.com/relative/four"), online[3]);
+  EXPECT_EQ(GURL("http://www.five.com"), online[4]);
+}
+
+TEST(ManifestParserTest, FallbackUrls) {
+  Manifest manifest;
+  const GURL kUrl("http://glorp.com");
+  const std::string kData("CACHE MANIFEST\r"
+    "# a comment\r"
+    "CACHE:\r"
+    "NETWORK:\r"
+    "UNKNOWN:\r"
+    "FALLBACK:\r"
+    "relative/one \t \t http://glorp.com/onefb  \t \r"
+    "https://glorp.com/wrong http://glorp.com/wrongfb\r"
+    "http://glorp.com/two#strip relative/twofb\r"
+    "HTTP://glorp.com/three relative/threefb#strip\n"
+    "http://glorp.com/three http://glorp.com/three-dup\r"
+    "http://glorp.com/solo \t \r\n"
+    "http://diff.com/ignore http://glorp.com/wronghost\r"
+    "http://glorp.com/wronghost http://diff.com/ohwell\r"
+    "relative/badscheme ftp://glorp.com/ignored\r"
+    "garbage\r\n"
+    "CACHE:\r"
+    "# only fallback urls in this test\r"
+    "FALLBACK:\n"
+    "relative/four#strip relative/fourfb#strip\r"
+    "http://www.glorp.com/notsame relative/skipped\r");
+
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+  EXPECT_TRUE(manifest.explicit_urls.empty());
+  EXPECT_TRUE(manifest.online_whitelisted_urls.empty());
+
+  FallbackUrlVector fallbacks = manifest.fallback_urls;
+  const size_t kExpected = 5;
+  ASSERT_EQ(kExpected, fallbacks.size());
+  EXPECT_EQ(GURL("http://glorp.com/relative/one"),
+            fallbacks[0].first);
+  EXPECT_EQ(GURL("http://glorp.com/onefb"),
+            fallbacks[0].second);
+  EXPECT_EQ(GURL("http://glorp.com/two"),
+            fallbacks[1].first);
+  EXPECT_EQ(GURL("http://glorp.com/relative/twofb"),
+            fallbacks[1].second);
+  EXPECT_EQ(GURL("http://glorp.com/three"),
+            fallbacks[2].first);
+  EXPECT_EQ(GURL("http://glorp.com/relative/threefb"),
+            fallbacks[2].second);
+  EXPECT_EQ(GURL("http://glorp.com/three"),       // duplicates are stored
+            fallbacks[3].first);
+  EXPECT_EQ(GURL("http://glorp.com/three-dup"),
+            fallbacks[3].second);
+  EXPECT_EQ(GURL("http://glorp.com/relative/four"),
+            fallbacks[4].first);
+  EXPECT_EQ(GURL("http://glorp.com/relative/fourfb"),
+            fallbacks[4].second);
+}
+
+TEST(ManifestParserTest, FallbackUrlsWithPort) {
+  Manifest manifest;
+  const GURL kUrl("http://www.portme.com:1234");
+  const std::string kData("CACHE MANIFEST\r"
+    "FALLBACK:\r"
+    "http://www.portme.com:1234/one relative/onefb\r"
+    "HTTP://www.portme.com:9876/wrong http://www.portme.com:1234/ignore\r"
+    "http://www.portme.com:1234/stillwrong http://www.portme.com:42/boo\r"
+    "relative/two relative/twofb\r"
+    "http://www.portme.com:1234/three HTTP://www.portme.com:1234/threefb\r"
+    "http://www.portme.com/noport http://www.portme.com:1234/skipped\r"
+    "http://www.portme.com:1234/skipme http://www.portme.com/noport\r");
+
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+  EXPECT_TRUE(manifest.explicit_urls.empty());
+  EXPECT_TRUE(manifest.online_whitelisted_urls.empty());
+
+  FallbackUrlVector fallbacks = manifest.fallback_urls;
+  const size_t kExpected = 3;
+  ASSERT_EQ(kExpected, fallbacks.size());
+  EXPECT_EQ(GURL("http://www.portme.com:1234/one"),
+            fallbacks[0].first);
+  EXPECT_EQ(GURL("http://www.portme.com:1234/relative/onefb"),
+            fallbacks[0].second);
+  EXPECT_EQ(GURL("http://www.portme.com:1234/relative/two"),
+            fallbacks[1].first);
+  EXPECT_EQ(GURL("http://www.portme.com:1234/relative/twofb"),
+            fallbacks[1].second);
+  EXPECT_EQ(GURL("http://www.portme.com:1234/three"),
+            fallbacks[2].first);
+  EXPECT_EQ(GURL("http://www.portme.com:1234/threefb"),
+            fallbacks[2].second);
+}
+
+TEST(ManifestParserTest, ComboUrls) {
+  Manifest manifest;
+  const GURL kUrl("http://combo.com:42");
+  const std::string kData("CACHE MANIFEST\r"
+    "relative/explicit-1\r"
+    "# some comment\r"
+    "http://combo.com:99/explicit-2#strip\r"
+    "NETWORK:\r"
+    "http://combo.com/whitelist-1\r"
+    "HTTP://www.diff.com/whitelist-2#strip\r"
+    "CACHE:\n\r"
+    "http://www.diff.com/explicit-3\r"
+    "FALLBACK:\r"
+    "http://combo.com:42/fallback-1 http://combo.com:42/fallback-1b\r"
+    "relative/fallback-2 relative/fallback-2b\r"
+    "UNKNOWN:\r\n"
+    "http://combo.com/ignoreme\r"
+    "relative/still-ignored\r"
+    "NETWORK:\r\n"
+    "relative/whitelist-3#strip\r"
+    "http://combo.com:99/whitelist-4\r");
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+
+  base::hash_set<std::string> urls = manifest.explicit_urls;
+  size_t expected = 3;
+  ASSERT_EQ(expected, urls.size());
+  EXPECT_TRUE(urls.find("http://combo.com:42/relative/explicit-1") !=
+              urls.end());
+  EXPECT_TRUE(urls.find("http://combo.com:99/explicit-2") != urls.end());
+  EXPECT_TRUE(urls.find("http://www.diff.com/explicit-3") != urls.end());
+
+  std::vector<GURL> online = manifest.online_whitelisted_urls;
+  expected = 4;
+  ASSERT_EQ(expected, online.size());
+  EXPECT_EQ(GURL("http://combo.com/whitelist-1"), online[0]);
+  EXPECT_EQ(GURL("http://www.diff.com/whitelist-2"), online[1]);
+  EXPECT_EQ(GURL("http://combo.com:42/relative/whitelist-3"), online[2]);
+  EXPECT_EQ(GURL("http://combo.com:99/whitelist-4"), online[3]);
+
+  FallbackUrlVector fallbacks = manifest.fallback_urls;
+  expected = 2;
+  ASSERT_EQ(expected, fallbacks.size());
+  EXPECT_EQ(GURL("http://combo.com:42/fallback-1"),
+            fallbacks[0].first);
+  EXPECT_EQ(GURL("http://combo.com:42/fallback-1b"),
+            fallbacks[0].second);
+  EXPECT_EQ(GURL("http://combo.com:42/relative/fallback-2"),
+            fallbacks[1].first);
+  EXPECT_EQ(GURL("http://combo.com:42/relative/fallback-2b"),
+            fallbacks[1].second);
+}
+
+TEST(ManifestParserTest, UnusualUtf8) {
+  Manifest manifest;
+  const GURL kUrl("http://bad.com");
+  const std::string kData("CACHE MANIFEST\r"
+    "\xC0" "invalidutf8\r"
+    "nonbmp" "\xF1\x84\xAB\xBC\r");
+  EXPECT_TRUE(ParseManifest(kUrl, kData.c_str(), kData.length(), manifest));
+
+  base::hash_set<std::string> urls = manifest.explicit_urls;
+  EXPECT_TRUE(urls.find("http://bad.com/%EF%BF%BDinvalidutf8") != urls.end());
+  EXPECT_TRUE(urls.find("http://bad.com/nonbmp%F1%84%AB%BC") != urls.end());
+}