Add first class support for user scripts.

Original review: http://codereview.chromium.org/340057

TBR=mpcomplete@chromium.org
BUG=22103
TEST=Install a user script (such as from userscripts.org). You should get the extension install UI and the
script should show up in the extension management UI. It should also work, though some scripts use
Firefox-specific APIs and those won't work in Chromium.

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@30925 0039d316-1c4b-4281-b951-d872f2087c98

8 files changed, 343 insertions, 42 deletions

diff --git a/chrome/browser/extensions/convert_user_script.cc b/chrome/browser/extensions/convert_user_script.cc
new file mode 100644
index 0000000..70de6d9
--- /dev/null
+++ b/chrome/browser/extensions/convert_user_script.cc
@@ -0,0 +1,138 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/extensions/convert_user_script.h"
+
+#include <string>
+#include <vector>
+
+#include "base/file_path.h"
+#include "base/file_util.h"
+#include "base/scoped_temp_dir.h"
+#include "base/sha2.h"
+#include "base/string_util.h"
+#include "chrome/browser/extensions/user_script_master.h"
+#include "chrome/common/extensions/extension.h"
+#include "chrome/common/extensions/extension_constants.h"
+#include "chrome/common/extensions/user_script.h"
+#include "chrome/common/json_value_serializer.h"
+#include "googleurl/src/gurl.h"
+#include "net/base/base64.h"
+
+namespace keys = extension_manifest_keys;
+
+Extension* ConvertUserScriptToExtension(const FilePath& user_script_path,
+                                        const GURL& original_url,
+                                        std::string* error){
+  std::string content;
+  if (!file_util::ReadFileToString(user_script_path, &content)) {
+    *error = "Could not read source file: " +
+        WideToASCII(user_script_path.ToWStringHack());
+    return NULL;
+  }
+
+  UserScript script;
+  if (!UserScriptMaster::ScriptReloader::ParseMetadataHeader(content,
+                                                             &script)) {
+    *error = "Invalid script header.";
+    return NULL;
+  }
+
+  ScopedTempDir temp_dir;
+  if (!temp_dir.CreateUniqueTempDir()) {
+    *error = "Could not create temporary directory.";
+    return NULL;
+  }
+
+  // Create the manifest
+  scoped_ptr<DictionaryValue> root(new DictionaryValue);
+  std::string script_name;
+  if (!script.name().empty() && !script.name_space().empty())
+    script_name = script.name_space() + "/" + script.name();
+  else
+    script_name = original_url.spec();
+
+  // Create the public key.
+  // User scripts are not signed, but the public key for an extension doubles as
+  // its unique identity, and we need one of those. A user script's unique
+  // identity is its namespace+name, so we hash that to create a public key.
+  // There will be no corresponding private key, which means user scripts cannot
+  // be auto-updated, or claimed in the gallery.
+  char raw[base::SHA256_LENGTH] = {0};
+  std::string key;
+  base::SHA256HashString(script_name, raw, base::SHA256_LENGTH);
+  net::Base64Encode(std::string(raw, base::SHA256_LENGTH), &key);
+
+  // The script may not have a name field, but we need one for an extension. If
+  // it is missing, use the filename of the original URL.
+  if (!script.name().empty())
+    root->SetString(keys::kName, script.name());
+  else
+    root->SetString(keys::kName, original_url.ExtractFileName());
+
+  root->SetString(keys::kDescription, script.description());
+  root->SetString(keys::kVersion, "1.0");
+  root->SetString(keys::kPublicKey, key);
+  root->SetBoolean(keys::kConvertedFromUserScript, true);
+
+  ListValue* js_files = new ListValue();
+  js_files->Append(Value::CreateStringValue("script.js"));
+
+  // If the script provides its own match patterns, we use those. Otherwise, we
+  // generate some using the include globs.
+  ListValue* matches = new ListValue();
+  if (!script.url_patterns().empty()) {
+    for (size_t i = 0; i < script.url_patterns().size(); ++i) {
+      matches->Append(Value::CreateStringValue(
+          script.url_patterns()[i].GetAsString()));
+    }
+  } else {
+    // TODO(aa): Derive tighter matches where possible.
+    matches->Append(Value::CreateStringValue("http://*/*"));
+    matches->Append(Value::CreateStringValue("https://*/*"));
+  }
+
+  ListValue* includes = new ListValue();
+  for (size_t i = 0; i < script.globs().size(); ++i)
+    includes->Append(Value::CreateStringValue(script.globs().at(i)));
+
+  ListValue* excludes = new ListValue();
+  for (size_t i = 0; i < script.exclude_globs().size(); ++i)
+    excludes->Append(Value::CreateStringValue(script.exclude_globs().at(i)));
+
+  DictionaryValue* content_script = new DictionaryValue();
+  content_script->Set(keys::kMatches, matches);
+  content_script->Set(keys::kIncludeGlobs, includes);
+  content_script->Set(keys::kExcludeGlobs, excludes);
+  content_script->Set(keys::kJs, js_files);
+
+  ListValue* content_scripts = new ListValue();
+  content_scripts->Append(content_script);
+
+  root->Set(keys::kContentScripts, content_scripts);
+
+  FilePath manifest_path = temp_dir.path().AppendASCII(
+      Extension::kManifestFilename);
+  JSONFileValueSerializer serializer(manifest_path);
+  if (!serializer.Serialize(*root)) {
+    *error = "Could not write JSON.";
+    return NULL;
+  }
+
+  // Write the script file.
+  if (!file_util::CopyFile(user_script_path,
+                           temp_dir.path().AppendASCII("script.js"))) {
+    *error = "Could not copy script file.";
+    return NULL;
+  }
+
+  scoped_ptr<Extension> extension(new Extension(temp_dir.path()));
+  if (!extension->InitFromValue(*root, false, error)) {
+    NOTREACHED() << "Could not init extension " << *error;
+    return NULL;
+  }
+
+  temp_dir.Take();  // The caller takes ownership of the directory.
+  return extension.release();
+}
diff --git a/chrome/browser/extensions/convert_user_script.h b/chrome/browser/extensions/convert_user_script.h
new file mode 100644
index 0000000..ebc9ee2
--- /dev/null
+++ b/chrome/browser/extensions/convert_user_script.h
@@ -0,0 +1,23 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_EXTENSIONS_CONVERT_USER_SCRIPT_H_
+#define CHROME_BROWSER_EXTENSIONS_CONVERT_USER_SCRIPT_H_
+
+#include <string>
+
+class Extension;
+class FilePath;
+class GURL;
+
+// Wraps the specified user script in an extension. The extension is created
+// unpacked in the system temp dir. Returns a valid extension that the caller
+// should take ownership on success, or NULL and |error| on failure.
+//
+// NOTE: This function does file IO and should not be called on the UI thread.
+Extension* ConvertUserScriptToExtension(const FilePath& user_script,
+                                        const GURL& original_url,
+                                        std::string* error);
+
+#endif  // CHROME_BROWSER_EXTENSIONS_CONVERT_USER_SCRIPT_H_
diff --git a/chrome/browser/extensions/convert_user_script_unittest.cc b/chrome/browser/extensions/convert_user_script_unittest.cc
new file mode 100644
index 0000000..b7aea54
--- /dev/null
+++ b/chrome/browser/extensions/convert_user_script_unittest.cc
@@ -0,0 +1,88 @@
+// Copyright (c) 2009 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <string>
+#include <vector>
+
+#include "base/file_path.h"
+#include "base/file_util.h"
+#include "base/path_service.h"
+#include "base/scoped_ptr.h"
+#include "chrome/browser/extensions/convert_user_script.h"
+#include "chrome/common/chrome_paths.h"
+#include "chrome/common/extensions/extension.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+TEST(ExtensionFromUserScript, Basic) {
+  FilePath test_file;
+  ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &test_file));
+  test_file = test_file.AppendASCII("extensions")
+                       .AppendASCII("user_script_basic.user.js");
+
+  std::string error;
+  scoped_ptr<Extension> extension(ConvertUserScriptToExtension(
+      test_file, GURL("http://www.google.com/foo"), &error));
+
+  ASSERT_TRUE(extension.get());
+  EXPECT_EQ("", error);
+
+  // Validate generated extension metadata.
+  EXPECT_EQ("My user script", extension->name());
+  EXPECT_EQ("1.0", extension->VersionString());
+  EXPECT_EQ("Does totally awesome stuff.", extension->description());
+  EXPECT_EQ("IhCFCg9PMQTAcJdc9ytUP99WME+4yh6aMnM1uupkovo=",
+            extension->public_key());
+
+  ASSERT_EQ(1u, extension->content_scripts().size());
+  const UserScript& script = extension->content_scripts()[0];
+  ASSERT_EQ(2u, script.globs().size());
+  EXPECT_EQ("http://www.google.com/*", script.globs().at(0));
+  EXPECT_EQ("http://www.yahoo.com/*", script.globs().at(1));
+  ASSERT_EQ(1u, script.exclude_globs().size());
+  EXPECT_EQ("*foo*", script.exclude_globs().at(0));
+  ASSERT_EQ(1u, script.url_patterns().size());
+  EXPECT_EQ("http://www.google.com/*", script.url_patterns()[0].GetAsString());
+
+  // Make sure the files actually exist on disk.
+  EXPECT_TRUE(file_util::PathExists(
+      extension->path().Append(script.js_scripts()[0].relative_path())));
+  EXPECT_TRUE(file_util::PathExists(
+      extension->path().AppendASCII(Extension::kManifestFilename)));
+}
+
+TEST(ExtensionFromUserScript, NoMetdata) {
+  FilePath test_file;
+  ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA, &test_file));
+  test_file = test_file.AppendASCII("extensions")
+                       .AppendASCII("user_script_no_metadata.user.js");
+
+  std::string error;
+  scoped_ptr<Extension> extension(ConvertUserScriptToExtension(
+      test_file, GURL("http://www.google.com/foo/bar.user.js?monkey"), &error));
+
+  ASSERT_TRUE(extension.get());
+  EXPECT_EQ("", error);
+
+  // Validate generated extension metadata.
+  EXPECT_EQ("bar.user.js", extension->name());
+  EXPECT_EQ("1.0", extension->VersionString());
+  EXPECT_EQ("", extension->description());
+  EXPECT_EQ("k1WxKx54hX6tfl5gQaXD/m4d9QUMwRdXWM4RW+QkWcY=",
+            extension->public_key());
+
+  ASSERT_EQ(1u, extension->content_scripts().size());
+  const UserScript& script = extension->content_scripts()[0];
+  ASSERT_EQ(1u, script.globs().size());
+  EXPECT_EQ("*", script.globs()[0]);
+  EXPECT_EQ(0u, script.exclude_globs().size());
+  ASSERT_EQ(2u, script.url_patterns().size());
+  EXPECT_EQ("http://*/*", script.url_patterns()[0].GetAsString());
+  EXPECT_EQ("https://*/*", script.url_patterns()[1].GetAsString());
+
+  // Make sure the files actually exist on disk.
+  EXPECT_TRUE(file_util::PathExists(
+      extension->path().Append(script.js_scripts()[0].relative_path())));
+  EXPECT_TRUE(file_util::PathExists(
+      extension->path().AppendASCII(Extension::kManifestFilename)));
+}
diff --git a/chrome/browser/extensions/crx_installer.cc b/chrome/browser/extensions/crx_installer.cc
index 6c6259f..cb5b4e4 100644
--- a/chrome/browser/extensions/crx_installer.cc
+++ b/chrome/browser/extensions/crx_installer.cc
@@ -11,6 +11,7 @@
 #include "base/task.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/chrome_thread.h"
+#include "chrome/browser/extensions/convert_user_script.h"
 #include "chrome/browser/extensions/extension_file_util.h"
 #include "chrome/common/extensions/extension_error_reporter.h"
 #include "chrome/common/notification_service.h"
@@ -31,41 +32,56 @@ void CrxInstaller::Start(const FilePath& crx_path,
                          const FilePath& install_directory,
                          Extension::Location install_source,
                          const std::string& expected_id,
-                         bool delete_crx,
+                         bool delete_source,
                          bool allow_privilege_increase,
                          ExtensionsService* frontend,
                          ExtensionInstallUI* client) {
   // Note: We don't keep a reference because this object manages its own
   // lifetime.
-  new CrxInstaller(crx_path, install_directory, install_source, expected_id,
-                   delete_crx, allow_privilege_increase, frontend, client);
+  CrxInstaller* installer = new CrxInstaller(crx_path, install_directory,
+                                             delete_source, frontend,
+                                             client);
+  installer->install_source_ = install_source;
+  installer->expected_id_ = expected_id;
+  installer->allow_privilege_increase_ = allow_privilege_increase;
+
+  installer->unpacker_ = new SandboxedExtensionUnpacker(
+      installer->source_file_, g_browser_process->resource_dispatcher_host(),
+      installer);
+
+  ChromeThread::PostTask(
+      ChromeThread::FILE, FROM_HERE,
+      NewRunnableMethod(installer->unpacker_, &SandboxedExtensionUnpacker::Start));
+}
+
+void CrxInstaller::InstallUserScript(const FilePath& source_file,
+                                     const GURL& original_url,
+                                     const FilePath& install_directory,
+                                     bool delete_source,
+                                     ExtensionsService* frontend,
+                                     ExtensionInstallUI* client) {
+  CrxInstaller* installer = new CrxInstaller(source_file, install_directory,
+                                             delete_source, frontend, client);
+  installer->original_url_ = original_url;
+
+  ChromeThread::PostTask(
+      ChromeThread::FILE, FROM_HERE,
+      NewRunnableMethod(installer, &CrxInstaller::ConvertUserScriptOnFileThread));
 }
 
-CrxInstaller::CrxInstaller(const FilePath& crx_path,
+CrxInstaller::CrxInstaller(const FilePath& source_file,
                            const FilePath& install_directory,
-                           Extension::Location install_source,
-                           const std::string& expected_id,
-                           bool delete_crx,
-                           bool allow_privilege_increase,
+                           bool delete_source,
                            ExtensionsService* frontend,
                            ExtensionInstallUI* client)
-    : crx_path_(crx_path),
+    : source_file_(source_file),
       install_directory_(install_directory),
-      install_source_(install_source),
-      expected_id_(expected_id),
-      delete_crx_(delete_crx),
-      allow_privilege_increase_(allow_privilege_increase),
+      install_source_(Extension::INTERNAL),
+      delete_source_(delete_source),
+      allow_privilege_increase_(false),
       frontend_(frontend),
       client_(client) {
-
   extensions_enabled_ = frontend_->extensions_enabled();
-
-  unpacker_ = new SandboxedExtensionUnpacker(
-      crx_path, g_browser_process->resource_dispatcher_host(), this);
-
-  ChromeThread::PostTask(
-      ChromeThread::FILE, FROM_HERE,
-      NewRunnableMethod(unpacker_, &SandboxedExtensionUnpacker::Start));
 }
 
 CrxInstaller::~CrxInstaller() {
@@ -78,11 +94,23 @@ CrxInstaller::~CrxInstaller() {
         NewRunnableFunction(&DeleteFileHelper, temp_dir_, true));
   }
 
-  if (delete_crx_) {
+  if (delete_source_) {
     ChromeThread::PostTask(
         ChromeThread::FILE, FROM_HERE,
-        NewRunnableFunction(&DeleteFileHelper, crx_path_, false));
+        NewRunnableFunction(&DeleteFileHelper, source_file_, false));
+  }
+}
+
+void CrxInstaller::ConvertUserScriptOnFileThread() {
+  std::string error;
+  Extension* extension = ConvertUserScriptToExtension(source_file_,
+                                                      original_url_, &error);
+  if (!extension) {
+    ReportFailureFromFileThread(error);
+    return;
   }
+
+  OnUnpackSuccess(extension->path(), extension->path(), extension);
 }
 
 void CrxInstaller::OnUnpackFailure(const std::string& error_message) {
@@ -102,7 +130,6 @@ void CrxInstaller::OnUnpackSuccess(const FilePath& temp_dir,
   // The unpack dir we don't have to delete explicity since it is a child of
   // the temp dir.
   unpacked_extension_root_ = extension_dir;
-  DCHECK(file_util::ContainsPath(temp_dir_, unpacked_extension_root_));
 
   // Determine whether to allow installation. We always allow themes and
   // external installs.
diff --git a/chrome/browser/extensions/crx_installer.h b/chrome/browser/extensions/crx_installer.h
index a0e50ef..bd78cf6 100644
--- a/chrome/browser/extensions/crx_installer.h
+++ b/chrome/browser/extensions/crx_installer.h
@@ -38,7 +38,7 @@ class CrxInstaller :
     public SandboxedExtensionUnpackerClient,
     public ExtensionInstallUI::Delegate {
  public:
-  // Starts the installation of the crx file in |crx_path| into
+  // Starts the installation of the crx file in |source_file| into
   // |install_directory|.
   //
   // Other params:
@@ -52,15 +52,25 @@ class CrxInstaller :
   //  client: Optional. If specified, will be used to confirm installation and
   //          also notified of success/fail. Note that we hold a reference to
   //          this, so it can outlive its creator (eg the UI).
-  static void Start(const FilePath& crx_path,
+  static void Start(const FilePath& source_file,
                     const FilePath& install_directory,
                     Extension::Location install_source,
                     const std::string& expected_id,
-                    bool delete_crx,
+                    bool delete_source,
                     bool allow_privilege_increase,
                     ExtensionsService* frontend,
                     ExtensionInstallUI* client);
 
+  // Starts the installation of the user script file in |source_file| into
+  // |install_directory|. The script will be converted to an extension.
+  // See Start() for argument descriptions.
+  static void InstallUserScript(const FilePath& source_file,
+                                const GURL& original_url,
+                                const FilePath& install_directory,
+                                bool delete_source,
+                                ExtensionsService* frontend,
+                                ExtensionInstallUI* client);
+
   // Given the path to the large icon from an extension, read it if present and
   // decode it into result.
   static void DecodeInstallIcon(const FilePath& large_icon_path,
@@ -71,16 +81,16 @@ class CrxInstaller :
   virtual void AbortInstall();
 
  private:
-  CrxInstaller(const FilePath& crx_path,
+  CrxInstaller(const FilePath& source_file,
                const FilePath& install_directory,
-               Extension::Location install_source,
-               const std::string& expected_id,
-               bool delete_crx,
-               bool allow_privilege_increase,
+               bool delete_source,
                ExtensionsService* frontend,
                ExtensionInstallUI* client);
   ~CrxInstaller();
 
+  // Converts the source user script to an extension.
+  void ConvertUserScriptOnFileThread();
+
   // SandboxedExtensionUnpackerClient
   virtual void OnUnpackFailure(const std::string& error_message);
   virtual void OnUnpackSuccess(const FilePath& temp_dir,
@@ -103,8 +113,11 @@ class CrxInstaller :
   void ReportSuccessFromFileThread();
   void ReportSuccessFromUIThread();
 
-  // The crx file we're installing.
-  FilePath crx_path_;
+  // The file we're installing.
+  FilePath source_file_;
+
+  // The URL the file was downloaded from. Only used for user scripts.
+  GURL original_url_;
 
   // The directory extensions are installed to.
   FilePath install_directory_;
@@ -123,8 +136,8 @@ class CrxInstaller :
   // allowed.
   bool extensions_enabled_;
 
-  // Whether we're supposed to delete the source crx file on destruction.
-  bool delete_crx_;
+  // Whether we're supposed to delete the source file on destruction.
+  bool delete_source_;
 
   // Whether privileges should be allowed to silently increaes from any
   // previously installed version of the extension.
diff --git a/chrome/browser/extensions/user_script_master.cc b/chrome/browser/extensions/user_script_master.cc
index 69fd746..c9228ff 100644
--- a/chrome/browser/extensions/user_script_master.cc
+++ b/chrome/browser/extensions/user_script_master.cc
@@ -51,7 +51,11 @@ bool UserScriptMaster::ScriptReloader::ParseMetadataHeader(
 
   static const base::StringPiece kUserScriptBegin("// ==UserScript==");
   static const base::StringPiece kUserScriptEng("// ==/UserScript==");
+  static const base::StringPiece kNamespaceDeclaration("// @namespace ");
+  static const base::StringPiece kNameDeclaration("// @name ");
+  static const base::StringPiece kDescriptionDeclaration("// @description ");
   static const base::StringPiece kIncludeDeclaration("// @include ");
+  static const base::StringPiece kExcludeDeclaration("// @exclude ");
   static const base::StringPiece kMatchDeclaration("// @match ");
   static const base::StringPiece kRunAtDeclaration("// @run-at ");
   static const base::StringPiece kRunAtDocumentStartValue("document-start");
@@ -79,6 +83,16 @@ bool UserScriptMaster::ScriptReloader::ParseMetadataHeader(
         ReplaceSubstringsAfterOffset(&value, 0, "\\", "\\\\");
         ReplaceSubstringsAfterOffset(&value, 0, "?", "\\?");
         script->add_glob(value);
+      } else if (GetDeclarationValue(line, kExcludeDeclaration, &value)) {
+        ReplaceSubstringsAfterOffset(&value, 0, "\\", "\\\\");
+        ReplaceSubstringsAfterOffset(&value, 0, "?", "\\?");
+        script->add_exclude_glob(value);
+      } else if (GetDeclarationValue(line, kNamespaceDeclaration, &value)) {
+        script->set_name_space(value);
+      } else if (GetDeclarationValue(line, kNameDeclaration, &value)) {
+        script->set_name(value);
+      } else if (GetDeclarationValue(line, kDescriptionDeclaration, &value)) {
+        script->set_description(value);
       } else if (GetDeclarationValue(line, kMatchDeclaration, &value)) {
         URLPattern pattern;
         if (!pattern.Parse(value))
@@ -97,10 +111,6 @@ bool UserScriptMaster::ScriptReloader::ParseMetadataHeader(
     line_start = line_end + 1;
   }
 
-  // It is probably a mistake to declare both @include and @match rules.
-  if (script->globs().size() > 0 && script->url_patterns().size() > 0)
-    return false;
-
   // If no patterns were specified, default to @include *. This is what
   // Greasemonkey does.
   if (script->globs().size() == 0 && script->url_patterns().size() == 0)
diff --git a/chrome/browser/extensions/user_script_master.h b/chrome/browser/extensions/user_script_master.h
index a062448..270a690 100644
--- a/chrome/browser/extensions/user_script_master.h
+++ b/chrome/browser/extensions/user_script_master.h
@@ -61,6 +61,7 @@ class UserScriptMaster : public base::RefCountedThreadSafe<UserScriptMaster>,
   FRIEND_TEST(UserScriptMasterTest, Parse5);
   FRIEND_TEST(UserScriptMasterTest, Parse6);
 
+ public:
   // We reload user scripts on the file thread to prevent blocking the UI.
   // ScriptReloader lives on the file thread and does the reload
   // work, and then sends a message back to its master with a new SharedMemory*.
@@ -118,6 +119,7 @@ class UserScriptMaster : public base::RefCountedThreadSafe<UserScriptMaster>,
     DISALLOW_COPY_AND_ASSIGN(ScriptReloader);
   };
 
+ private:
   // DirectoryWatcher::Delegate implementation.
   virtual void OnDirectoryChanged(const FilePath& path);
 
diff --git a/chrome/browser/extensions/user_script_master_unittest.cc b/chrome/browser/extensions/user_script_master_unittest.cc
index 840d8a3..29faaea 100644
--- a/chrome/browser/extensions/user_script_master_unittest.cc
+++ b/chrome/browser/extensions/user_script_master_unittest.cc
@@ -213,8 +213,8 @@ TEST_F(UserScriptMasterTest, Parse6) {
     "// @match  \t http://mail.yahoo.com/*\n"
     "// ==/UserScript==\n");
 
-  // Not allowed to mix @include and @value.
+  // Allowed to match @include and @match.
   UserScript script;
-  EXPECT_FALSE(UserScriptMaster::ScriptReloader::ParseMetadataHeader(
+  EXPECT_TRUE(UserScriptMaster::ScriptReloader::ParseMetadataHeader(
       text, &script));
 }