Remove the fallback Mozilla code for parsing FTP LIST response.

The new parser seems to be compatible enough to do that. The Mozilla code was very
helpful in the process of developing the new parser.

Also add UI encouraging users to submit bug reports when we can't parse the listings,
and an option to see the raw data sent by the server. This should allow us to fix
remaining compatibility problems with very rare listing types or variations.

When ?raw is found at the end of an FTP url and it is a directory listing, the parsing logic is bypassed and the data is displayed as-is with text/plain MIME type.

TEST=none
BUG=25520

Review URL: http://codereview.chromium.org/549053

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@36632 0039d316-1c4b-4281-b951-d872f2087c98

4 files changed, 55 insertions, 210 deletions

diff --git a/webkit/glue/DEPS b/webkit/glue/DEPS
index 05f658c..1e84e42 100644
--- a/webkit/glue/DEPS
+++ b/webkit/glue/DEPS
@@ -1,7 +1,6 @@
 include_rules = [
   "+app",
   "+media",
-  "+net/third_party/parseftp",
   "+skia/ext",
   "+skia/include",
   "+webkit/tools/test_shell",  # Needed for test shell tests.
diff --git a/webkit/glue/ftp_directory_listing_response_delegate.cc b/webkit/glue/ftp_directory_listing_response_delegate.cc
index c5b39b0..cd481d8 100644
--- a/webkit/glue/ftp_directory_listing_response_delegate.cc
+++ b/webkit/glue/ftp_directory_listing_response_delegate.cc
@@ -20,6 +20,8 @@
 #include "third_party/WebKit/WebKit/chromium/public/WebURL.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebURLLoaderClient.h"
 
+using net::FtpDirectoryListingEntry;
+
 using WebKit::WebURLLoader;
 using WebKit::WebURLLoaderClient;
 using WebKit::WebURLResponse;
@@ -64,53 +66,6 @@ string16 RawByteSequenceToFilename(const char* raw_filename,
   return filename;
 }
 
-void ExtractFullLinesFromBuffer(std::string* buffer,
-                                std::vector<std::string>* lines) {
-  int cut_pos = 0;
-  for (size_t i = 0; i < buffer->length(); i++) {
-    if ((*buffer)[i] != '\n')
-      continue;
-    size_t line_length = i - cut_pos;
-    if (line_length > 0 && (*buffer)[i - 1] == '\r')
-      line_length--;
-    lines->push_back(buffer->substr(cut_pos, line_length));
-    cut_pos = i + 1;
-  }
-  buffer->erase(0, cut_pos);
-}
-
-void LogFtpServerType(char server_type) {
-  switch (server_type) {
-    case 'E':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_EPLF);
-      break;
-    case 'V':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_VMS);
-      break;
-    case 'C':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_CMS);
-      break;
-    case 'W':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_DOS);
-      break;
-    case 'O':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_OS2);
-      break;
-    case 'U':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_LSL);
-      break;
-    case 'w':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_W16);
-      break;
-    case 'D':
-      net::UpdateFtpServerTypeHistograms(net::SERVER_MOZ_DLS);
-      break;
-    default:
-      net::UpdateFtpServerTypeHistograms(net::SERVER_UNKNOWN);
-      break;
-  }
-}
-
 }  // namespace
 
 namespace webkit_glue {
@@ -122,51 +77,33 @@ FtpDirectoryListingResponseDelegate::FtpDirectoryListingResponseDelegate(
     : client_(client),
       loader_(loader),
       original_response_(response),
-      parser_fallback_(false) {
+      updated_histograms_(false),
+      had_parsing_error_(false) {
   Init();
 }
 
 void FtpDirectoryListingResponseDelegate::OnReceivedData(const char* data,
                                                          int data_len) {
-  // Keep the raw data in case we have to use the old parser later.
-  input_buffer_.append(data, data_len);
+  if (had_parsing_error_)
+    return;
 
-  if (!parser_fallback_) {
-    if (buffer_.ConsumeData(data, data_len) == net::OK)
-      return;
-    parser_fallback_ = true;
-  }
-
-  FeedFallbackParser();
-  SendResponseBufferToClient();
+  if (buffer_.ConsumeData(data, data_len) == net::OK)
+    ProcessReceivedEntries();
+  else
+    had_parsing_error_ = true;
 }
 
 void FtpDirectoryListingResponseDelegate::OnCompletedRequest() {
-  if (!parser_fallback_) {
-    if (buffer_.ProcessRemainingData() == net::OK) {
-      if (buffer_.EntryAvailable()) {
-        // Only log the server type if we got enough data to reliably detect it.
-        net::UpdateFtpServerTypeHistograms(buffer_.GetServerType());
-      }
-      while (buffer_.EntryAvailable())
-        AppendEntryToResponseBuffer(buffer_.PopEntry());
-      SendResponseBufferToClient();
-      return;
-    }
-    parser_fallback_ = true;
-  }
+  if (!had_parsing_error_ && buffer_.ProcessRemainingData() == net::OK)
+    ProcessReceivedEntries();
+  else
+    had_parsing_error_ = true;
 
-  FeedFallbackParser();
-  SendResponseBufferToClient();
-
-  // Only log the server type if we got enough data to reliably detect it.
-  if (parse_state_.parsed_one)
-    LogFtpServerType(parse_state_.lstyle);
+  if (had_parsing_error_)
+    SendDataToClient("<script>onListingParsingError();</script>\n");
 }
 
 void FtpDirectoryListingResponseDelegate::Init() {
-  memset(&parse_state_, 0, sizeof(parse_state_));
-
   GURL response_url(original_response_.url());
   UnescapeRule::Type unescape_rules = UnescapeRule::SPACES |
                                       UnescapeRule::URL_SPECIAL_CHARS;
@@ -189,123 +126,37 @@ void FtpDirectoryListingResponseDelegate::Init() {
       path_utf16 = WideToUTF16Hack(base::SysNativeMBToWide(unescaped_path));
   }
 
-  response_buffer_ = net::GetDirectoryListingHeader(path_utf16);
+  SendDataToClient(net::GetDirectoryListingHeader(path_utf16));
 
   // If this isn't top level directory (i.e. the path isn't "/",)
   // add a link to the parent directory.
   if (response_url.path().length() > 1) {
-    response_buffer_.append(
-        net::GetDirectoryListingEntry(ASCIIToUTF16(".."),
-                                      std::string(),
-                                      false, 0,
-                                      base::Time()));
+    SendDataToClient(net::GetDirectoryListingEntry(
+        ASCIIToUTF16(".."), std::string(), false, 0, base::Time()));
   }
 }
 
-void FtpDirectoryListingResponseDelegate::FeedFallbackParser() {
-  // If all we've seen so far is ASCII, encoding_ is empty. Try to detect the
-  // encoding. We don't do the separate UTF-8 check here because the encoding
-  // detection with a longer chunk (as opposed to the relatively short path
-  // component of the url) is unlikely to mistake UTF-8 for a legacy encoding.
-  // If it turns out to be wrong, a separate UTF-8 check has to be added.
-  //
-  // TODO(jungshik): UTF-8 has to be 'enforced' without any heuristics when
-  // we're talking to an FTP server compliant to RFC 2640 (that is, its response
-  // to FEAT command includes 'UTF8').
-  // See http://wiki.filezilla-project.org/Character_Set
-  if (encoding_.empty())
-    encoding_ = DetectEncoding(input_buffer_);
-
-  std::vector<std::string> lines;
-  ExtractFullLinesFromBuffer(&input_buffer_, &lines);
-
-  for (std::vector<std::string>::const_iterator line = lines.begin();
-       line != lines.end(); ++line) {
-    struct net::list_result result;
-    int line_type = net::ParseFTPList(line->c_str(), &parse_state_, &result);
-
-    // The original code assumed months are in range 0-11 (PRExplodedTime),
-    // but our Time class expects a 1-12 range. Adjust it here, because
-    // the third-party parsing code uses bit-shifting on the month,
-    // and it'd be too easy to break that logic.
-    result.fe_time.month++;
-    DCHECK_LE(1, result.fe_time.month);
-    DCHECK_GE(12, result.fe_time.month);
-
-    int64 file_size;
-    switch (line_type) {
-      case 'd':  // Directory entry.
-        response_buffer_.append(net::GetDirectoryListingEntry(
-            RawByteSequenceToFilename(result.fe_fname, encoding_),
-            result.fe_fname, true, 0,
-            base::Time::FromLocalExploded(result.fe_time)));
-        break;
-      case 'f':  // File entry.
-        if (StringToInt64(result.fe_size, &file_size)) {
-          response_buffer_.append(net::GetDirectoryListingEntry(
-              RawByteSequenceToFilename(result.fe_fname, encoding_),
-              result.fe_fname, false, file_size,
-              base::Time::FromLocalExploded(result.fe_time)));
-        }
-        break;
-      case 'l': {  // Symlink entry.
-          std::string filename(result.fe_fname, result.fe_fnlen);
-
-          // Parsers for styles 'U' and 'W' handle " -> " themselves.
-          if (parse_state_.lstyle != 'U' && parse_state_.lstyle != 'W') {
-            std::string::size_type offset = filename.find(" -> ");
-            if (offset != std::string::npos)
-              filename = filename.substr(0, offset);
-          }
-
-          if (StringToInt64(result.fe_size, &file_size)) {
-            response_buffer_.append(net::GetDirectoryListingEntry(
-                RawByteSequenceToFilename(filename.c_str(), encoding_),
-                filename, false, file_size,
-                base::Time::FromLocalExploded(result.fe_time)));
-          }
-        }
-        break;
-      case '?':  // Junk entry.
-      case '"':  // Comment entry.
-        break;
-      default:
-        NOTREACHED();
-        break;
-    }
+void FtpDirectoryListingResponseDelegate::ProcessReceivedEntries() {
+  if (!updated_histograms_ && buffer_.EntryAvailable()) {
+    // Only log the server type if we got enough data to reliably detect it.
+    net::UpdateFtpServerTypeHistograms(buffer_.GetServerType());
+    updated_histograms_ = true;
   }
-}
 
-void FtpDirectoryListingResponseDelegate::AppendEntryToResponseBuffer(
-    const net::FtpDirectoryListingEntry& entry) {
-  switch (entry.type) {
-    case net::FtpDirectoryListingEntry::FILE:
-      response_buffer_.append(
-          net::GetDirectoryListingEntry(entry.name, std::string(), false,
-                                        entry.size, entry.last_modified));
-      break;
-    case net::FtpDirectoryListingEntry::DIRECTORY:
-      response_buffer_.append(
-          net::GetDirectoryListingEntry(entry.name, std::string(), true,
-                                        0, entry.last_modified));
-      break;
-    case net::FtpDirectoryListingEntry::SYMLINK:
-      response_buffer_.append(
-          net::GetDirectoryListingEntry(entry.name, std::string(), false,
-                                        0, entry.last_modified));
-      break;
-    default:
-      NOTREACHED();
-      break;
+  while (buffer_.EntryAvailable()) {
+    FtpDirectoryListingEntry entry = buffer_.PopEntry();
+    bool is_directory = (entry.type == FtpDirectoryListingEntry::DIRECTORY);
+    int64 size = entry.size;
+    if (entry.type != FtpDirectoryListingEntry::FILE)
+      size = 0;
+    SendDataToClient(net::GetDirectoryListingEntry(
+        entry.name, std::string(), is_directory, size, entry.last_modified));
   }
 }
 
-void FtpDirectoryListingResponseDelegate::SendResponseBufferToClient() {
-  if (!response_buffer_.empty()) {
-    client_->didReceiveData(loader_, response_buffer_.data(),
-                            response_buffer_.length());
-    response_buffer_.clear();
-  }
+void FtpDirectoryListingResponseDelegate::SendDataToClient(
+    const std::string& data) {
+  client_->didReceiveData(loader_, data.data(), data.length());
 }
 
 }  // namespace webkit_glue
diff --git a/webkit/glue/ftp_directory_listing_response_delegate.h b/webkit/glue/ftp_directory_listing_response_delegate.h
index d5c7c21..86b5c436 100644
--- a/webkit/glue/ftp_directory_listing_response_delegate.h
+++ b/webkit/glue/ftp_directory_listing_response_delegate.h
@@ -11,7 +11,6 @@
 #include <string>
 
 #include "net/ftp/ftp_directory_listing_buffer.h"
-#include "net/third_party/parseftp/ParseFTPList.h"
 #include "third_party/WebKit/WebKit/chromium/public/WebURLResponse.h"
 
 namespace WebKit {
@@ -34,12 +33,9 @@ class FtpDirectoryListingResponseDelegate {
  private:
   void Init();
 
-  // Use the old parser to process received listing data.
-  void FeedFallbackParser();
+  void ProcessReceivedEntries();
 
-  void AppendEntryToResponseBuffer(const net::FtpDirectoryListingEntry& entry);
-
-  void SendResponseBufferToClient();
+  void SendDataToClient(const std::string& data);
 
   // Pointers to the client and associated loader so we can make callbacks as
   // we parse pieces of data.
@@ -50,25 +46,14 @@ class FtpDirectoryListingResponseDelegate {
   // starting point for each parts response.
   WebKit::WebURLResponse original_response_;
 
-  // Data buffer also responsible for parsing the listing data (the new parser).
-  // TODO(phajdan.jr): Use only the new parser, when it is more compatible.
+  // Data buffer also responsible for parsing the listing data.
   net::FtpDirectoryListingBuffer buffer_;
 
-  // True if the new parser couldn't recognize the received listing format
-  // and we switched to the old parser.
-  bool parser_fallback_;
-
-  // State kept between parsing each line of the response.
-  struct net::list_state parse_state_;
-
-  // Detected encoding of the response.
-  std::string encoding_;
-
-  // Buffer to hold not-yet-parsed input.
-  std::string input_buffer_;
+  // True if we updated histogram data (we only want to do it once).
+  bool updated_histograms_;
 
-  // Buffer to hold response not-yet-sent to the caller.
-  std::string response_buffer_;
+  // True if we got an error when parsing the response.
+  bool had_parsing_error_;
 };
 
 }  // namespace webkit_glue
diff --git a/webkit/glue/weburlloader_impl.cc b/webkit/glue/weburlloader_impl.cc
index ebfb11d..421cb8f 100644
--- a/webkit/glue/weburlloader_impl.cc
+++ b/webkit/glue/weburlloader_impl.cc
@@ -452,8 +452,17 @@ void WebURLLoaderImpl::Context::OnReceivedResponse(
   PopulateURLResponse(request_.url(), info, &response);
   response.setIsContentFiltered(content_filtered);
 
-  if (info.mime_type == "text/vnd.chromium.ftp-dir")
-    response.setMIMEType(WebString::fromUTF8("text/html"));
+  bool show_raw_listing = (GURL(request_.url()).query() == "raw");
+
+  if (info.mime_type == "text/vnd.chromium.ftp-dir") {
+    if (show_raw_listing) {
+      // Set the MIME type to plain text to prevent any active content.
+      response.setMIMEType("text/plain");
+    } else {
+      // We're going to produce a parsed listing in HTML.
+      response.setMIMEType("text/html");
+    }
+  }
 
   client_->didReceiveResponse(loader_, response);
 
@@ -477,7 +486,8 @@ void WebURLLoaderImpl::Context::OnReceivedResponse(
       multipart_delegate_.reset(
           new MultipartResponseDelegate(client_, loader_, response, boundary));
     }
-  } else if (info.mime_type == "text/vnd.chromium.ftp-dir") {
+  } else if (info.mime_type == "text/vnd.chromium.ftp-dir" &&
+             !show_raw_listing) {
     ftp_listing_delegate_.reset(
         new FtpDirectoryListingResponseDelegate(client_, loader_, response));
   }