Make new FTP LIST parsing code more robust.

Some highlights:
 - more tests (including non-ASCII characters in different encodings)
 - handling more variants of "ls" listing
 - handling spaces in file names in "ls" listings

TEST=Covered by net_unittests.
BUG=25520

Review URL: http://codereview.chromium.org/449011

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@33688 0039d316-1c4b-4281-b951-d872f2087c98

15 files changed, 386 insertions, 16 deletions

diff --git a/net/data/ftp/dir-listing-ls-10 b/net/data/ftp/dir-listing-ls-10
new file mode 100644
index 0000000..deff68f
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-10
@@ -0,0 +1,10 @@
+Gesamt 4352
+---------- 1  1 1              0 Okt 25  1999 .notar
+lrwxrwxrwx 1  1 1              7 Okt 23  2007 bin -> usr/bin
+d--x--x--x 1  2 2            512 Apr 23  2002 dev
+d--x--x--x 1  2 2            512 Apr  1  2004 etc
+drwx------ 1  7 root        1536 Aug 14 13:49 lost+found
+drwxr-xr-x 1  3 1            512 Mär 10  2003 private
+drwxrwsr-x 1 25 1260        1024 Aug 10  2006 pub
+-rw------- 1  1 1        2211496 Okt 23  2007 restoresymtable
+d--x--x--x 1  6 2            512 Apr 23  2002 usr
diff --git a/net/data/ftp/dir-listing-ls-10.expected b/net/data/ftp/dir-listing-ls-10.expected
new file mode 100644
index 0000000..bbc4cd3
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-10.expected
@@ -0,0 +1,80 @@
+-
+.notar
+0
+1999
+10
+25
+0
+0
+
+l
+bin
+-1
+2007
+10
+23
+0
+0
+
+d
+dev
+-1
+2002
+4
+23
+0
+0
+
+d
+etc
+-1
+2004
+4
+1
+0
+0
+
+d
+lost+found
+-1
+current
+8
+14
+13
+49
+
+d
+private
+-1
+2003
+3
+10
+0
+0
+
+d
+pub
+-1
+2006
+8
+10
+0
+0
+
+-
+restoresymtable
+2211496 
+2007
+10
+23
+0
+0
+
+d
+usr
+-1
+2002
+4
+23
+0
+0
diff --git a/net/data/ftp/dir-listing-ls-6 b/net/data/ftp/dir-listing-ls-6
new file mode 100644
index 0000000..5cc802c
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-6
@@ -0,0 +1,7 @@
+total 14
+dr-xr-xr-x   7 0        1            512 Jun 19  2006 .
+dr-xr-xr-x   7 0        1            512 Jun 19  2006 ..
+dr-xr-xr-x   2 0        1            512 Mar 24  2003 bin
+dr-xr-xr-x   2 0        1            512 Mar 24  2003 etc
+dr-xr-xr-x  12 0        0            512 Apr  7  2009 pub
+dr-xr-xr-x   3 0        1            512 Mar 24  2003 usr
diff --git a/net/data/ftp/dir-listing-ls-6.expected b/net/data/ftp/dir-listing-ls-6.expected
new file mode 100644
index 0000000..80b61dd
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-6.expected
@@ -0,0 +1,53 @@
+d
+.
+-1
+2006
+6
+19
+0
+0
+
+d
+..
+-1
+2006
+6
+19
+0
+0
+
+d
+bin
+-1
+2003
+3
+24
+0
+0
+
+d
+etc
+-1
+2003
+3
+24
+0
+0
+
+d
+pub
+-1
+2009
+4
+7
+0
+0
+
+d
+usr
+-1
+2003
+3
+24
+0
+0
diff --git a/net/data/ftp/dir-listing-ls-7 b/net/data/ftp/dir-listing-ls-7
new file mode 100644
index 0000000..93dd804
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-7
@@ -0,0 +1,6 @@
+-rw-r--r--    1 0        100          3108 Mar 07  2001 00readme.html
+drwxr-xr-x    3 1164     100          4096 Oct 19 13:45 OCU
+lrwxrwxrwx    1 203      1              10 Jun 15  2006 año2000 -> ./urte2000
+drwxr-xr-x    2 0        0            4096 Mar 07  2001 bin
+drwxr-xr-x    2 0        100          4096 Mar 07  2001 dev
+drwxr-xr-x    3 0        100          4096 Apr 20  2005 doc
diff --git a/net/data/ftp/dir-listing-ls-7.expected b/net/data/ftp/dir-listing-ls-7.expected
new file mode 100644
index 0000000..8f4cfe0
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-7.expected
@@ -0,0 +1,53 @@
+-
+00readme.html
+3108
+2001
+3
+7
+0
+0
+
+d
+OCU
+-1
+current
+10
+19
+13
+45
+
+l
+año2000
+-1
+2006
+6
+15
+0
+0
+
+d
+bin
+-1
+2001
+3
+7
+0
+0
+
+d
+dev
+-1
+2001
+3
+7
+0
+0
+
+d
+doc
+-1
+2005
+4
+20
+0
+0
diff --git a/net/data/ftp/dir-listing-ls-8 b/net/data/ftp/dir-listing-ls-8
new file mode 100644
index 0000000..9281104
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-8
@@ -0,0 +1 @@
+total 0
diff --git a/net/data/ftp/dir-listing-ls-8.expected b/net/data/ftp/dir-listing-ls-8.expected
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-8.expected
diff --git a/net/data/ftp/dir-listing-ls-9 b/net/data/ftp/dir-listing-ls-9
new file mode 100644
index 0000000..89df17c
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-9
@@ -0,0 +1,4 @@
+total 510528
+-rw-r--r--  1 nobody  nogroup  174680068 Jun  4 23:20 Akademia Teatralna spot.mpg
+-rw-r--r--  1 nobody  nogroup    3447432 May 18  2009 Foo - Instrukcja_Obsługi.pdf
+-rw-r--r--  1 nobody  nogroup   23197684 Jun  9 13:36 Zdjecia.zip
diff --git a/net/data/ftp/dir-listing-ls-9.expected b/net/data/ftp/dir-listing-ls-9.expected
new file mode 100644
index 0000000..08bdb20
--- /dev/null
+++ b/net/data/ftp/dir-listing-ls-9.expected
@@ -0,0 +1,26 @@
+-
+Akademia Teatralna spot.mpg
+174680068 
+current
+6
+4
+23
+20
+
+-
+Foo - Instrukcja_Obsługi.pdf
+3447432 
+2009
+5
+18
+0
+0
+
+-
+Zdjecia.zip
+23197684 
+current
+6
+9
+13
+36
diff --git a/net/data/ftp/dir-listing-vms-5 b/net/data/ftp/dir-listing-vms-5
new file mode 100644
index 0000000..d769993
--- /dev/null
+++ b/net/data/ftp/dir-listing-vms-5
@@ -0,0 +1,12 @@
+
+SYS:[ANON_LOGS]
+
+FTP_SERVER.LOG;1682         0  27-NOV-2009 14:35 [NET,ANONYMOUS] (RWED,RWED,,)
+FTP_SERVER_LOG.KEEP;11400
+                            2  19-DEC-1994 15:40 [NET,ANONYMOUS] (RWED,RWED,,)
+FTP_SERVER_LOG.SEARCH;1
+                          274   7-DEC-1993 15:54 [NET,ANONYMOUS] (RWED,RWED,,)
+TESTLOG.DAT;1               0  27-APR-1995 13:18 [NET,ANONYMOUS] (RWED,RWED,,)
+
+
+Total of 276 blocks in 4 files.
diff --git a/net/data/ftp/dir-listing-vms-5.expected b/net/data/ftp/dir-listing-vms-5.expected
new file mode 100644
index 0000000..a768a23
--- /dev/null
+++ b/net/data/ftp/dir-listing-vms-5.expected
@@ -0,0 +1,36 @@
+-
+ftp_server.log
+0
+2009
+11
+27
+14
+35
+
+-
+ftp_server_log.keep
+1024
+1994
+12
+19
+15
+40
+
+-
+ftp_server_log.search
+140288
+1993
+12
+7
+15
+54
+
+-
+testlog.dat
+0
+1995
+4
+27
+13
+18
+
diff --git a/net/ftp/ftp_directory_listing_buffer_unittest.cc b/net/ftp/ftp_directory_listing_buffer_unittest.cc
index d9da0cf..9aaa16f 100644
--- a/net/ftp/ftp_directory_listing_buffer_unittest.cc
+++ b/net/ftp/ftp_directory_listing_buffer_unittest.cc
@@ -23,12 +23,18 @@ TEST(FtpDirectoryListingBufferTest, Parse) {
     "dir-listing-ls-3",
     "dir-listing-ls-4",
     "dir-listing-ls-5",
+    "dir-listing-ls-6",
+    "dir-listing-ls-7",
+    "dir-listing-ls-8",
+    "dir-listing-ls-9",
+    "dir-listing-ls-10",
     "dir-listing-windows-1",
     "dir-listing-windows-2",
     "dir-listing-vms-1",
     "dir-listing-vms-2",
     "dir-listing-vms-3",
     "dir-listing-vms-4",
+    "dir-listing-vms-5",
   };
 
   FilePath test_dir;
diff --git a/net/ftp/ftp_directory_listing_parsers.cc b/net/ftp/ftp_directory_listing_parsers.cc
index 8963624..a559a69 100644
--- a/net/ftp/ftp_directory_listing_parsers.cc
+++ b/net/ftp/ftp_directory_listing_parsers.cc
@@ -4,6 +4,8 @@
 
 #include "net/ftp/ftp_directory_listing_parsers.h"
 
+#include <ctype.h>
+
 #include "base/string_util.h"
 
 namespace {
@@ -12,10 +14,16 @@ bool LooksLikeUnixPermission(const string16& text) {
   if (text.length() != 3)
     return false;
 
+  // Meaning of the flags:
+  // r - file is readable
+  // w - file is writable
+  // x - file is executable
+  // s or S - setuid/setgid bit set
+  // t or T - "sticky" bit set
   return ((text[0] == 'r' || text[0] == '-') &&
           (text[1] == 'w' || text[1] == '-') &&
           (text[2] == 'x' || text[2] == 's' || text[2] == 'S' ||
-           text[2] == '-'));
+           text[2] == 't' || text[2] == 'T' || text[2] == '-'));
 }
 
 bool LooksLikeUnixPermissionsListing(const string16& text) {
@@ -40,6 +48,22 @@ bool IsStringNonNegativeInteger(const string16& text) {
   return number >= 0;
 }
 
+string16 GetStringPartAfterColumns(const string16& text, int columns) {
+  DCHECK_LE(1, columns);
+  int columns_so_far = 0;
+  size_t last = 0;
+  for (size_t i = 1; i < text.length(); ++i) {
+    if (!isspace(text[i - 1]) && isspace(text[i])) {
+      last = i;
+      if (++columns_so_far == columns)
+        break;
+    }
+  }
+  string16 result(text.substr(last));
+  TrimWhitespace(result, TRIM_ALL, &result);
+  return result;
+}
+
 bool ThreeLetterMonthToNumber(const string16& text, int* number) {
   const static char* months[] = { "jan", "feb", "mar", "apr", "may", "jun",
                                   "jul", "aug", "sep", "oct", "nov", "dec" };
@@ -51,12 +75,36 @@ bool ThreeLetterMonthToNumber(const string16& text, int* number) {
     }
   }
 
+  // Special cases for listings in German (other three-letter month
+  // abbreviations are the same as in English). Note that we don't need to do
+  // a case-insensitive compare here. Only "ls -l" style listings may use
+  // localized month names, and they will always start capitalized. Also,
+  // converting non-ASCII characters to lowercase would be more complicated.
+  if (text == UTF8ToUTF16("M\xc3\xa4r")) {
+    // The full month name is M-(a-umlaut)-rz (March), which is M-(a-umlaut)r
+    // when abbreviated.
+    *number = 3;
+    return true;
+  }
+  if (text == ASCIIToUTF16("Mai")) {
+    *number = 5;
+    return true;
+  }
+  if (text == ASCIIToUTF16("Okt")) {
+    *number = 10;
+    return true;
+  }
+  if (text == ASCIIToUTF16("Dez")) {
+    *number = 12;
+    return true;
+  }
+
   return false;
 }
 
 bool UnixDateListingToTime(const std::vector<string16>& columns,
                            base::Time* time) {
-  DCHECK_EQ(9U, columns.size());
+  DCHECK_LE(9U, columns.size());
 
   base::Time::Exploded time_exploded = { 0 };
 
@@ -289,25 +337,38 @@ FtpLsDirectoryListingParser::FtpLsDirectoryListingParser()
 }
 
 bool FtpLsDirectoryListingParser::ConsumeLine(const string16& line) {
-  // Allow empty lines only at the beginning of the listing. For example VMS
-  // systems in Unix emulation mode add an empty line before the first listing
-  // entry.
-  if (line.empty() && !received_nonempty_line_)
+  if (StartsWith(line, ASCIIToUTF16("total "), true) ||
+      StartsWith(line, ASCIIToUTF16("Gesamt "), true)) {
+    // Some FTP servers put a "total n" line at the beginning of the listing
+    // (n is an integer). Allow such a line, but only once, and only if it's
+    // the first non-empty line.
+    //
+    // Note: "Gesamt" is a German word for "total". The case is important here:
+    // for "ls -l" style listings, "total" will be lowercase, and Gesamt will be
+    // capitalized. This helps us distinguish that from a VMS-style listing,
+    // which would use "Total" (note the uppercase first letter).
+
+    if (received_nonempty_line_)
+      return false;
+
+    received_nonempty_line_ = true;
+    return true;
+  }
+  if (line.empty() && !received_nonempty_line_) {
+    // Allow empty lines only at the beginning of the listing. For example VMS
+    // systems in Unix emulation mode add an empty line before the first listing
+    // entry.
     return true;
+  }
   received_nonempty_line_ = true;
 
   std::vector<string16> columns;
   SplitString(CollapseWhitespace(line, false), ' ', &columns);
-  if (columns.size() == 11) {
-    // Check if it is a symlink.
-    if (!EqualsASCII(columns[9], "->"))
-      return false;
-
-    // Drop the symlink target from columns, we don't use it.
-    columns.resize(9);
-  }
 
-  if (columns.size() != 9)
+  // We may receive file names containing spaces, which can make the number of
+  // columns arbitrarily large. We will handle that later. For now just make
+  // sure we have all the columns that should normally be there.
+  if (columns.size() < 9)
     return false;
 
   if (!LooksLikeUnixPermissionsListing(columns[0]))
@@ -335,7 +396,13 @@ bool FtpLsDirectoryListingParser::ConsumeLine(const string16& line) {
   if (!UnixDateListingToTime(columns, &entry.last_modified))
     return false;
 
-  entry.name = columns[8];
+  entry.name = GetStringPartAfterColumns(line, 8);
+  if (entry.type == FtpDirectoryListingEntry::SYMLINK) {
+    string16::size_type pos = entry.name.rfind(ASCIIToUTF16(" -> "));
+    if (pos == string16::npos)
+      return false;
+    entry.name = entry.name.substr(0, pos);
+  }
 
   entries_.push(entry);
   return true;
diff --git a/net/ftp/ftp_directory_listing_parsers_unittest.cc b/net/ftp/ftp_directory_listing_parsers_unittest.cc
index 80ddc7e..5bb69e7 100644
--- a/net/ftp/ftp_directory_listing_parsers_unittest.cc
+++ b/net/ftp/ftp_directory_listing_parsers_unittest.cc
@@ -71,6 +71,15 @@ TEST_F(FtpDirectoryListingParsersTest, Ls) {
     { "drwxrwsr-x    4 501      501          4096 Feb 20  2007 pub",
       net::FtpDirectoryListingEntry::DIRECTORY, "pub", -1,
       2007, 2, 20, 0, 0 },
+    { "drwxr-xr-x   4 (?)      (?)          4096 Apr  8  2007 jigdo",
+      net::FtpDirectoryListingEntry::DIRECTORY, "jigdo", -1,
+      2007, 4, 8, 0, 0 },
+    { "drwx-wx-wt  2 root  wheel  512 Jul  1 02:15 incoming",
+      net::FtpDirectoryListingEntry::DIRECTORY, "incoming", -1,
+      now_exploded.year, 7, 1, 2, 15 },
+    { "-rw-r--r-- 1 2 3 3447432 May 18  2009 Foo - Manual.pdf",
+      net::FtpDirectoryListingEntry::FILE, "Foo - Manual.pdf", 3447432,
+      2009, 5, 18, 0, 0 },
   };
   for (size_t i = 0; i < arraysize(good_cases); i++) {
     SCOPED_TRACE(StringPrintf("Test[%" PRIuS "]: %s", i, good_cases[i].input));