Fix parsing of vmstat, improve meminfo parsing, and add unittests

The parsing of both vmstat and meminfo was previously based on fixed
positions for various entries with the files, and meminfo already had
some iteration code for where this assumption wasn't working.
Neither of those files make guarantees about positions or even
existence of some of the entries.  This fixes the issue entirely by
iterating through the files for all entries, and adds unit tests to
make sure they work properly with files from different kernels.

This does change the behavior of the meminfo parsing code a bit to
make it more permissive than previously, but that should also ensure
that it works better on older or different kernels.

BUG=295111,309930
TEST=base_unittests  --gtest_filter=SystemMetricsTest.*
R=brettw@chromium.org

Review URL: https://codereview.chromium.org/34683007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@233102 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 311 insertions, 80 deletions

diff --git a/base/process/process_metrics_linux.cc b/base/process/process_metrics_linux.cc
index e2a97c3..afa8848 100644
--- a/base/process/process_metrics_linux.cc
+++ b/base/process/process_metrics_linux.cc
@@ -405,33 +405,6 @@ int GetNumberOfThreads(ProcessHandle process) {
 
 namespace {
 
-// The format of /proc/meminfo is:
-//
-// MemTotal:      8235324 kB
-// MemFree:       1628304 kB
-// Buffers:        429596 kB
-// Cached:        4728232 kB
-// ...
-const size_t kMemTotalIndex = 1;
-const size_t kMemFreeIndex = 4;
-const size_t kMemBuffersIndex = 7;
-const size_t kMemCachedIndex = 10;
-const size_t kMemActiveAnonIndex = 22;
-const size_t kMemInactiveAnonIndex = 25;
-const size_t kMemActiveFileIndex = 28;
-const size_t kMemInactiveFileIndex = 31;
-
-// The format of /proc/vmstat is:
-//
-// nr_free_pages 299878
-// nr_inactive_anon 239863
-// nr_active_anon 1318966
-// nr_inactive_file 2015629
-// ...
-const size_t kVMPagesSwappedIn = 75;
-const size_t kVMPagesSwappedOut = 77;
-const size_t kVMPageMajorFaults = 95;
-
 // The format of /proc/diskstats is:
 //  Device major number
 //  Device minor number
@@ -541,53 +514,81 @@ scoped_ptr<Value> SystemMemoryInfoKB::ToValue() const {
 // exposed for testing
 bool ParseProcMeminfo(const std::string& meminfo_data,
                       SystemMemoryInfoKB* meminfo) {
-  std::vector<std::string> meminfo_fields;
-  SplitStringAlongWhitespace(meminfo_data, &meminfo_fields);
-
-  if (meminfo_fields.size() < kMemCachedIndex) {
-    return false;
-  }
-
-  DCHECK_EQ(meminfo_fields[kMemTotalIndex-1], "MemTotal:");
-  DCHECK_EQ(meminfo_fields[kMemFreeIndex-1], "MemFree:");
-  DCHECK_EQ(meminfo_fields[kMemBuffersIndex-1], "Buffers:");
-  DCHECK_EQ(meminfo_fields[kMemCachedIndex-1], "Cached:");
-  DCHECK_EQ(meminfo_fields[kMemActiveAnonIndex-1], "Active(anon):");
-  DCHECK_EQ(meminfo_fields[kMemInactiveAnonIndex-1], "Inactive(anon):");
-  DCHECK_EQ(meminfo_fields[kMemActiveFileIndex-1], "Active(file):");
-  DCHECK_EQ(meminfo_fields[kMemInactiveFileIndex-1], "Inactive(file):");
-
-  StringToInt(meminfo_fields[kMemTotalIndex], &meminfo->total);
-  StringToInt(meminfo_fields[kMemFreeIndex], &meminfo->free);
-  StringToInt(meminfo_fields[kMemBuffersIndex], &meminfo->buffers);
-  StringToInt(meminfo_fields[kMemCachedIndex], &meminfo->cached);
-  StringToInt(meminfo_fields[kMemActiveAnonIndex], &meminfo->active_anon);
-  StringToInt(meminfo_fields[kMemInactiveAnonIndex], &meminfo->inactive_anon);
-  StringToInt(meminfo_fields[kMemActiveFileIndex], &meminfo->active_file);
-  StringToInt(meminfo_fields[kMemInactiveFileIndex], &meminfo->inactive_file);
-
-  // We don't know when these fields appear, so we must search for them.
-  for (size_t i = kMemCachedIndex+2; i < meminfo_fields.size(); i += 3) {
-    if (meminfo_fields[i] == "SwapTotal:")
-      StringToInt(meminfo_fields[i+1], &meminfo->swap_total);
-    if (meminfo_fields[i] == "SwapFree:")
-      StringToInt(meminfo_fields[i+1], &meminfo->swap_free);
-    if (meminfo_fields[i] == "Dirty:")
-      StringToInt(meminfo_fields[i+1], &meminfo->dirty);
-  }
-
+  // The format of /proc/meminfo is:
+  //
+  // MemTotal:      8235324 kB
+  // MemFree:       1628304 kB
+  // Buffers:        429596 kB
+  // Cached:        4728232 kB
+  // ...
+  // There is no guarantee on the ordering or position
+  // though it doesn't appear to change very often
+
+  // As a basic sanity check, let's make sure we at least get non-zero
+  // MemTotal value
+  meminfo->total = 0;
+
+  std::vector<std::string> meminfo_lines;
+  Tokenize(meminfo_data, "\n", &meminfo_lines);
+  for (std::vector<std::string>::iterator it = meminfo_lines.begin();
+       it != meminfo_lines.end(); ++it) {
+    std::vector<std::string> tokens;
+    SplitStringAlongWhitespace(*it, &tokens);
+    // HugePages_* only has a number and no suffix so we can't rely on
+    // there being exactly 3 tokens.
+    if (tokens.size() > 1) {
+      if (tokens[0] == "MemTotal:") {
+        StringToInt(tokens[1], &meminfo->total);
+        continue;
+      } if (tokens[0] == "MemFree:") {
+        StringToInt(tokens[1], &meminfo->free);
+        continue;
+      } if (tokens[0] == "Buffers:") {
+        StringToInt(tokens[1], &meminfo->buffers);
+        continue;
+      } if (tokens[0] == "Cached:") {
+        StringToInt(tokens[1], &meminfo->cached);
+        continue;
+      } if (tokens[0] == "Active(anon):") {
+        StringToInt(tokens[1], &meminfo->active_anon);
+        continue;
+      } if (tokens[0] == "Inactive(anon):") {
+        StringToInt(tokens[1], &meminfo->inactive_anon);
+        continue;
+      } if (tokens[0] == "Active(file):") {
+        StringToInt(tokens[1], &meminfo->active_file);
+        continue;
+      } if (tokens[0] == "Inactive(file):") {
+        StringToInt(tokens[1], &meminfo->inactive_file);
+        continue;
+      } if (tokens[0] == "SwapTotal:") {
+        StringToInt(tokens[1], &meminfo->swap_total);
+        continue;
+      } if (tokens[0] == "SwapFree:") {
+        StringToInt(tokens[1], &meminfo->swap_free);
+        continue;
+      } if (tokens[0] == "Dirty:") {
+        StringToInt(tokens[1], &meminfo->dirty);
+        continue;
 #if defined(OS_CHROMEOS)
-  // Chrome OS has a tweaked kernel that allows us to query Shmem, which is
-  // usually video memory otherwise invisible to the OS.  Unfortunately, the
-  // meminfo format varies on different hardware so we have to search for the
-  // string.  It always appears after "Cached:".
-  for (size_t i = kMemCachedIndex+2; i < meminfo_fields.size(); i += 3) {
-    if (meminfo_fields[i] == "Shmem:")
-      StringToInt(meminfo_fields[i+1], &meminfo->shmem);
-    if (meminfo_fields[i] == "Slab:")
-      StringToInt(meminfo_fields[i+1], &meminfo->slab);
-  }
+      // Chrome OS has a tweaked kernel that allows us to query Shmem, which is
+      // usually video memory otherwise invisible to the OS.
+      } if (tokens[0] == "Shmem:") {
+        StringToInt(tokens[1], &meminfo->shmem);
+        continue;
+      } if (tokens[0] == "Slab:") {
+        StringToInt(tokens[1], &meminfo->slab);
+        continue;
 #endif
+      }
+    } else
+      DLOG(WARNING) << "meminfo: tokens: " << tokens.size()
+                    << " malformed line: " << *it;
+  }
+
+  // Make sure we got a valid MemTotal.
+  if (!meminfo->total)
+    return false;
 
   return true;
 }
@@ -595,14 +596,34 @@ bool ParseProcMeminfo(const std::string& meminfo_data,
 // exposed for testing
 bool ParseProcVmstat(const std::string& vmstat_data,
                      SystemMemoryInfoKB* meminfo) {
-  std::vector<std::string> vmstat_fields;
-  SplitStringAlongWhitespace(vmstat_data, &vmstat_fields);
-  if (vmstat_fields[kVMPagesSwappedIn-1] == "pswpin")
-    StringToInt(vmstat_fields[kVMPagesSwappedIn], &meminfo->pswpin);
-  if (vmstat_fields[kVMPagesSwappedOut-1] == "pswpout")
-    StringToInt(vmstat_fields[kVMPagesSwappedOut], &meminfo->pswpout);
-  if (vmstat_fields[kVMPageMajorFaults-1] == "pgmajfault")
-    StringToInt(vmstat_fields[kVMPageMajorFaults], &meminfo->pgmajfault);
+  // The format of /proc/vmstat is:
+  //
+  // nr_free_pages 299878
+  // nr_inactive_anon 239863
+  // nr_active_anon 1318966
+  // nr_inactive_file 2015629
+  // ...
+  //
+  // We iterate through the whole file because the position of the
+  // fields are dependent on the kernel version and configuration.
+
+  std::vector<std::string> vmstat_lines;
+  Tokenize(vmstat_data, "\n", &vmstat_lines);
+  for (std::vector<std::string>::iterator it = vmstat_lines.begin();
+       it != vmstat_lines.end(); ++it) {
+    std::vector<std::string> tokens;
+    SplitString(*it, ' ', &tokens);
+    if (tokens.size() == 2) {
+      if (tokens[0] == "pswpin") {
+        StringToInt(tokens[1], &meminfo->pswpin);
+        continue;
+      } if (tokens[0] == "pswpout") {
+        StringToInt(tokens[1], &meminfo->pswpout);
+        continue;
+      } if (tokens[0] == "pgmajfault")
+        StringToInt(tokens[1], &meminfo->pgmajfault);
+    }
+  }
 
   return true;
 }
diff --git a/base/process/process_metrics_unittests.cc b/base/process/process_metrics_unittests.cc
index 387d860..5014557 100644
--- a/base/process/process_metrics_unittests.cc
+++ b/base/process/process_metrics_unittests.cc
@@ -57,6 +57,216 @@ TEST_F(SystemMetricsTest, IsValidDiskName) {
   EXPECT_TRUE(IsValidDiskName(valid_input4));
   EXPECT_TRUE(IsValidDiskName(valid_input5));
 }
+
+TEST_F(SystemMetricsTest, ParseMeminfo) {
+  struct SystemMemoryInfoKB meminfo;
+  std::string invalid_input1 = "abc";
+  std::string invalid_input2 = "MemTotal:";
+  // Partial file with no MemTotal
+  std::string invalid_input3 =
+    "MemFree:         3913968 kB\n"
+    "Buffers:         2348340 kB\n"
+    "Cached:         49071596 kB\n"
+    "SwapCached:           12 kB\n"
+    "Active:         36393900 kB\n"
+    "Inactive:       21221496 kB\n"
+    "Active(anon):    5674352 kB\n"
+    "Inactive(anon):   633992 kB\n";
+  EXPECT_FALSE(ParseProcMeminfo(invalid_input1, &meminfo));
+  EXPECT_FALSE(ParseProcMeminfo(invalid_input2, &meminfo));
+  EXPECT_FALSE(ParseProcMeminfo(invalid_input3, &meminfo));
+
+  std::string valid_input1 =
+    "MemTotal:        3981504 kB\n"
+    "MemFree:          140764 kB\n"
+    "Buffers:          116480 kB\n"
+    "Cached:           406160 kB\n"
+    "SwapCached:        21304 kB\n"
+    "Active:          3152040 kB\n"
+    "Inactive:         472856 kB\n"
+    "Active(anon):    2972352 kB\n"
+    "Inactive(anon):   270108 kB\n"
+    "Active(file):     179688 kB\n"
+    "Inactive(file):   202748 kB\n"
+    "Unevictable:           0 kB\n"
+    "Mlocked:               0 kB\n"
+    "SwapTotal:       5832280 kB\n"
+    "SwapFree:        3672368 kB\n"
+    "Dirty:               184 kB\n"
+    "Writeback:             0 kB\n"
+    "AnonPages:       3101224 kB\n"
+    "Mapped:           142296 kB\n"
+    "Shmem:            140204 kB\n"
+    "Slab:              54212 kB\n"
+    "SReclaimable:      30936 kB\n"
+    "SUnreclaim:        23276 kB\n"
+    "KernelStack:        2464 kB\n"
+    "PageTables:        24812 kB\n"
+    "NFS_Unstable:          0 kB\n"
+    "Bounce:                0 kB\n"
+    "WritebackTmp:          0 kB\n"
+    "CommitLimit:     7823032 kB\n"
+    "Committed_AS:    7973536 kB\n"
+    "VmallocTotal:   34359738367 kB\n"
+    "VmallocUsed:      375940 kB\n"
+    "VmallocChunk:   34359361127 kB\n"
+    "DirectMap4k:       72448 kB\n"
+    "DirectMap2M:     4061184 kB\n";
+  // output from a much older kernel where the Active and Inactive aren't
+  // broken down into anon and file and Huge Pages are enabled
+  std::string valid_input2 =
+    "MemTotal:       255908 kB\n"
+    "MemFree:         69936 kB\n"
+    "Buffers:         15812 kB\n"
+    "Cached:         115124 kB\n"
+    "SwapCached:          0 kB\n"
+    "Active:          92700 kB\n"
+    "Inactive:        63792 kB\n"
+    "HighTotal:           0 kB\n"
+    "HighFree:            0 kB\n"
+    "LowTotal:       255908 kB\n"
+    "LowFree:         69936 kB\n"
+    "SwapTotal:      524280 kB\n"
+    "SwapFree:       524200 kB\n"
+    "Dirty:               4 kB\n"
+    "Writeback:           0 kB\n"
+    "Mapped:          42236 kB\n"
+    "Slab:            25912 kB\n"
+    "Committed_AS:   118680 kB\n"
+    "PageTables:       1236 kB\n"
+    "VmallocTotal:  3874808 kB\n"
+    "VmallocUsed:      1416 kB\n"
+    "VmallocChunk:  3872908 kB\n"
+    "HugePages_Total:     0\n"
+    "HugePages_Free:      0\n"
+    "Hugepagesize:     4096 kB\n";
+
+  EXPECT_TRUE(ParseProcMeminfo(valid_input1, &meminfo));
+  EXPECT_TRUE(meminfo.total == 3981504);
+  EXPECT_TRUE(meminfo.free == 140764);
+  EXPECT_TRUE(meminfo.buffers == 116480);
+  EXPECT_TRUE(meminfo.cached == 406160);
+  EXPECT_TRUE(meminfo.active_anon == 2972352);
+  EXPECT_TRUE(meminfo.active_file == 179688);
+  EXPECT_TRUE(meminfo.inactive_anon == 270108);
+  EXPECT_TRUE(meminfo.inactive_file == 202748);
+  EXPECT_TRUE(meminfo.swap_total == 5832280);
+  EXPECT_TRUE(meminfo.swap_free == 3672368);
+  EXPECT_TRUE(meminfo.dirty == 184);
+#if defined(OS_CHROMEOS)
+  EXPECT_TRUE(meminfo.shmem == 140204);
+  EXPECT_TRUE(meminfo.slab == 54212);
+#endif
+  EXPECT_TRUE(ParseProcMeminfo(valid_input2, &meminfo));
+  EXPECT_TRUE(meminfo.total == 255908);
+  EXPECT_TRUE(meminfo.free == 69936);
+  EXPECT_TRUE(meminfo.buffers == 15812);
+  EXPECT_TRUE(meminfo.cached == 115124);
+  EXPECT_TRUE(meminfo.swap_total == 524280);
+  EXPECT_TRUE(meminfo.swap_free == 524200);
+  EXPECT_TRUE(meminfo.dirty == 4);
+}
+
+TEST_F(SystemMetricsTest, ParseVmstat) {
+  struct SystemMemoryInfoKB meminfo;
+  // part of vmstat from a 3.2 kernel with numa enabled
+  std::string valid_input1 =
+    "nr_free_pages 905104\n"
+    "nr_inactive_anon 142478"
+    "nr_active_anon 1520046\n"
+    "nr_inactive_file 4481001\n"
+    "nr_active_file 8313439\n"
+    "nr_unevictable 5044\n"
+    "nr_mlock 5044\n"
+    "nr_anon_pages 1633780\n"
+    "nr_mapped 104742\n"
+    "nr_file_pages 12828218\n"
+    "nr_dirty 245\n"
+    "nr_writeback 0\n"
+    "nr_slab_reclaimable 831609\n"
+    "nr_slab_unreclaimable 41164\n"
+    "nr_page_table_pages 31470\n"
+    "nr_kernel_stack 1735\n"
+    "nr_unstable 0\n"
+    "nr_bounce 0\n"
+    "nr_vmscan_write 406\n"
+    "nr_vmscan_immediate_reclaim 281\n"
+    "nr_writeback_temp 0\n"
+    "nr_isolated_anon 0\n"
+    "nr_isolated_file 0\n"
+    "nr_shmem 28820\n"
+    "nr_dirtied 84674644\n"
+    "nr_written 75307109\n"
+    "nr_anon_transparent_hugepages 0\n"
+    "nr_dirty_threshold 1536206\n"
+    "nr_dirty_background_threshold 768103\n"
+    "pgpgin 30777108\n"
+    "pgpgout 319023278\n"
+    "pswpin 179\n"
+    "pswpout 406\n"
+    "pgalloc_dma 0\n"
+    "pgalloc_dma32 20833399\n"
+    "pgalloc_normal 1622609290\n"
+    "pgalloc_movable 0\n"
+    "pgfree 1644355583\n"
+    "pgactivate 75391882\n"
+    "pgdeactivate 4121019\n"
+    "pgfault 2542879679\n"
+    "pgmajfault 487192\n";
+  std::string valid_input2 =
+    "nr_free_pages 180125\n"
+    "nr_inactive_anon 51\n"
+    "nr_active_anon 38832\n"
+    "nr_inactive_file 50171\n"
+    "nr_active_file 47510\n"
+    "nr_unevictable 0\n"
+    "nr_mlock 0\n"
+    "nr_anon_pages 38825\n"
+    "nr_mapped 24043\n"
+    "nr_file_pages 97733\n"
+    "nr_dirty 0\n"
+    "nr_writeback 0\n"
+    "nr_slab_reclaimable 4032\n"
+    "nr_slab_unreclaimable 2848\n"
+    "nr_page_table_pages 1505\n"
+    "nr_kernel_stack 626\n"
+    "nr_unstable 0\n"
+    "nr_bounce 0\n"
+    "nr_vmscan_write 0\n"
+    "nr_vmscan_immediate_reclaim 0\n"
+    "nr_writeback_temp 0\n"
+    "nr_isolated_anon 0\n"
+    "nr_isolated_file 0\n"
+    "nr_shmem 58\n"
+    "nr_dirtied 435358\n"
+    "nr_written 401258\n"
+    "nr_anon_transparent_hugepages 0\n"
+    "nr_dirty_threshold 18566\n"
+    "nr_dirty_background_threshold 4641\n"
+    "pgpgin 299464\n"
+    "pgpgout 2437788\n"
+    "pswpin 12\n"
+    "pswpout 901\n"
+    "pgalloc_normal 144213030\n"
+    "pgalloc_high 164501274\n"
+    "pgalloc_movable 0\n"
+    "pgfree 308894908\n"
+    "pgactivate 239320\n"
+    "pgdeactivate 1\n"
+    "pgfault 716044601\n"
+    "pgmajfault 2023\n"
+    "pgrefill_normal 0\n"
+    "pgrefill_high 0\n"
+    "pgrefill_movable 0\n";
+  EXPECT_TRUE(ParseProcVmstat(valid_input1, &meminfo));
+  EXPECT_TRUE(meminfo.pswpin == 179);
+  EXPECT_TRUE(meminfo.pswpout == 406);
+  EXPECT_TRUE(meminfo.pgmajfault == 487192);
+  EXPECT_TRUE(ParseProcVmstat(valid_input2, &meminfo));
+  EXPECT_TRUE(meminfo.pswpin == 12);
+  EXPECT_TRUE(meminfo.pswpout == 901);
+  EXPECT_TRUE(meminfo.pgmajfault == 2023);
+}
 #endif  // defined(OS_LINUX) || defined(OS_ANDROID)
 
 }  // namespace debug