Add dirty memory accounting to tools/android/memdump.

Using /proc/kpageflags to account for dirty/unevictable pages.

R=digit@chromium.org, pliard@google.com

Review URL: https://codereview.chromium.org/18386005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@210201 0039d316-1c4b-4281-b951-d872f2087c98

2 files changed, 90 insertions, 22 deletions

diff --git a/tools/android/memdump/memdump.cc b/tools/android/memdump/memdump.cc
index e4f20d1..21ef291 100644
--- a/tools/android/memdump/memdump.cc
+++ b/tools/android/memdump/memdump.cc
@@ -11,6 +11,7 @@
 #include <cstring>
 #include <fstream>
 #include <iostream>
+#include <limits>
 #include <string>
 #include <utility>
 #include <vector>
@@ -36,10 +37,11 @@ struct PageMapEntry {
   uint present : 1;
 };
 
-// Describes a virtual page.
+// Describes a memory page.
 struct PageInfo {
-  int64 page_frame_number : 55;  // Physical page id, also known as PFN.
-  int64 times_mapped : 9 + 32;
+  int64 page_frame_number; // Physical page id, also known as PFN.
+  int64 flags;
+  int32 times_mapped;
 };
 
 struct MemoryMap {
@@ -48,7 +50,9 @@ struct MemoryMap {
   uint start_address;
   uint end_address;
   int private_count;
+  int unevictable_private_count;
   int other_shared_count;
+  int unevictable_other_shared_count;
   // app_shared_counts[i] contains the number of pages mapped in i+2 processes
   // (only among the processes that are being analyzed).
   std::vector<int> app_shared_counts;
@@ -60,6 +64,20 @@ struct ProcessMemory {
   std::vector<MemoryMap> memory_maps;
 };
 
+bool PageIsUnevictable(const PageInfo& page_info) {
+  // These constants are taken from kernel-page-flags.h.
+  const int KPF_DIRTY = 4; // Note that only file-mapped pages can be DIRTY.
+  const int KPF_ANON = 12; // Anonymous pages are dirty per definition.
+  const int KPF_UNEVICTABLE = 18;
+  const int KPF_MLOCKED = 33;
+
+  return (page_info.flags & ((1ll << KPF_DIRTY) |
+                             (1ll << KPF_ANON) |
+                             (1ll << KPF_UNEVICTABLE) |
+                             (1ll << KPF_MLOCKED))) ?
+                             true : false;
+}
+
 // Number of times a physical page is mapped in a process.
 typedef base::hash_map<uint64, int> PFNMap;
 
@@ -169,16 +187,29 @@ bool GetPagesForMemoryMap(int pagemap_fd,
   return true;
 }
 
-bool SetTimesMapped(int pagecount_fd, std::vector<PageInfo>* pages) {
+// Fills |committed_pages| with mapping count and flags information gathered
+// looking-up /proc/kpagecount and /proc/kpageflags.
+bool SetPagesInfo(int pagecount_fd,
+                  int pageflags_fd,
+                  std::vector<PageInfo>* pages) {
   for (std::vector<PageInfo>::iterator it = pages->begin();
        it != pages->end(); ++it) {
     PageInfo* const page_info = &*it;
+
     int64 times_mapped;
     if (!ReadFromFileAtOffset(
             pagecount_fd, page_info->page_frame_number, &times_mapped)) {
       return false;
     }
-    page_info->times_mapped = times_mapped;
+    DCHECK(times_mapped <= std::numeric_limits<int32_t>::max());
+    page_info->times_mapped = static_cast<int32>(times_mapped);
+
+    int64 page_flags;
+    if (!ReadFromFileAtOffset(
+            pageflags_fd, page_info->page_frame_number, &page_flags)) {
+      return false;
+    }
+    page_info->flags = page_flags;
   }
   return true;
 }
@@ -230,6 +261,8 @@ void ClassifyPages(std::vector<ProcessMemory>* processes_memory) {
         const PageInfo& page_info = *it;
         if (page_info.times_mapped == 1) {
           ++memory_map->private_count;
+          if (PageIsUnevictable(page_info))
+            ++memory_map->unevictable_private_count;
           continue;
         }
         const uint64 page_frame_number = page_info.page_frame_number;
@@ -264,9 +297,14 @@ void ClassifyPages(std::vector<ProcessMemory>* processes_memory) {
           } else {
             // The physical page is mapped multiple times in the same process.
             ++memory_map->private_count;
+            if (PageIsUnevictable(page_info))
+              ++memory_map->unevictable_private_count;
           }
         } else {
           ++memory_map->other_shared_count;
+          if (PageIsUnevictable(page_info))
+            ++memory_map->unevictable_other_shared_count;
+
         }
       }
     }
@@ -300,11 +338,15 @@ void DumpProcessesMemoryMaps(
       app_shared_buf.clear();
       AppendAppSharedField(memory_map.app_shared_counts, &app_shared_buf);
       base::SStringPrintf(
-          &buf, "%x-%x %s private=%d shared_app=%s shared_other=%d %s\n",
+          &buf,
+          "%x-%x %s private_unevictable=%d private=%d shared_app=%s "
+          "shared_other_unevictable=%d shared_other=%d %s\n",
           memory_map.start_address,
           memory_map.end_address, memory_map.flags.c_str(),
+          memory_map.unevictable_private_count * PAGE_SIZE,
           memory_map.private_count * PAGE_SIZE,
           app_shared_buf.c_str(),
+          memory_map.unevictable_other_shared_count * PAGE_SIZE,
           memory_map.other_shared_count * PAGE_SIZE,
           memory_map.name.c_str());
       std::cout << buf;
@@ -346,6 +388,7 @@ void DumpProcessesMemoryMapsInShortFormat(
 }
 
 bool CollectProcessMemoryInformation(int page_count_fd,
+                                     int page_flags_fd,
                                      ProcessMemory* process_memory) {
   const pid_t pid = process_memory->pid;
   int pagemap_fd = open(
@@ -362,7 +405,7 @@ bool CollectProcessMemoryInformation(int page_count_fd,
        it != process_maps->end(); ++it) {
     std::vector<PageInfo>* const committed_pages = &it->committed_pages;
     GetPagesForMemoryMap(pagemap_fd, *it, committed_pages);
-    SetTimesMapped(page_count_fd, committed_pages);
+    SetPagesInfo(page_count_fd, page_flags_fd, committed_pages);
   }
   return true;
 }
@@ -402,10 +445,19 @@ int main(int argc, char** argv) {
   {
     int page_count_fd = open("/proc/kpagecount", O_RDONLY);
     if (page_count_fd < 0) {
-      PLOG(ERROR) << "open";
+      PLOG(ERROR) << "open /proc/kpagecount";
+      return EXIT_FAILURE;
+    }
+
+    int page_flags_fd = open("/proc/kpageflags", O_RDONLY);
+    if (page_flags_fd < 0) {
+      PLOG(ERROR) << "open /proc/kpageflags";
       return EXIT_FAILURE;
     }
+
     file_util::ScopedFD page_count_fd_closer(&page_count_fd);
+    file_util::ScopedFD page_flags_fd_closer(&page_flags_fd);
+
     base::ScopedClosureRunner auto_resume_processes(
         base::Bind(&KillAll, pids, SIGCONT));
     KillAll(pids, SIGSTOP);
@@ -414,10 +466,13 @@ int main(int argc, char** argv) {
       ProcessMemory* const process_memory =
           &processes_memory[it - pids.begin()];
       process_memory->pid = *it;
-      if (!CollectProcessMemoryInformation(page_count_fd, process_memory))
+      if (!CollectProcessMemoryInformation(page_count_fd,
+                                           page_flags_fd,
+                                           process_memory))
         return EXIT_FAILURE;
     }
   }
+
   ClassifyPages(&processes_memory);
   if (short_output)
     DumpProcessesMemoryMapsInShortFormat(processes_memory);
diff --git a/tools/android/memdump/memreport.py b/tools/android/memdump/memreport.py
index 02e080d..1db1cb6 100755
--- a/tools/android/memdump/memreport.py
+++ b/tools/android/memdump/memreport.py
@@ -13,12 +13,14 @@ from sets import Set
 _ENTRIES = [
     ('Total', '.* r... .*'),
     ('Read-only', '.* r--. .*'),
-    ('Read-write', '.* rw-. .*'),
+    ('Read-write', '.* rw.. .*'),
     ('Executable', '.* ..x. .*'),
-    ('File read-write', '.* rw.. .* /.*'),
+    ('Anonymous total', '.* .... .* .*other=[0-9]+ ($|.*chromium:.*)'),
     ('Anonymous read-write', '.* rw.. .* .*other=[0-9]+ ($|.*chromium:.*)'),
-    ('File executable', '.* ..x. .* /.*'),
     ('Anonymous executable (JIT\'ed code)', '.* ..x. .* shared_other=[0-9]+ $'),
+    ('File total', '.* .... .* /.*'),
+    ('File read-write', '.* rw.. .* /.*'),
+    ('File executable', '.* ..x. .* /.*'),
     ('chromium mmap', '.* r... .*chromium:.*'),
     ('chromium TransferBuffer', '.* r... .*chromium:.*CreateTransferBuffer.*'),
     ('Galaxy Nexus GL driver', '.* r... .*pvrsrvkm.*'),
@@ -51,15 +53,17 @@ def _CollectMemoryStats(memdump, region_filters):
         matched_regions.add(region_filter)
         if not region_filter in mem_usage_for_regions:
           mem_usage_for_regions[region_filter] = {
+              'private_unevictable': 0,
               'private': 0,
               'shared_app': 0.0,
+              'shared_other_unevictable': 0,
               'shared_other': 0,
           }
     for matched_region in matched_regions:
       mem_usage = mem_usage_for_regions[matched_region]
       for key in mem_usage:
         for token in line.split(' '):
-          if key in token:
+          if (key + '=') in token:
             field = token.split('=')[1]
             if key != 'shared_app':
               mem_usage[key] += int(field)
@@ -80,24 +84,33 @@ def _DumpCSV(processes_stats):
   i = 0
   for process in processes_stats:
     i += 1
-    print ',Process ' + str(i) + ',private,shared_app,shared_other,'
+    print (',Process ' + str(i) + ',private,private_unevictable,shared_app,' +
+           'shared_other,shared_other_unevictable,')
     for (k, v) in _ENTRIES:
       if not v in process:
-        print ',' + k + ',0,0,0,'
+        print ',' + k + ',0,0,0,0,'
         continue
       if not v in total_map:
-        total_map[v] = 0
-      total_map[v] += process[v]['private'] + process[v]['shared_app']
-      print ',' + k + ',' + _ConvertMemoryField(process[v]['private']) + ',' + (
-          _ConvertMemoryField(process[v]['shared_app']) + ',' + (
-          _ConvertMemoryField(process[v]['shared_other'])) + ',')
+        total_map[v] = {'resident':0, 'unevictable':0}
+      total_map[v]['resident'] += (process[v]['private'] +
+                                   process[v]['shared_app'])
+      total_map[v]['unevictable'] += process[v]['private_unevictable']
+      print (
+          ',' + k + ',' +
+          _ConvertMemoryField(process[v]['private']) + ',' +
+          _ConvertMemoryField(process[v]['private_unevictable']) + ',' +
+          _ConvertMemoryField(process[v]['shared_app']) + ',' +
+          _ConvertMemoryField(process[v]['shared_other']) + ',' +
+          _ConvertMemoryField(process[v]['shared_other_unevictable']) + ','
+          )
     print ''
 
   for (k, v) in _ENTRIES:
     if not v in total_map:
-      print ',' + k + ',0,0'
+      print ',' + k + ',0,0,'
       continue
-    print ',' + k + ',' + _ConvertMemoryField(total_map[v]) + ','
+    print (',' + k + ',' + _ConvertMemoryField(total_map[v]['resident']) + ',' +
+           _ConvertMemoryField(total_map[v]['unevictable']) + ',')
   print ''