Add support for RIP relative addresses on x86_64.

Visual studio uses rip-relative addressing (rra) extensively in 64 bit binaries. ParseRel32RelocsFromSection does not find these addresses, which causes courgette to miss a lot of rva pointers, and thus missing a lot of compresssion opportunities.

This patch makes the ParseRel32RelocsFromSection find rip relative call/jmp/lea. It also finds mov's that load from memory using rra's.

Based on CL by niels.werensteijn.teamspeak@gmail.com in crrev.com/212563003

This change gives a noticeable improvement on 64-bit binaries. Against test binaries (64-bit chrome.dll 40.0.2214.115->43.0.2317.0) the patch sizes were:

Uncompressed:
before: 10,948,152
after: 9,948,442 (9.1% reduction)

Compressed (7z ultra):
before: 6,084,670
after: 5,581,502 (8.3% reduction)

BUG=459064
TEST=courgette_unittests

Review URL: https://codereview.chromium.org/878043002

Cr-Commit-Position: refs/heads/master@{#321524}

2 files changed, 24 insertions, 5 deletions

diff --git a/courgette/disassembler_win32_x64.cc b/courgette/disassembler_win32_x64.cc
index 73ad0b4..667b4e1 100644
--- a/courgette/disassembler_win32_x64.cc
+++ b/courgette/disassembler_win32_x64.cc
@@ -462,18 +462,36 @@ void DisassemblerWin32X64::ParseRel32RelocsFromSection(const Section* section) {
     // next few bytes the start of an instruction containing a rel32
     // addressing mode?
     const uint8* rel32 = NULL;
+    bool is_rip_relative = false;
 
     if (p + 5 <= end_pointer) {
-      if (*p == 0xE8 || *p == 0xE9) {  // jmp rel32 and call rel32
+      if (*p == 0xE8 || *p == 0xE9)  // jmp rel32 and call rel32
         rel32 = p + 1;
-      }
     }
     if (p + 6 <= end_pointer) {
-      if (*p == 0x0F  &&  (*(p+1) & 0xF0) == 0x80) {  // Jcc long form
+      if (*p == 0x0F && (*(p + 1) & 0xF0) == 0x80) {  // Jcc long form
         if (p[1] != 0x8A && p[1] != 0x8B)  // JPE/JPO unlikely
           rel32 = p + 2;
+      } else if (*p == 0xFF && (*(p + 1) == 0x15 || *(p + 1) == 0x25)) {
+        // rip relative call/jmp
+        rel32 = p + 2;
+        is_rip_relative = true;
+      }
+    }
+    if (p + 7 <= end_pointer) {
+      if ((*p & 0xFB) == 0x48 && *(p + 1) == 0x8D &&
+          (*(p + 2) & 0xC7) == 0x05) {
+        // rip relative lea
+        rel32 = p + 3;
+        is_rip_relative = true;
+      } else if ((*p & 0xFB) == 0x48 && *(p + 1) == 0x8B &&
+                 (*(p + 2) & 0xC7) == 0x05) {
+        // rip relative mov
+        rel32 = p + 3;
+        is_rip_relative = true;
       }
     }
+
     if (rel32) {
       RVA rel32_rva = static_cast<RVA>(rel32 - adjust_pointer_to_rva);
 
@@ -495,7 +513,8 @@ void DisassemblerWin32X64::ParseRel32RelocsFromSection(const Section* section) {
       // To be valid, rel32 target must be within image, and within this
       // section.
       if (IsValidRVA(target_rva) &&
-          start_rva <= target_rva && target_rva < end_rva) {
+          (is_rip_relative ||
+           (start_rva <= target_rva && target_rva < end_rva))) {
         rel32_locations_.push_back(rel32_rva);
 #if COURGETTE_HISTOGRAM_TARGETS
         ++rel32_target_rvas_[target_rva];
diff --git a/courgette/encode_decode_unittest.cc b/courgette/encode_decode_unittest.cc
index 4240c5c..79cc822 100644
--- a/courgette/encode_decode_unittest.cc
+++ b/courgette/encode_decode_unittest.cc
@@ -75,7 +75,7 @@ TEST_F(EncodeDecodeTest, PE) {
 
 TEST_F(EncodeDecodeTest, PE64) {
   std::string file = FileContents("chrome64_1.exe");
-  TestAssembleToStreamDisassemble(file, 803782);
+  TestAssembleToStreamDisassemble(file, 808845);
 }
 
 TEST_F(EncodeDecodeTest, Elf_Small) {