Revert "Revert "ART: Key-Value Store in Oat header""

This reverts commit 452bee5da9811f62123978e142bd67b385e9ff82.

Heap-allocate a couple of objects in dex2oat to avoid large frame
size.

Includes fixes originally in 100596 and 100605.

Change-Id: Id51a44198c973c91f0a3f87b9d992a5dc110c6f8

7 files changed, 391 insertions, 44 deletions

diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 5180e34..860cbd2 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -1159,7 +1159,9 @@ void ClassLinker::InitFromImage() {
   OatFile& oat_file = GetImageOatFile(space);
   CHECK_EQ(oat_file.GetOatHeader().GetImageFileLocationOatChecksum(), 0U);
   CHECK_EQ(oat_file.GetOatHeader().GetImageFileLocationOatDataBegin(), 0U);
-  CHECK(oat_file.GetOatHeader().GetImageFileLocation().empty());
+  const char* image_file_location = oat_file.GetOatHeader().
+      GetStoreValueByKey(OatHeader::kImageLocationKey);
+  CHECK(image_file_location == nullptr || *image_file_location == 0);
   portable_resolution_trampoline_ = oat_file.GetOatHeader().GetPortableResolutionTrampoline();
   quick_resolution_trampoline_ = oat_file.GetOatHeader().GetQuickResolutionTrampoline();
   portable_imt_conflict_trampoline_ = oat_file.GetOatHeader().GetPortableImtConflictTrampoline();
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index d534bcb..a87aa89 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -341,6 +341,10 @@ bool ImageSpace::ValidateOatFile(std::string* error_msg) const {
   return true;
 }
 
+const OatFile* ImageSpace::GetOatFile() const {
+  return oat_file_.get();
+}
+
 OatFile* ImageSpace::ReleaseOatFile() {
   CHECK(oat_file_.get() != NULL);
   return oat_file_.release();
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index 372db3a..dd9b580 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -51,6 +51,9 @@ class ImageSpace : public MemMapSpace {
   static ImageHeader* ReadImageHeaderOrDie(const char* image_location,
                                            InstructionSet image_isa);
 
+  // Give access to the OatFile.
+  const OatFile* GetOatFile() const;
+
   // Releases the OatFile from the ImageSpace so it can be transfer to
   // the caller, presumably the ClassLinker.
   OatFile* ReleaseOatFile()
diff --git a/runtime/implicit_check_options.h b/runtime/implicit_check_options.h
new file mode 100644
index 0000000..a6595b8
--- /dev/null
+++ b/runtime/implicit_check_options.h
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_RUNTIME_IMPLICIT_CHECK_OPTIONS_H_
+#define ART_RUNTIME_IMPLICIT_CHECK_OPTIONS_H_
+
+#include "gc/heap.h"
+#include "gc/space/image_space.h"
+#include "instruction_set.h"
+#include "runtime.h"
+
+#include <string>
+
+namespace art {
+
+class ImplicitCheckOptions {
+ public:
+  static constexpr const char* kImplicitChecksOatHeaderKey = "implicit-checks";
+
+  static std::string Serialize(bool explicit_null_checks, bool explicit_stack_overflow_checks,
+                               bool explicit_suspend_checks) {
+    char tmp[4];
+    tmp[0] = explicit_null_checks ? 'N' : 'n';
+    tmp[1] = explicit_stack_overflow_checks ? 'O' : 'o';
+    tmp[2] = explicit_suspend_checks ? 'S' : 's';
+    tmp[3] = 0;
+    return std::string(tmp);
+  }
+
+  static bool Parse(const char* str, bool* explicit_null_checks,
+                    bool* explicit_stack_overflow_checks, bool* explicit_suspend_checks) {
+    if (str != nullptr && str[0] != 0 && str[1] != 0 && str[2] != 0 &&
+        (str[0] == 'n' || str[0] == 'N') &&
+        (str[1] == 'o' || str[1] == 'O') &&
+        (str[2] == 's' || str[2] == 'S')) {
+      *explicit_null_checks = str[0] == 'N';
+      *explicit_stack_overflow_checks = str[1] == 'O';
+      *explicit_suspend_checks = str[2] == 'S';
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  // Check whether the given flags are correct with respect to the current runtime and the given
+  // executable flag.
+  static bool CheckRuntimeSupport(bool executable, bool explicit_null_checks,
+                                  bool explicit_stack_overflow_checks,
+                                  bool explicit_suspend_checks, std::string* error_msg) {
+    if (!executable) {
+      // Not meant to be run, i.e., either we are compiling or dumping. Just accept.
+      return true;
+    }
+
+    Runtime* runtime = Runtime::Current();
+    // We really should have a runtime.
+    DCHECK_NE(static_cast<Runtime*>(nullptr), runtime);
+
+    if (runtime->GetInstrumentation()->IsForcedInterpretOnly()) {
+      // We are an interpret-only environment. Ignore the check value.
+      return true;
+    }
+
+    if (runtime->ExplicitNullChecks() != explicit_null_checks ||
+        runtime->ExplicitStackOverflowChecks() != explicit_stack_overflow_checks ||
+        runtime->ExplicitSuspendChecks() != explicit_suspend_checks) {
+      if (error_msg != nullptr) {
+        // Create an error message.
+
+        std::ostringstream os;
+        os << "Explicit check options do not match runtime: ";
+        os << runtime->ExplicitNullChecks() << " vs " << explicit_null_checks << " | ";
+        os << runtime->ExplicitStackOverflowChecks() << " vs " << explicit_stack_overflow_checks
+            << " | ";
+        os << runtime->ExplicitSuspendChecks() << " vs " << explicit_suspend_checks;
+
+        *error_msg = os.str();
+      }
+
+      // Currently we do not create correct images when pre-opting, so the emulator will fail with
+      // this change. Once the change is in the tree, REMOVE.
+      if (true) {
+        // At least try to log it, though.
+        if (error_msg != nullptr) {
+          LOG(WARNING) << *error_msg;
+        }
+        return true;
+      } else {
+        return false;
+      }
+    }
+
+    // Accepted.
+    return true;
+  }
+
+  // Check (and override) the flags depending on current support in the ISA.
+  // Right now will reset all flags to explicit except on ARM.
+  static void CheckISASupport(InstructionSet isa, bool* explicit_null_checks,
+                              bool* explicit_stack_overflow_checks, bool* explicit_suspend_checks) {
+    switch (isa) {
+      case kArm:
+      case kThumb2:
+        break;  // All checks implemented, leave as is.
+
+      default:  // No checks implemented, reset all to explicit checks.
+        *explicit_null_checks = true;
+        *explicit_stack_overflow_checks = true;
+        *explicit_suspend_checks = true;
+    }
+  }
+
+  static bool CheckForCompiling(InstructionSet host, InstructionSet target,
+                                bool* explicit_null_checks, bool* explicit_stack_overflow_checks,
+                                bool* explicit_suspend_checks) {
+    // Check the boot image settings.
+    Runtime* runtime = Runtime::Current();
+    if (runtime != nullptr) {
+      gc::space::ImageSpace* ispace = runtime->GetHeap()->GetImageSpace();
+      if (ispace != nullptr) {
+        const OatFile* oat_file = ispace->GetOatFile();
+        if (oat_file != nullptr) {
+          const char* v = oat_file->GetOatHeader().GetStoreValueByKey(kImplicitChecksOatHeaderKey);
+          if (!Parse(v, explicit_null_checks, explicit_stack_overflow_checks,
+                     explicit_suspend_checks)) {
+            LOG(FATAL) << "Should have been able to parse boot image implicit check values";
+          }
+          return true;
+        }
+      }
+    }
+
+    // Check the current runtime.
+    bool cross_compiling = true;
+    switch (host) {
+      case kArm:
+      case kThumb2:
+        cross_compiling = target != kArm && target != kThumb2;
+        break;
+      default:
+        cross_compiling = host != target;
+        break;
+    }
+    if (!cross_compiling) {
+      Runtime* runtime = Runtime::Current();
+      *explicit_null_checks = runtime->ExplicitNullChecks();
+      *explicit_stack_overflow_checks = runtime->ExplicitStackOverflowChecks();
+      *explicit_suspend_checks = runtime->ExplicitSuspendChecks();
+      return true;
+    }
+
+    // Give up.
+    return false;
+  }
+};
+
+}  // namespace art
+
+#endif  // ART_RUNTIME_IMPLICIT_CHECK_OPTIONS_H_
diff --git a/runtime/oat.cc b/runtime/oat.cc
index 7c8e5bc..1421baf 100644
--- a/runtime/oat.cc
+++ b/runtime/oat.cc
@@ -17,6 +17,7 @@
 #include "oat.h"
 #include "utils.h"
 
+#include <string.h>
 #include <zlib.h>
 
 namespace art {
@@ -24,8 +25,38 @@ namespace art {
 const uint8_t OatHeader::kOatMagic[] = { 'o', 'a', 't', '\n' };
 const uint8_t OatHeader::kOatVersion[] = { '0', '3', '7', '\0' };
 
-OatHeader::OatHeader() {
-  memset(this, 0, sizeof(*this));
+static size_t ComputeOatHeaderSize(const SafeMap<std::string, std::string>* variable_data) {
+  size_t estimate = 0U;
+  if (variable_data != nullptr) {
+    SafeMap<std::string, std::string>::const_iterator it = variable_data->begin();
+    SafeMap<std::string, std::string>::const_iterator end = variable_data->end();
+    for ( ; it != end; ++it) {
+      estimate += it->first.length() + 1;
+      estimate += it->second.length() + 1;
+    }
+  }
+  return sizeof(OatHeader) + estimate;
+}
+
+OatHeader* OatHeader::Create(InstructionSet instruction_set,
+                             const InstructionSetFeatures& instruction_set_features,
+                             const std::vector<const DexFile*>* dex_files,
+                             uint32_t image_file_location_oat_checksum,
+                             uint32_t image_file_location_oat_data_begin,
+                             const SafeMap<std::string, std::string>* variable_data) {
+  // Estimate size of optional data.
+  size_t needed_size = ComputeOatHeaderSize(variable_data);
+
+  // Reserve enough memory.
+  void* memory = operator new (needed_size);
+
+  // Create the OatHeader in-place.
+  return new (memory) OatHeader(instruction_set,
+                                instruction_set_features,
+                                dex_files,
+                                image_file_location_oat_checksum,
+                                image_file_location_oat_data_begin,
+                                variable_data);
 }
 
 OatHeader::OatHeader(InstructionSet instruction_set,
@@ -33,7 +64,7 @@ OatHeader::OatHeader(InstructionSet instruction_set,
                      const std::vector<const DexFile*>* dex_files,
                      uint32_t image_file_location_oat_checksum,
                      uint32_t image_file_location_oat_data_begin,
-                     const std::string& image_file_location) {
+                     const SafeMap<std::string, std::string>* variable_data) {
   memcpy(magic_, kOatMagic, sizeof(kOatMagic));
   memcpy(version_, kOatVersion, sizeof(kOatVersion));
 
@@ -56,9 +87,16 @@ OatHeader::OatHeader(InstructionSet instruction_set,
   image_file_location_oat_data_begin_ = image_file_location_oat_data_begin;
   UpdateChecksum(&image_file_location_oat_data_begin_, sizeof(image_file_location_oat_data_begin_));
 
-  image_file_location_size_ = image_file_location.size();
-  UpdateChecksum(&image_file_location_size_, sizeof(image_file_location_size_));
-  UpdateChecksum(image_file_location.data(), image_file_location_size_);
+  // Flatten the map. Will also update variable_size_data_size_.
+  Flatten(variable_data);
+
+  // Update checksum for variable data size.
+  UpdateChecksum(&key_value_store_size_, sizeof(key_value_store_size_));
+
+  // Update for data, if existing.
+  if (key_value_store_size_ > 0U) {
+    UpdateChecksum(&key_value_store_, key_value_store_size_);
+  }
 
   executable_offset_ = 0;
   interpreter_to_interpreter_bridge_offset_ = 0;
@@ -327,20 +365,97 @@ uint32_t OatHeader::GetImageFileLocationOatDataBegin() const {
   return image_file_location_oat_data_begin_;
 }
 
-uint32_t OatHeader::GetImageFileLocationSize() const {
+uint32_t OatHeader::GetKeyValueStoreSize() const {
   CHECK(IsValid());
-  return image_file_location_size_;
+  return key_value_store_size_;
 }
 
-const uint8_t* OatHeader::GetImageFileLocationData() const {
+const uint8_t* OatHeader::GetKeyValueStore() const {
   CHECK(IsValid());
-  return image_file_location_data_;
+  return key_value_store_;
 }
 
-std::string OatHeader::GetImageFileLocation() const {
-  CHECK(IsValid());
-  return std::string(reinterpret_cast<const char*>(GetImageFileLocationData()),
-                     GetImageFileLocationSize());
+// Advance start until it is either end or \0.
+static const char* ParseString(const char* start, const char* end) {
+  while (start < end && *start != 0) {
+    start++;
+  }
+  return start;
+}
+
+const char* OatHeader::GetStoreValueByKey(const char* key) const {
+  const char* ptr = reinterpret_cast<const char*>(&key_value_store_);
+  const char* end = ptr + key_value_store_size_;
+
+  while (ptr < end) {
+    // Scan for a closing zero.
+    const char* str_end = ParseString(ptr, end);
+    if (str_end < end) {
+      if (strcmp(key, ptr) == 0) {
+        // Same as key. Check if value is OK.
+        if (ParseString(str_end + 1, end) < end) {
+          return str_end + 1;
+        }
+      } else {
+        // Different from key. Advance over the value.
+        ptr = ParseString(str_end + 1, end) + 1;
+      }
+    } else {
+      break;
+    }
+  }
+  // Not found.
+  return nullptr;
+}
+
+bool OatHeader::GetStoreKeyValuePairByIndex(size_t index, const char** key,
+                                            const char** value) const {
+  const char* ptr = reinterpret_cast<const char*>(&key_value_store_);
+  const char* end = ptr + key_value_store_size_;
+  ssize_t counter = static_cast<ssize_t>(index);
+
+  while (ptr < end && counter >= 0) {
+    // Scan for a closing zero.
+    const char* str_end = ParseString(ptr, end);
+    if (str_end < end) {
+      const char* maybe_key = ptr;
+      ptr = ParseString(str_end + 1, end) + 1;
+      if (ptr <= end) {
+        if (counter == 0) {
+          *key = maybe_key;
+          *value = str_end + 1;
+          return true;
+        } else {
+          counter--;
+        }
+      } else {
+        return false;
+      }
+    } else {
+      break;
+    }
+  }
+  // Not found.
+  return false;
+}
+
+size_t OatHeader::GetHeaderSize() const {
+  return sizeof(OatHeader) + key_value_store_size_;
+}
+
+void OatHeader::Flatten(const SafeMap<std::string, std::string>* key_value_store) {
+  char* data_ptr = reinterpret_cast<char*>(&key_value_store_);
+  if (key_value_store != nullptr) {
+    SafeMap<std::string, std::string>::const_iterator it = key_value_store->begin();
+    SafeMap<std::string, std::string>::const_iterator end = key_value_store->end();
+    for ( ; it != end; ++it) {
+      strcpy(data_ptr, it->first.c_str());
+      data_ptr += it->first.length() + 1;
+      strcpy(data_ptr, it->second.c_str());
+      data_ptr += it->second.length() + 1;
+    }
+  }
+  key_value_store_size_ = data_ptr - reinterpret_cast<char*>(&key_value_store_);
 }
 
 OatMethodOffsets::OatMethodOffsets()
diff --git a/runtime/oat.h b/runtime/oat.h
index 7be768c..fbed596 100644
--- a/runtime/oat.h
+++ b/runtime/oat.h
@@ -23,6 +23,7 @@
 #include "dex_file.h"
 #include "instruction_set.h"
 #include "quick/quick_method_frame_info.h"
+#include "safe_map.h"
 
 namespace art {
 
@@ -31,13 +32,16 @@ class PACKED(4) OatHeader {
   static const uint8_t kOatMagic[4];
   static const uint8_t kOatVersion[4];
 
-  OatHeader();
-  OatHeader(InstructionSet instruction_set,
-            const InstructionSetFeatures& instruction_set_features,
-            const std::vector<const DexFile*>* dex_files,
-            uint32_t image_file_location_oat_checksum,
-            uint32_t image_file_location_oat_data_begin,
-            const std::string& image_file_location);
+  static constexpr const char* kImageLocationKey = "image-location";
+  static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
+  static constexpr const char* kDex2OatHostKey = "dex2oat-host";
+
+  static OatHeader* Create(InstructionSet instruction_set,
+                           const InstructionSetFeatures& instruction_set_features,
+                           const std::vector<const DexFile*>* dex_files,
+                           uint32_t image_file_location_oat_checksum,
+                           uint32_t image_file_location_oat_data_begin,
+                           const SafeMap<std::string, std::string>* variable_data);
 
   bool IsValid() const;
   const char* GetMagic() const;
@@ -88,11 +92,24 @@ class PACKED(4) OatHeader {
   const InstructionSetFeatures& GetInstructionSetFeatures() const;
   uint32_t GetImageFileLocationOatChecksum() const;
   uint32_t GetImageFileLocationOatDataBegin() const;
-  uint32_t GetImageFileLocationSize() const;
-  const uint8_t* GetImageFileLocationData() const;
-  std::string GetImageFileLocation() const;
+
+  uint32_t GetKeyValueStoreSize() const;
+  const uint8_t* GetKeyValueStore() const;
+  const char* GetStoreValueByKey(const char* key) const;
+  bool GetStoreKeyValuePairByIndex(size_t index, const char** key, const char** value) const;
+
+  size_t GetHeaderSize() const;
 
  private:
+  OatHeader(InstructionSet instruction_set,
+            const InstructionSetFeatures& instruction_set_features,
+            const std::vector<const DexFile*>* dex_files,
+            uint32_t image_file_location_oat_checksum,
+            uint32_t image_file_location_oat_data_begin,
+            const SafeMap<std::string, std::string>* variable_data);
+
+  void Flatten(const SafeMap<std::string, std::string>* variable_data);
+
   uint8_t magic_[4];
   uint8_t version_[4];
   uint32_t adler32_checksum_;
@@ -114,8 +131,9 @@ class PACKED(4) OatHeader {
 
   uint32_t image_file_location_oat_checksum_;
   uint32_t image_file_location_oat_data_begin_;
-  uint32_t image_file_location_size_;
-  uint8_t image_file_location_data_[0];  // note variable width data at end
+
+  uint32_t key_value_store_size_;
+  uint8_t key_value_store_[0];  // note variable width data at end
 
   DISALLOW_COPY_AND_ASSIGN(OatHeader);
 };
diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc
index 6c44aa9..9cefcb6 100644
--- a/runtime/oat_file.cc
+++ b/runtime/oat_file.cc
@@ -17,17 +17,20 @@
 #include "oat_file.h"
 
 #include <dlfcn.h>
+#include <sstream>
 
 #include "base/bit_vector.h"
 #include "base/stl_util.h"
 #include "base/unix_file/fd_file.h"
 #include "elf_file.h"
+#include "implicit_check_options.h"
 #include "oat.h"
 #include "mirror/art_method.h"
 #include "mirror/art_method-inl.h"
 #include "mirror/class.h"
 #include "mirror/object-inl.h"
 #include "os.h"
+#include "runtime.h"
 #include "utils.h"
 #include "vmap_table.h"
 
@@ -55,28 +58,58 @@ OatFile* OatFile::Open(const std::string& filename,
                        std::string* error_msg) {
   CHECK(!filename.empty()) << location;
   CheckLocation(filename);
-  if (kUsePortableCompiler) {
+  std::unique_ptr<OatFile> ret;
+  if (kUsePortableCompiler && executable) {
     // If we are using PORTABLE, use dlopen to deal with relocations.
     //
     // We use our own ELF loader for Quick to deal with legacy apps that
     // open a generated dex file by name, remove the file, then open
     // another generated dex file with the same name. http://b/10614658
-    if (executable) {
-      return OpenDlopen(filename, location, requested_base, error_msg);
+    ret.reset(OpenDlopen(filename, location, requested_base, error_msg));
+  } else {
+    // If we aren't trying to execute, we just use our own ElfFile loader for a couple reasons:
+    //
+    // On target, dlopen may fail when compiling due to selinux restrictions on installd.
+    //
+    // On host, dlopen is expected to fail when cross compiling, so fall back to OpenElfFile.
+    // This won't work for portable runtime execution because it doesn't process relocations.
+    std::unique_ptr<File> file(OS::OpenFileForReading(filename.c_str()));
+    if (file.get() == NULL) {
+      *error_msg = StringPrintf("Failed to open oat filename for reading: %s", strerror(errno));
+      return nullptr;
     }
+    ret.reset(OpenElfFile(file.get(), location, requested_base, false, executable, error_msg));
+  }
+
+  if (ret.get() == nullptr) {
+    return nullptr;
+  }
+
+  // Embedded options check. Right now only implicit checks.
+  // TODO: Refactor to somewhere else?
+  const char* implicit_checks_value = ret->GetOatHeader().
+      GetStoreValueByKey(ImplicitCheckOptions::kImplicitChecksOatHeaderKey);
+
+  if (implicit_checks_value == nullptr) {
+    *error_msg = "Did not find implicit checks value.";
+    return nullptr;
   }
-  // If we aren't trying to execute, we just use our own ElfFile loader for a couple reasons:
-  //
-  // On target, dlopen may fail when compiling due to selinux restrictions on installd.
-  //
-  // On host, dlopen is expected to fail when cross compiling, so fall back to OpenElfFile.
-  // This won't work for portable runtime execution because it doesn't process relocations.
-  std::unique_ptr<File> file(OS::OpenFileForReading(filename.c_str()));
-  if (file.get() == NULL) {
-    *error_msg = StringPrintf("Failed to open oat filename for reading: %s", strerror(errno));
-    return NULL;
+
+  bool explicit_null_checks, explicit_so_checks, explicit_suspend_checks;
+  if (ImplicitCheckOptions::Parse(implicit_checks_value, &explicit_null_checks,
+                                  &explicit_so_checks, &explicit_suspend_checks)) {
+    // Check whether the runtime agrees with the recorded checks.
+    if (ImplicitCheckOptions::CheckRuntimeSupport(executable, explicit_null_checks,
+                                                  explicit_so_checks, explicit_suspend_checks,
+                                                  error_msg)) {
+      return ret.release();
+    } else {
+      return nullptr;
+    }
+  } else {
+    *error_msg = "Failed parsing implicit check options.";
+    return nullptr;
   }
-  return OpenElfFile(file.get(), location, requested_base, false, executable, error_msg);
 }
 
 OatFile* OatFile::OpenWritable(File* file, const std::string& location, std::string* error_msg) {
@@ -206,11 +239,11 @@ bool OatFile::Setup(std::string* error_msg) {
     return false;
   }
 
-  oat += GetOatHeader().GetImageFileLocationSize();
+  oat += GetOatHeader().GetKeyValueStoreSize();
   if (oat > End()) {
-    *error_msg = StringPrintf("In oat file '%s' found truncated image file location: "
+    *error_msg = StringPrintf("In oat file '%s' found truncated variable-size data: "
                               "%p + %zd + %ud <= %p", GetLocation().c_str(),
-                              Begin(), sizeof(OatHeader), GetOatHeader().GetImageFileLocationSize(),
+                              Begin(), sizeof(OatHeader), GetOatHeader().GetKeyValueStoreSize(),
                               End());
     return false;
   }