Fix OOM throwing if it happens in finalizer reference

The Class::Alloc should return null if OOM happened during
adding finalizer reference, even if finalizable object is
allocated succesfully.

Change-Id: I66c1cdda50228bf1302839785ce4d4889b676f5b
Signed-off-by: Dmitry Petrochenko <dmitry.petrochenko@intel.com>
Signed-off-by: Serguei Katkov <serguei.i.katkov@intel.com>

1 files changed, 46 insertions, 4 deletions

diff --git a/test/080-oom-throw/src/Main.java b/test/080-oom-throw/src/Main.java
index c93f8bb..63c5215 100644
--- a/test/080-oom-throw/src/Main.java
+++ b/test/080-oom-throw/src/Main.java
@@ -15,13 +15,15 @@
  */
 
 public class Main {
+    static char [][] holder;
+
     static class ArrayMemEater {
         static boolean sawOome;
 
         static void blowup(char[][] holder) {
             try {
                 for (int i = 0; i < holder.length; ++i) {
-                    holder[i] = new char[1024 * 1024];
+                    holder[i] = new char[1022 * 1024];
                 }
             } catch (OutOfMemoryError oome) {
                 ArrayMemEater.sawOome = true;
@@ -50,8 +52,30 @@ public class Main {
         }
     }
 
-    static boolean triggerArrayOOM() {
-        ArrayMemEater.blowup(new char[128 * 1024][]);
+    static class InstanceFinalizerMemEater {
+        static boolean sawOome;
+        static InstanceFinalizerMemEater hook;
+
+        InstanceFinalizerMemEater next;
+
+        static InstanceFinalizerMemEater allocate() {
+            try {
+                return new InstanceFinalizerMemEater();
+            } catch (OutOfMemoryError e) {
+                InstanceFinalizerMemEater.sawOome = true;
+                return null;
+            }
+        }
+
+        static void confuseCompilerOptimization(InstanceFinalizerMemEater instance) {
+            hook = instance;
+        }
+
+        protected void finalize() {}
+    }
+
+    static boolean triggerArrayOOM(char[][] holder) {
+        ArrayMemEater.blowup(holder);
         return ArrayMemEater.sawOome;
     }
 
@@ -67,11 +91,29 @@ public class Main {
         return InstanceMemEater.sawOome;
     }
 
+    static boolean triggerInstanceFinalizerOOM() {
+        InstanceFinalizerMemEater memEater = InstanceFinalizerMemEater.allocate();
+        InstanceFinalizerMemEater lastMemEater = memEater;
+        do {
+            lastMemEater.next = InstanceFinalizerMemEater.allocate();
+            lastMemEater = lastMemEater.next;
+        } while (lastMemEater != null);
+        memEater.confuseCompilerOptimization(memEater);
+        InstanceFinalizerMemEater.hook = null;
+        return InstanceFinalizerMemEater.sawOome;
+    }
+
     public static void main(String[] args) {
-        if (triggerArrayOOM()) {
+        // Keep holder alive to make instance OOM happen faster
+        holder = new char[128 * 1024][];
+        if (triggerArrayOOM(holder)) {
             System.out.println("NEW_ARRAY correctly threw OOME");
         }
 
+        if (!triggerInstanceFinalizerOOM()) {
+            System.out.println("NEW_INSTANCE (finalize) did not threw OOME");
+        }
+
         if (triggerInstanceOOM()) {
             System.out.println("NEW_INSTANCE correctly threw OOME");
         }