8178870: instrumentation.retransformClasses cause coredump

Summary: Don't double-free cached class bytes on redefinition loading failure.
Reviewed-by: sspitsyn, jiangli

4 files changed, 340 insertions, 3 deletions

diff --git a/src/share/vm/prims/jvmtiRedefineClasses.cpp b/src/share/vm/prims/jvmtiRedefineClasses.cpp
index aac5f088d..8b99b217d 100644
--- a/src/share/vm/prims/jvmtiRedefineClasses.cpp
+++ b/src/share/vm/prims/jvmtiRedefineClasses.cpp
@@ -151,6 +151,11 @@ bool VM_RedefineClasses::doit_prologue() {
         ClassLoaderData* cld = _scratch_classes[i]->class_loader_data();
         // Free the memory for this class at class unloading time.  Not before
         // because CMS might think this is still live.
+        InstanceKlass* ik = get_ik(_class_defs[i].klass);
+        if (ik->get_cached_class_file() == ((InstanceKlass*)_scratch_classes[i])->get_cached_class_file()) {
+          // Don't double-free cached_class_file copied from the original class if error.
+          ((InstanceKlass*)_scratch_classes[i])->set_cached_class_file(NULL);
+        }
         cld->add_to_deallocate_list((InstanceKlass*)_scratch_classes[i]);
       }
     }
@@ -4019,12 +4024,12 @@ void VM_RedefineClasses::redefine_single_class(jclass the_jclass,
   // with them was cached on the scratch class, move to the_class.
   // Note: we still want to do this if nothing needed caching since it
   // should get cleared in the_class too.
-  if (the_class->get_cached_class_file_bytes() == 0) {
+  if (the_class->get_cached_class_file() == 0) {
     // the_class doesn't have a cache yet so copy it
     the_class->set_cached_class_file(scratch_class->get_cached_class_file());
   }
-  else if (scratch_class->get_cached_class_file_bytes() !=
-           the_class->get_cached_class_file_bytes()) {
+  else if (scratch_class->get_cached_class_file() !=
+           the_class->get_cached_class_file()) {
     // The same class can be present twice in the scratch classes list or there
     // are multiple concurrent RetransformClasses calls on different threads.
     // In such cases we have to deallocate scratch_class cached_class_file.
diff --git a/test/runtime/RedefineTests/RedefineDoubleDelete.java b/test/runtime/RedefineTests/RedefineDoubleDelete.java
new file mode 100644
index 000000000..7b18b45f1
--- /dev/null
+++ b/test/runtime/RedefineTests/RedefineDoubleDelete.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8178870
+ * @summary Redefine class with CFLH twice to test deleting the cached_class_file
+ */
+
+public class RedefineDoubleDelete {
+
+    // Class gets a redefinition error because it adds a data member
+    public static String newB =
+                "class RedefineDoubleDelete$B {" +
+                "   int count1 = 0;" +
+                "}";
+
+    public static String newerB =
+                "class RedefineDoubleDelete$B { " +
+                "   int faa() { System.out.println(\"baa\"); return 2; }" +
+                "}";
+
+    // The ClassFileLoadHook for this class turns foo into faa and prints out faa.
+    static class B {
+      int faa() { System.out.println("foo"); return 1; }
+    }
+
+    public static void main(String args[]) throws Exception {
+
+        B b = new B();
+        int val = b.faa();
+        if (val != 1) {
+            throw new RuntimeException("return value wrong " + val);
+        }
+
+        // Redefine B twice to get cached_class_file in both B scratch classes
+        try {
+            RedefineClassHelper.redefineClass(B.class, newB);
+        } catch (java.lang.UnsupportedOperationException e) {
+            // this is expected
+        }
+        try {
+            RedefineClassHelper.redefineClass(B.class, newB);
+        } catch (java.lang.UnsupportedOperationException e) {
+            // this is expected
+        }
+
+        // Do a full GC.
+        System.gc();
+
+        // Redefine with a compatible class
+        RedefineClassHelper.redefineClass(B.class, newerB);
+        val = b.faa();
+        if (val != 2) {
+            throw new RuntimeException("return value wrong " + val);
+        }
+
+        // Do another full GC to clean things up.
+        System.gc();
+    }
+}
diff --git a/test/runtime/RedefineTests/libRedefineDoubleDelete.c b/test/runtime/RedefineTests/libRedefineDoubleDelete.c
new file mode 100644
index 000000000..d855e873a
--- /dev/null
+++ b/test/runtime/RedefineTests/libRedefineDoubleDelete.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include "jvmti.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef JNI_ENV_ARG
+
+#ifdef __cplusplus
+#define JNI_ENV_ARG(x, y) y
+#define JNI_ENV_PTR(x) x
+#else
+#define JNI_ENV_ARG(x,y) x, y
+#define JNI_ENV_PTR(x) (*x)
+#endif
+
+#endif
+
+#define TranslateError(err) "JVMTI error"
+
+static jvmtiEnv *jvmti = NULL;
+
+static jint Agent_Initialize(JavaVM *jvm, char *options, void *reserved);
+
+JNIEXPORT
+jint JNICALL Agent_OnLoad(JavaVM *jvm, char *options, void *reserved) {
+    return Agent_Initialize(jvm, options, reserved);
+}
+
+JNIEXPORT
+jint JNICALL Agent_OnAttach(JavaVM *jvm, char *options, void *reserved) {
+    return Agent_Initialize(jvm, options, reserved);
+}
+
+JNIEXPORT
+jint JNICALL JNI_OnLoad(JavaVM *jvm, void *reserved) {
+    return JNI_VERSION_1_8;
+}
+
+
+static jint newClassDataLen = 0;
+static unsigned char* newClassData = NULL;
+
+static jint
+getBytecodes(jvmtiEnv *jvmti_env,
+             jint class_data_len, const unsigned char* class_data) {
+    int i;
+    jint res;
+
+    newClassDataLen = class_data_len;
+    res = (*jvmti_env)->Allocate(jvmti_env, newClassDataLen, &newClassData);
+    if (res != JNI_OK) {
+        printf("    Unable to allocate bytes\n");
+        return JNI_ERR;
+    }
+    for (i = 0; i < newClassDataLen; i++) {
+        newClassData[i] = class_data[i];
+        // Rewrite oo in class to aa
+        if (i > 0 && class_data[i] == 'o' && class_data[i-1] == 'o') {
+            newClassData[i] = newClassData[i-1] = 'a';
+        }
+    }
+    printf("  ... copied bytecode: %d bytes\n", (int)newClassDataLen);
+    return JNI_OK;
+}
+
+
+static void JNICALL
+Callback_ClassFileLoadHook(jvmtiEnv *jvmti_env, JNIEnv *env,
+                           jclass class_being_redefined,
+                           jobject loader, const char* name, jobject protection_domain,
+                           jint class_data_len, const unsigned char* class_data,
+                           jint *new_class_data_len, unsigned char** new_class_data) {
+    if (name != NULL && strcmp(name, "RedefineDoubleDelete$B") == 0) {
+        if (newClassData == NULL) {
+            jint res = getBytecodes(jvmti_env, class_data_len, class_data);
+            if (res == JNI_ERR) {
+              printf(">>>    ClassFileLoadHook event: class name %s FAILED\n", name);
+              return;
+            }
+            // Only change for first CFLH event.
+            *new_class_data_len = newClassDataLen;
+            *new_class_data = newClassData;
+        }
+        printf(">>>    ClassFileLoadHook event: class name %s\n", name);
+    }
+}
+
+static
+jint Agent_Initialize(JavaVM *jvm, char *options, void *reserved) {
+    jint res, size;
+    jvmtiCapabilities caps;
+    jvmtiEventCallbacks callbacks;
+    jvmtiError err;
+
+    res = JNI_ENV_PTR(jvm)->GetEnv(JNI_ENV_ARG(jvm, (void **) &jvmti),
+        JVMTI_VERSION_1_2);
+    if (res != JNI_OK || jvmti == NULL) {
+        printf("    Error: wrong result of a valid call to GetEnv!\n");
+        return JNI_ERR;
+    }
+
+    printf("Enabling following capabilities: can_generate_all_class_hook_events, "
+           "can_retransform_classes, can_redefine_classes");
+    memset(&caps, 0, sizeof(caps));
+    caps.can_generate_all_class_hook_events = 1;
+    caps.can_retransform_classes = 1;
+    caps.can_redefine_classes = 1;
+    printf("\n");
+
+    err = (*jvmti)->AddCapabilities(jvmti, &caps);
+    if (err != JVMTI_ERROR_NONE) {
+        printf("    Error in AddCapabilites: %s (%d)\n", TranslateError(err), err);
+        return JNI_ERR;
+    }
+
+    size = (jint)sizeof(callbacks);
+
+    memset(&callbacks, 0, sizeof(callbacks));
+    callbacks.ClassFileLoadHook = Callback_ClassFileLoadHook;
+
+    err = (*jvmti)->SetEventCallbacks(jvmti, &callbacks, size);
+    if (err != JVMTI_ERROR_NONE) {
+        printf("    Error in SetEventCallbacks: %s (%d)\n", TranslateError(err), err);
+        return JNI_ERR;
+    }
+
+    err = (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, NULL);
+    if (err != JVMTI_ERROR_NONE) {
+        printf("    Error in SetEventNotificationMode: %s (%d)\n", TranslateError(err), err);
+        return JNI_ERR;
+    }
+
+    return JNI_OK;
+}
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/test/runtime/RedefineTests/test8178870.sh b/test/runtime/RedefineTests/test8178870.sh
new file mode 100644
index 000000000..4a90e2655
--- /dev/null
+++ b/test/runtime/RedefineTests/test8178870.sh
@@ -0,0 +1,87 @@
+#!/bin/sh
+
+#
+# Copyright (c) 2019, Red Hat, Inc. All rights reserved.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+#
+
+## @test test.sh
+## @bug 8178870
+## @summary test instrumentation.retransformClasses
+## @run shell test.sh
+
+if [ "${TESTSRC}" = "" ]
+then
+  TESTSRC=${PWD}
+  echo "TESTSRC not set.  Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+## Adding common setup Variables for running shell tests.
+. ${TESTSRC}/../../test_env.sh
+
+LIB_SRC=${TESTSRC}/../../testlibrary/
+
+# set platform-dependent variables
+OS=`uname -s`
+echo "Testing on " $OS
+case "$OS" in
+  Linux)
+    cc_cmd=`which gcc`
+    if [ "x$cc_cmd" == "x" ]; then
+        echo "WARNING: gcc not found. Cannot execute test." 2>&1
+        exit 0;
+    fi
+    ;;
+  Solaris)
+    cc_cmd=`which cc`
+    if [ "x$cc_cmd" == "x" ]; then
+        echo "WARNING: cc not found. Cannot execute test." 2>&1
+        exit 0;
+    fi
+    ;;
+  *)
+    echo "Test passed. Only on Linux and Solaris"
+    exit 0;
+    ;;
+esac
+
+THIS_DIR=.
+
+cp ${TESTSRC}/RedefineDoubleDelete.java ${THIS_DIR}
+mkdir -p ${THIS_DIR}/classes
+${TESTJAVA}/bin/javac -sourcepath  ${LIB_SRC} -d ${THIS_DIR}/classes ${LIB_SRC}RedefineClassHelper.java
+${TESTJAVA}/bin/javac -cp .:${THIS_DIR}/classes:${TESTJAVA}/lib/tools.jar -d ${THIS_DIR} RedefineDoubleDelete.java
+
+$cc_cmd -fPIC -shared -o ${THIS_DIR}${FS}libRedefineDoubleDelete.so \
+    -I${TESTJAVA}/include -I${TESTJAVA}/include/linux \
+    ${TESTSRC}/libRedefineDoubleDelete.c
+
+LD_LIBRARY_PATH=${THIS_DIR}
+echo   LD_LIBRARY_PATH = ${LD_LIBRARY_PATH}
+export LD_LIBRARY_PATH
+
+# Install redefineagent.jar
+${TESTJAVA}/bin${FS}java -cp ${THIS_DIR}/classes RedefineClassHelper
+
+echo
+echo ${TESTJAVA}/bin/java -agentlib:RedefineDoubleDelete RedefineDoubleDelete
+${TESTJAVA}/bin/java -cp .:${THIS_DIR}${FS}classes -javaagent:redefineagent.jar -agentlib:RedefineDoubleDelete RedefineDoubleDelete
+