bpo-45367: Specialize BINARY_MULTIPLY (GH-28727)

3 files changed, 116 insertions, 23 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index aef83b9c35..de71ae5da0 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1933,14 +1933,73 @@ check_eval_breaker:
         }
 
         TARGET(BINARY_MULTIPLY) {
+            PREDICTED(BINARY_MULTIPLY);
+            STAT_INC(BINARY_MULTIPLY, unquickened);
             PyObject *right = POP();
             PyObject *left = TOP();
             PyObject *res = PyNumber_Multiply(left, right);
             Py_DECREF(left);
             Py_DECREF(right);
             SET_TOP(res);
-            if (res == NULL)
+            if (res == NULL) {
                 goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(BINARY_MULTIPLY_ADAPTIVE) {
+            if (oparg == 0) {
+                PyObject *left = SECOND();
+                PyObject *right = TOP();
+                next_instr--;
+                if (_Py_Specialize_BinaryMultiply(left, right, next_instr) < 0) {
+                    goto error;
+                }
+                DISPATCH();
+            }
+            else {
+                STAT_INC(BINARY_MULTIPLY, deferred);
+                UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1);
+                STAT_DEC(BINARY_MULTIPLY, unquickened);
+                JUMP_TO_INSTRUCTION(BINARY_MULTIPLY);
+            }
+        }
+
+        TARGET(BINARY_MULTIPLY_INT) {
+            PyObject *left = SECOND();
+            PyObject *right = TOP();
+            DEOPT_IF(!PyLong_CheckExact(left), BINARY_MULTIPLY);
+            DEOPT_IF(!PyLong_CheckExact(right), BINARY_MULTIPLY);
+            STAT_INC(BINARY_MULTIPLY, hit);
+            record_hit_inline(next_instr, oparg);
+            PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
+            SET_SECOND(prod);
+            Py_DECREF(right);
+            Py_DECREF(left);
+            STACK_SHRINK(1);
+            if (prod == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(BINARY_MULTIPLY_FLOAT) {
+            PyObject *left = SECOND();
+            PyObject *right = TOP();
+            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_MULTIPLY);
+            DEOPT_IF(!PyFloat_CheckExact(right), BINARY_MULTIPLY);
+            STAT_INC(BINARY_MULTIPLY, hit);
+            record_hit_inline(next_instr, oparg);
+            double dprod = ((PyFloatObject *)left)->ob_fval *
+                ((PyFloatObject *)right)->ob_fval;
+            PyObject *prod = PyFloat_FromDouble(dprod);
+            SET_SECOND(prod);
+            Py_DECREF(right);
+            Py_DECREF(left);
+            STACK_SHRINK(1);
+            if (prod == NULL) {
+                goto error;
+            }
             DISPATCH();
         }
 
@@ -4954,6 +5013,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
 MISS_WITH_CACHE(LOAD_METHOD)
 MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR)
 MISS_WITH_OPARG_COUNTER(BINARY_ADD)
+MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY)
 
 binary_subscr_dict_error:
         {
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 773f9254cf..7d701e81a9 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -20,7 +20,7 @@ static void *opcode_targets[256] = {
     &&TARGET_BINARY_ADD_UNICODE_INPLACE_FAST,
     &&TARGET_BINARY_POWER,
     &&TARGET_BINARY_MULTIPLY,
-    &&TARGET_BINARY_SUBSCR_ADAPTIVE,
+    &&TARGET_BINARY_MULTIPLY_ADAPTIVE,
     &&TARGET_BINARY_MODULO,
     &&TARGET_BINARY_ADD,
     &&TARGET_BINARY_SUBTRACT,
@@ -35,8 +35,11 @@ static void *opcode_targets[256] = {
     &&TARGET_MATCH_KEYS,
     &&TARGET_COPY_DICT_WITHOUT_KEYS,
     &&TARGET_PUSH_EXC_INFO,
-    &&TARGET_BINARY_SUBSCR_LIST_INT,
+    &&TARGET_BINARY_MULTIPLY_INT,
     &&TARGET_POP_EXCEPT_AND_RERAISE,
+    &&TARGET_BINARY_MULTIPLY_FLOAT,
+    &&TARGET_BINARY_SUBSCR_ADAPTIVE,
+    &&TARGET_BINARY_SUBSCR_LIST_INT,
     &&TARGET_BINARY_SUBSCR_TUPLE_INT,
     &&TARGET_BINARY_SUBSCR_DICT,
     &&TARGET_JUMP_ABSOLUTE_QUICK,
@@ -45,9 +48,6 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LOAD_ATTR_SLOT,
     &&TARGET_LOAD_ATTR_MODULE,
-    &&TARGET_LOAD_GLOBAL_ADAPTIVE,
-    &&TARGET_LOAD_GLOBAL_MODULE,
-    &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
@@ -57,7 +57,7 @@ static void *opcode_targets[256] = {
     &&TARGET_INPLACE_ADD,
     &&TARGET_INPLACE_SUBTRACT,
     &&TARGET_INPLACE_MULTIPLY,
-    &&TARGET_LOAD_METHOD_ADAPTIVE,
+    &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_INPLACE_MODULO,
     &&TARGET_STORE_SUBSCR,
     &&TARGET_DELETE_SUBSCR,
@@ -79,15 +79,15 @@ static void *opcode_targets[256] = {
     &&TARGET_INPLACE_AND,
     &&TARGET_INPLACE_XOR,
     &&TARGET_INPLACE_OR,
-    &&TARGET_LOAD_METHOD_CACHED,
-    &&TARGET_LOAD_METHOD_CLASS,
+    &&TARGET_LOAD_GLOBAL_MODULE,
+    &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
     &&TARGET_YIELD_VALUE,
-    &&TARGET_LOAD_METHOD_MODULE,
-    &&TARGET_LOAD_METHOD_NO_DICT,
+    &&TARGET_LOAD_METHOD_ADAPTIVE,
+    &&TARGET_LOAD_METHOD_CACHED,
     &&TARGET_POP_EXCEPT,
     &&TARGET_STORE_NAME,
     &&TARGET_DELETE_NAME,
@@ -119,40 +119,40 @@ static void *opcode_targets[256] = {
     &&TARGET_IS_OP,
     &&TARGET_CONTAINS_OP,
     &&TARGET_RERAISE,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
+    &&TARGET_LOAD_METHOD_CLASS,
     &&TARGET_JUMP_IF_NOT_EXC_MATCH,
-    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
-    &&TARGET_STORE_ATTR_SLOT,
+    &&TARGET_LOAD_METHOD_MODULE,
+    &&TARGET_LOAD_METHOD_NO_DICT,
     &&TARGET_LOAD_FAST,
     &&TARGET_STORE_FAST,
     &&TARGET_DELETE_FAST,
-    &&TARGET_STORE_ATTR_WITH_HINT,
-    &&TARGET_LOAD_FAST__LOAD_FAST,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
+    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_GEN_START,
     &&TARGET_RAISE_VARARGS,
     &&TARGET_CALL_FUNCTION,
     &&TARGET_MAKE_FUNCTION,
     &&TARGET_BUILD_SLICE,
-    &&TARGET_STORE_FAST__LOAD_FAST,
+    &&TARGET_STORE_ATTR_SLOT,
     &&TARGET_MAKE_CELL,
     &&TARGET_LOAD_CLOSURE,
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
-    &&TARGET_LOAD_FAST__LOAD_CONST,
+    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_CALL_FUNCTION_KW,
     &&TARGET_CALL_FUNCTION_EX,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
+    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
     &&TARGET_MAP_ADD,
     &&TARGET_LOAD_CLASSDEREF,
-    &&TARGET_STORE_FAST__STORE_FAST,
-    &&_unknown_opcode,
-    &&_unknown_opcode,
+    &&TARGET_STORE_FAST__LOAD_FAST,
+    &&TARGET_LOAD_FAST__LOAD_CONST,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_MATCH_CLASS,
-    &&_unknown_opcode,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&_unknown_opcode,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
diff --git a/Python/specialize.c b/Python/specialize.c
index 6efee7643a..529eabf6bc 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -124,6 +124,7 @@ _Py_GetSpecializationStats(void) {
     err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
     err += add_stat_dict(stats, LOAD_METHOD, "load_method");
     err += add_stat_dict(stats, BINARY_ADD, "binary_add");
+    err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply");
     err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
     err += add_stat_dict(stats, STORE_ATTR, "store_attr");
     if (err < 0) {
@@ -180,6 +181,7 @@ _Py_PrintSpecializationStats(void)
     print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global");
     print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method");
     print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add");
+    print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply");
     print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr");
     print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr");
     if (out != stderr) {
@@ -230,6 +232,7 @@ static uint8_t adaptive_opcodes[256] = {
     [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
     [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
     [BINARY_ADD] = BINARY_ADD_ADAPTIVE,
+    [BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE,
     [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
     [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
 };
@@ -240,6 +243,7 @@ static uint8_t cache_requirements[256] = {
     [LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */
     [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
     [BINARY_ADD] = 0,
+    [BINARY_MULTIPLY] = 0,
     [BINARY_SUBSCR] = 0,
     [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
 };
@@ -1188,3 +1192,32 @@ success:
     assert(!PyErr_Occurred());
     return 0;
 }
+
+int
+_Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
+{
+    if (!Py_IS_TYPE(left, Py_TYPE(right))) {
+        SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_DIFFERENT_TYPES);
+        goto fail;
+    }
+    if (PyLong_CheckExact(left)) {
+        *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_INT, saturating_start());
+        goto success;
+    }
+    else if (PyFloat_CheckExact(left)) {
+        *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_FLOAT, saturating_start());
+        goto success;
+    }
+    else {
+        SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_OTHER);
+    }
+fail:
+    STAT_INC(BINARY_MULTIPLY, specialization_failure);
+    assert(!PyErr_Occurred());
+    *instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF);
+    return 0;
+success:
+    STAT_INC(BINARY_MULTIPLY, specialization_success);
+    assert(!PyErr_Occurred());
+    return 0;
+}