bpo-44511: Improve the bytecode for class and mapping patterns (GH-26922)

* Refactor mapping patterns and speed up class patterns.

* Simplify MATCH_KEYS and MATCH_CLASS.

* Add COPY opcode.

10 files changed, 130 insertions, 116 deletions

diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index 9958dea..85cc4af 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -755,15 +755,6 @@ iterations of the loop.
    .. versionadded:: 3.11
 
 
-.. opcode:: COPY_DICT_WITHOUT_KEYS
-
-   TOS is a tuple of mapping keys, and TOS1 is the match subject.  Replace TOS
-   with a :class:`dict` formed from the items of TOS1, but without any of the
-   keys in TOS.
-
-   .. versionadded:: 3.10
-
-
 .. opcode:: GET_LEN
 
    Push ``len(TOS)`` onto the stack.
@@ -795,11 +786,14 @@ iterations of the loop.
 
    TOS is a tuple of mapping keys, and TOS1 is the match subject.  If TOS1
    contains all of the keys in TOS, push a :class:`tuple` containing the
-   corresponding values, followed by ``True``. Otherwise, push ``None``,
-   followed by ``False``.
+   corresponding values. Otherwise, push ``None``.
 
    .. versionadded:: 3.10
 
+   .. versionchanged:: 3.11
+      Previously, this instruction also pushed a boolean value indicating
+      success (``True``) or failure (``False``).
+
 
 All of the following opcodes use their arguments.
 
@@ -1277,12 +1271,16 @@ All of the following opcodes use their arguments.
    against, and TOS2 is the match subject.  *count* is the number of positional
    sub-patterns.
 
-   Pop TOS.  If TOS2 is an instance of TOS1 and has the positional and keyword
-   attributes required by *count* and TOS, set TOS to ``True`` and TOS1 to a
-   tuple of extracted attributes.  Otherwise, set TOS to ``False``.
+   Pop TOS, TOS1, and TOS2.  If TOS2 is an instance of TOS1 and has the
+   positional and keyword attributes required by *count* and TOS, push a tuple
+   of extracted attributes.  Otherwise, push ``None``.
 
    .. versionadded:: 3.10
 
+   .. versionchanged:: 3.11
+      Previously, this instruction also pushed a boolean value indicating
+      success (``True``) or failure (``False``).
+
 .. opcode:: GEN_START (kind)
 
     Pops TOS. If TOS was not ``None``, raises an exception. The ``kind``
@@ -1301,6 +1299,14 @@ All of the following opcodes use their arguments.
    .. versionadded:: 3.10
 
 
+.. opcode:: COPY (i)
+
+   Push the *i*-th item to the top of the stack. The item is not removed from its
+   original location.
+
+   .. versionadded:: 3.11
+
+
 .. opcode:: HAVE_ARGUMENT
 
    This is not really an opcode.  It identifies the dividing line between
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 7b3ce9b..21ad466 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -308,6 +308,16 @@ CPython bytecode changes
   fashion as :opcode:`CALL_METHOD`, but also supports keyword arguments.  Works
   in tandem with :opcode:`LOAD_METHOD`.
 
+* Removed ``COPY_DICT_WITHOUT_KEYS``.
+
+* :opcode:`MATCH_CLASS` and :opcode:`MATCH_KEYS` no longer push an additional
+  boolean value indicating whether the match succeeded or failed. Instead, they
+  indicate failure with :const:`None` (where a tuple of extracted values would
+  otherwise be).
+
+* Added :opcode:`COPY`, which pushes the *i*-th item to the top of the stack.
+  The item is not removed from its original location.
+
 
 Deprecated
 ==========
diff --git a/Include/opcode.h b/Include/opcode.h
index f8c02b8..87ed32c 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -34,7 +34,6 @@ extern "C" {
 #define MATCH_MAPPING            31
 #define MATCH_SEQUENCE           32
 #define MATCH_KEYS               33
-#define COPY_DICT_WITHOUT_KEYS   34
 #define PUSH_EXC_INFO            35
 #define POP_EXCEPT_AND_RERAISE   37
 #define WITH_EXCEPT_START        49
@@ -104,6 +103,7 @@ extern "C" {
 #define IS_OP                   117
 #define CONTAINS_OP             118
 #define RERAISE                 119
+#define COPY                    120
 #define JUMP_IF_NOT_EXC_MATCH   121
 #define LOAD_FAST               124
 #define STORE_FAST              125
@@ -142,24 +142,24 @@ extern "C" {
 #define BINARY_ADD_UNICODE       14
 #define BINARY_ADD_UNICODE_INPLACE_FAST  18
 #define BINARY_MULTIPLY_ADAPTIVE  21
-#define BINARY_MULTIPLY_INT      36
-#define BINARY_MULTIPLY_FLOAT    38
-#define BINARY_SUBSCR_ADAPTIVE   39
-#define BINARY_SUBSCR_LIST_INT   40
-#define BINARY_SUBSCR_TUPLE_INT  41
-#define BINARY_SUBSCR_DICT       42
-#define CALL_FUNCTION_ADAPTIVE   43
-#define CALL_FUNCTION_BUILTIN_O  44
-#define CALL_FUNCTION_BUILTIN_FAST  45
-#define CALL_FUNCTION_LEN        46
-#define CALL_FUNCTION_ISINSTANCE  47
-#define CALL_FUNCTION_PY_SIMPLE  48
-#define JUMP_ABSOLUTE_QUICK      58
-#define LOAD_ATTR_ADAPTIVE       80
-#define LOAD_ATTR_INSTANCE_VALUE  81
-#define LOAD_ATTR_WITH_HINT      87
-#define LOAD_ATTR_SLOT           88
-#define LOAD_ATTR_MODULE        120
+#define BINARY_MULTIPLY_INT      34
+#define BINARY_MULTIPLY_FLOAT    36
+#define BINARY_SUBSCR_ADAPTIVE   38
+#define BINARY_SUBSCR_LIST_INT   39
+#define BINARY_SUBSCR_TUPLE_INT  40
+#define BINARY_SUBSCR_DICT       41
+#define CALL_FUNCTION_ADAPTIVE   42
+#define CALL_FUNCTION_BUILTIN_O  43
+#define CALL_FUNCTION_BUILTIN_FAST  44
+#define CALL_FUNCTION_LEN        45
+#define CALL_FUNCTION_ISINSTANCE  46
+#define CALL_FUNCTION_PY_SIMPLE  47
+#define JUMP_ABSOLUTE_QUICK      48
+#define LOAD_ATTR_ADAPTIVE       58
+#define LOAD_ATTR_INSTANCE_VALUE  80
+#define LOAD_ATTR_WITH_HINT      81
+#define LOAD_ATTR_SLOT           87
+#define LOAD_ATTR_MODULE         88
 #define LOAD_GLOBAL_ADAPTIVE    122
 #define LOAD_GLOBAL_MODULE      123
 #define LOAD_GLOBAL_BUILTIN     127
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index ef4f23a..4edcf9a 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -364,6 +364,8 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions)
 #     Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530)
 #     Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards)
+#     Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change
+#                         MATCH_CLASS and MATCH_KEYS, and add COPY)
 
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -373,7 +375,7 @@ _code_type = type(_write_atomic.__code__)
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3461).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3462).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'
diff --git a/Lib/opcode.py b/Lib/opcode.py
index 5377ec3..66d5ca7 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -85,7 +85,7 @@ def_op('GET_LEN', 30)
 def_op('MATCH_MAPPING', 31)
 def_op('MATCH_SEQUENCE', 32)
 def_op('MATCH_KEYS', 33)
-def_op('COPY_DICT_WITHOUT_KEYS', 34)
+
 def_op('PUSH_EXC_INFO', 35)
 
 def_op('POP_EXCEPT_AND_RERAISE', 37)
@@ -165,7 +165,7 @@ name_op('LOAD_GLOBAL', 116)     # Index in name list
 def_op('IS_OP', 117)
 def_op('CONTAINS_OP', 118)
 def_op('RERAISE', 119)
-
+def_op('COPY', 120)
 jabs_op('JUMP_IF_NOT_EXC_MATCH', 121)
 
 def_op('LOAD_FAST', 124)        # Local variable number
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-06-26-16-55-08.bpo-44511.k8sMvV.rst b/Misc/NEWS.d/next/Core and Builtins/2021-06-26-16-55-08.bpo-44511.k8sMvV.rst
new file mode 100644
index 0000000..352e046
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-06-26-16-55-08.bpo-44511.k8sMvV.rst
@@ -0,0 +1 @@
+Improve the generated bytecode for class and mapping patterns.
diff --git a/Programs/_freeze_importlib b/Programs/_freeze_importlib
new file mode 100755
index 0000000..8178564
--- /dev/null
+++ b/Programs/_freeze_importlib
diff --git a/Python/ceval.c b/Python/ceval.c
index adc7b53..a0f4c80 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -4143,25 +4143,30 @@ check_eval_breaker:
         }
 
         TARGET(MATCH_CLASS) {
-            // Pop TOS. On success, set TOS to True and TOS1 to a tuple of
-            // attributes. On failure, set TOS to False.
+            // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
+            // None on failure.
             PyObject *names = POP();
-            PyObject *type = TOP();
-            PyObject *subject = SECOND();
+            PyObject *type = POP();
+            PyObject *subject = TOP();
             assert(PyTuple_CheckExact(names));
             PyObject *attrs = match_class(tstate, subject, type, oparg, names);
             Py_DECREF(names);
+            Py_DECREF(type);
             if (attrs) {
                 // Success!
                 assert(PyTuple_CheckExact(attrs));
-                Py_DECREF(subject);
-                SET_SECOND(attrs);
+                SET_TOP(attrs);
             }
             else if (_PyErr_Occurred(tstate)) {
+                // Error!
                 goto error;
             }
-            Py_DECREF(type);
-            SET_TOP(PyBool_FromLong(!!attrs));
+            else {
+                // Failure!
+                Py_INCREF(Py_None);
+                SET_TOP(Py_None);
+            }
+            Py_DECREF(subject);
             DISPATCH();
         }
 
@@ -4171,6 +4176,7 @@ check_eval_breaker:
             PyObject *res = match ? Py_True : Py_False;
             Py_INCREF(res);
             PUSH(res);
+            PREDICT(POP_JUMP_IF_FALSE);
             DISPATCH();
         }
 
@@ -4180,12 +4186,12 @@ check_eval_breaker:
             PyObject *res = match ? Py_True : Py_False;
             Py_INCREF(res);
             PUSH(res);
+            PREDICT(POP_JUMP_IF_FALSE);
             DISPATCH();
         }
 
         TARGET(MATCH_KEYS) {
-            // On successful match for all keys, PUSH(values) and PUSH(True).
-            // Otherwise, PUSH(None) and PUSH(False).
+            // On successful match, PUSH(values). Otherwise, PUSH(None).
             PyObject *keys = TOP();
             PyObject *subject = SECOND();
             PyObject *values_or_none = match_keys(tstate, subject, keys);
@@ -4193,40 +4199,6 @@ check_eval_breaker:
                 goto error;
             }
             PUSH(values_or_none);
-            if (Py_IsNone(values_or_none)) {
-                Py_INCREF(Py_False);
-                PUSH(Py_False);
-                DISPATCH();
-            }
-            assert(PyTuple_CheckExact(values_or_none));
-            Py_INCREF(Py_True);
-            PUSH(Py_True);
-            DISPATCH();
-        }
-
-        TARGET(COPY_DICT_WITHOUT_KEYS) {
-            // rest = dict(TOS1)
-            // for key in TOS:
-            //     del rest[key]
-            // SET_TOP(rest)
-            PyObject *keys = TOP();
-            PyObject *subject = SECOND();
-            PyObject *rest = PyDict_New();
-            if (rest == NULL || PyDict_Update(rest, subject)) {
-                Py_XDECREF(rest);
-                goto error;
-            }
-            // This may seem a bit inefficient, but keys is rarely big enough to
-            // actually impact runtime.
-            assert(PyTuple_CheckExact(keys));
-            for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(keys); i++) {
-                if (PyDict_DelItem(rest, PyTuple_GET_ITEM(keys, i))) {
-                    Py_DECREF(rest);
-                    goto error;
-                }
-            }
-            Py_DECREF(keys);
-            SET_TOP(rest);
             DISPATCH();
         }
 
@@ -5027,6 +4999,14 @@ check_eval_breaker:
             DISPATCH();
         }
 
+        TARGET(COPY) {
+            assert(oparg != 0);
+            PyObject *peek = PEEK(oparg);
+            Py_INCREF(peek);
+            PUSH(peek);
+            DISPATCH();
+        }
+
         TARGET(EXTENDED_ARG) {
             int oldoparg = oparg;
             NEXTOPARG();
diff --git a/Python/compile.c b/Python/compile.c
index 0c025ac..28b5c07 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1248,18 +1248,17 @@ stack_effect(int opcode, int oparg, int jump)
         case DICT_MERGE:
         case DICT_UPDATE:
             return -1;
-        case COPY_DICT_WITHOUT_KEYS:
-            return 0;
         case MATCH_CLASS:
-            return -1;
+            return -2;
         case GET_LEN:
         case MATCH_MAPPING:
         case MATCH_SEQUENCE:
-            return 1;
         case MATCH_KEYS:
-            return 2;
+            return 1;
         case ROT_N:
             return 0;
+        case COPY:
+            return 1;
         default:
             return PY_INVALID_STACK_EFFECT;
     }
@@ -6118,10 +6117,16 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc)
     }
     ADDOP_LOAD_CONST_NEW(c, attr_names);
     ADDOP_I(c, MATCH_CLASS, nargs);
-    // TOS is now a tuple of (nargs + nattrs) attributes. Preserve it:
+    ADDOP(c, DUP_TOP);
+    ADDOP_LOAD_CONST(c, Py_None);
+    ADDOP_I(c, IS_OP, 1);
+    // TOS is now a tuple of (nargs + nattrs) attributes (or None):
     pc->on_top++;
     RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE));
+    ADDOP_I(c, UNPACK_SEQUENCE, nargs + nattrs);
+    pc->on_top += nargs + nattrs - 1;
     for (i = 0; i < nargs + nattrs; i++) {
+        pc->on_top--;
         pattern_ty pattern;
         if (i < nargs) {
             // Positional:
@@ -6132,17 +6137,12 @@ compiler_pattern_class(struct compiler *c, pattern_ty p, pattern_context *pc)
             pattern = asdl_seq_GET(kwd_patterns, i - nargs);
         }
         if (WILDCARD_CHECK(pattern)) {
+            ADDOP(c, POP_TOP);
             continue;
         }
-        // Get the i-th attribute, and match it against the i-th pattern:
-        ADDOP(c, DUP_TOP);
-        ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(i));
-        ADDOP(c, BINARY_SUBSCR);
         RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc));
     }
     // Success! Pop the tuple of attributes:
-    pc->on_top--;
-    ADDOP(c, POP_TOP);
     return 1;
 }
 
@@ -6183,7 +6183,7 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc)
         return compiler_error(c, "too many sub-patterns in mapping pattern");
     }
     // Collect all of the keys into a tuple for MATCH_KEYS and
-    // COPY_DICT_WITHOUT_KEYS. They can either be dotted names or literals:
+    // **rest. They can either be dotted names or literals:
 
     // Maintaining a set of Constant_kind kind keys allows us to raise a
     // SyntaxError in the case of duplicates.
@@ -6235,35 +6235,45 @@ compiler_pattern_mapping(struct compiler *c, pattern_ty p, pattern_context *pc)
     ADDOP(c, MATCH_KEYS);
     // There's now a tuple of keys and a tuple of values on top of the subject:
     pc->on_top += 2;
+    ADDOP(c, DUP_TOP);
+    ADDOP_LOAD_CONST(c, Py_None);
+    ADDOP_I(c, IS_OP, 1);
     RETURN_IF_FALSE(jump_to_fail_pop(c, pc, POP_JUMP_IF_FALSE));
     // So far so good. Use that tuple of values on the stack to match
     // sub-patterns against:
+    ADDOP_I(c, UNPACK_SEQUENCE, size);
+    pc->on_top += size - 1;
     for (Py_ssize_t i = 0; i < size; i++) {
+        pc->on_top--;
         pattern_ty pattern = asdl_seq_GET(patterns, i);
-        if (WILDCARD_CHECK(pattern)) {
-            continue;
-        }
-        ADDOP(c, DUP_TOP);
-        ADDOP_LOAD_CONST_NEW(c, PyLong_FromSsize_t(i));
-        ADDOP(c, BINARY_SUBSCR);
         RETURN_IF_FALSE(compiler_pattern_subpattern(c, pattern, pc));
     }
-    // If we get this far, it's a match! We're done with the tuple of values,
-    // and whatever happens next should consume the tuple of keys underneath it:
+    // If we get this far, it's a match! Whatever happens next should consume
+    // the tuple of keys and the subject:
     pc->on_top -= 2;
-    ADDOP(c, POP_TOP);
     if (star_target) {
-        // If we have a starred name, bind a dict of remaining items to it:
-        ADDOP(c, COPY_DICT_WITHOUT_KEYS);
+        // If we have a starred name, bind a dict of remaining items to it (this may
+        // seem a bit inefficient, but keys is rarely big enough to actually impact
+        // runtime):
+        // rest = dict(TOS1)
+        // for key in TOS:
+        //     del rest[key]
+        ADDOP_I(c, BUILD_MAP, 0);           // [subject, keys, empty]
+        ADDOP(c, ROT_THREE);                // [empty, subject, keys]
+        ADDOP(c, ROT_TWO);                  // [empty, keys, subject]
+        ADDOP_I(c, DICT_UPDATE, 2);         // [copy, keys]
+        ADDOP_I(c, UNPACK_SEQUENCE, size);  // [copy, keys...]
+        while (size) {
+            ADDOP_I(c, COPY, 1 + size--);   // [copy, keys..., copy]
+            ADDOP(c, ROT_TWO);              // [copy, keys..., copy, key]
+            ADDOP(c, DELETE_SUBSCR);        // [copy, keys...]
+        }
         RETURN_IF_FALSE(pattern_helper_store_name(c, star_target, pc));
     }
     else {
-        // Otherwise, we don't care about this tuple of keys anymore:
-        ADDOP(c, POP_TOP);
+        ADDOP(c, POP_TOP);  // Tuple of keys.
+        ADDOP(c, POP_TOP);  // Subject.
     }
-    // Pop the subject:
-    pc->on_top--;
-    ADDOP(c, POP_TOP);
     return 1;
 
 error:
@@ -8362,6 +8372,11 @@ optimize_basic_block(struct compiler *c, basicblock *bb, PyObject *consts)
                     fold_rotations(inst - oparg + 1, oparg);
                 }
                 break;
+            case COPY:
+                if (oparg == 1) {
+                    inst->i_opcode = DUP_TOP;
+                }
+                break;
             default:
                 /* All HAS_CONST opcodes should be handled with LOAD_CONST */
                 assert (!HAS_CONST(inst->i_opcode));
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 5c7d3ad..93a9e4a 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -33,11 +33,10 @@ static void *opcode_targets[256] = {
     &&TARGET_MATCH_MAPPING,
     &&TARGET_MATCH_SEQUENCE,
     &&TARGET_MATCH_KEYS,
-    &&TARGET_COPY_DICT_WITHOUT_KEYS,
-    &&TARGET_PUSH_EXC_INFO,
     &&TARGET_BINARY_MULTIPLY_INT,
-    &&TARGET_POP_EXCEPT_AND_RERAISE,
+    &&TARGET_PUSH_EXC_INFO,
     &&TARGET_BINARY_MULTIPLY_FLOAT,
+    &&TARGET_POP_EXCEPT_AND_RERAISE,
     &&TARGET_BINARY_SUBSCR_ADAPTIVE,
     &&TARGET_BINARY_SUBSCR_LIST_INT,
     &&TARGET_BINARY_SUBSCR_TUPLE_INT,
@@ -48,6 +47,7 @@ static void *opcode_targets[256] = {
     &&TARGET_CALL_FUNCTION_LEN,
     &&TARGET_CALL_FUNCTION_ISINSTANCE,
     &&TARGET_CALL_FUNCTION_PY_SIMPLE,
+    &&TARGET_JUMP_ABSOLUTE_QUICK,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
@@ -57,7 +57,7 @@ static void *opcode_targets[256] = {
     &&TARGET_INPLACE_ADD,
     &&TARGET_INPLACE_SUBTRACT,
     &&TARGET_INPLACE_MULTIPLY,
-    &&TARGET_JUMP_ABSOLUTE_QUICK,
+    &&TARGET_LOAD_ATTR_ADAPTIVE,
     &&TARGET_INPLACE_MODULO,
     &&TARGET_STORE_SUBSCR,
     &&TARGET_DELETE_SUBSCR,
@@ -79,15 +79,15 @@ static void *opcode_targets[256] = {
     &&TARGET_INPLACE_AND,
     &&TARGET_INPLACE_XOR,
     &&TARGET_INPLACE_OR,
-    &&TARGET_LOAD_ATTR_ADAPTIVE,
     &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
+    &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
     &&TARGET_YIELD_VALUE,
-    &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LOAD_ATTR_SLOT,
+    &&TARGET_LOAD_ATTR_MODULE,
     &&TARGET_POP_EXCEPT,
     &&TARGET_STORE_NAME,
     &&TARGET_DELETE_NAME,
@@ -119,7 +119,7 @@ static void *opcode_targets[256] = {
     &&TARGET_IS_OP,
     &&TARGET_CONTAINS_OP,
     &&TARGET_RERAISE,
-    &&TARGET_LOAD_ATTR_MODULE,
+    &&TARGET_COPY,
     &&TARGET_JUMP_IF_NOT_EXC_MATCH,
     &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_LOAD_GLOBAL_MODULE,