summaryrefslogtreecommitdiffstats
path: root/Python/pystrtod.c
blob: 209c9086c87b9c061cf015ea371275f9be12f795 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
/* -*- Mode: C; c-file-style: "python" -*- */

#include <Python.h>
#include <locale.h>

/* Case-insensitive string match used for nan and inf detection; t should be
   lower-case.  Returns 1 for a successful match, 0 otherwise. */

static int
case_insensitive_match(const char *s, const char *t)
{
    while(*t && Py_TOLOWER(*s) == *t) {
        s++;
        t++;
    }
    return *t ? 0 : 1;
}

/* _Py_parse_inf_or_nan: Attempt to parse a string of the form "nan", "inf" or
   "infinity", with an optional leading sign of "+" or "-".  On success,
   return the NaN or Infinity as a double and set *endptr to point just beyond
   the successfully parsed portion of the string.  On failure, return -1.0 and
   set *endptr to point to the start of the string. */

#ifndef PY_NO_SHORT_FLOAT_REPR

double
_Py_parse_inf_or_nan(const char *p, char **endptr)
{
    double retval;
    const char *s;
    int negate = 0;

    s = p;
    if (*s == '-') {
        negate = 1;
        s++;
    }
    else if (*s == '+') {
        s++;
    }
    if (case_insensitive_match(s, "inf")) {
        s += 3;
        if (case_insensitive_match(s, "inity"))
            s += 5;
        retval = _Py_dg_infinity(negate);
    }
    else if (case_insensitive_match(s, "nan")) {
        s += 3;
        retval = _Py_dg_stdnan(negate);
    }
    else {
        s = p;
        retval = -1.0;
    }
    *endptr = (char *)s;
    return retval;
}

#else

double
_Py_parse_inf_or_nan(const char *p, char **endptr)
{
    double retval;
    const char *s;
    int negate = 0;

    s = p;
    if (*s == '-') {
        negate = 1;
        s++;
    }
    else if (*s == '+') {
        s++;
    }
    if (case_insensitive_match(s, "inf")) {
        s += 3;
        if (case_insensitive_match(s, "inity"))
            s += 5;
        retval = negate ? -Py_HUGE_VAL : Py_HUGE_VAL;
    }
#ifdef Py_NAN
    else if (case_insensitive_match(s, "nan")) {
        s += 3;
        retval = negate ? -Py_NAN : Py_NAN;
    }
#endif
    else {
        s = p;
        retval = -1.0;
    }
    *endptr = (char *)s;
    return retval;
}

#endif

/**
 * _PyOS_ascii_strtod:
 * @nptr:    the string to convert to a numeric value.
 * @endptr:  if non-%NULL, it returns the character after
 *           the last character used in the conversion.
 *
 * Converts a string to a #gdouble value.
 * This function behaves like the standard strtod() function
 * does in the C locale. It does this without actually
 * changing the current locale, since that would not be
 * thread-safe.
 *
 * This function is typically used when reading configuration
 * files or other non-user input that should be locale independent.
 * To handle input from the user you should normally use the
 * locale-sensitive system strtod() function.
 *
 * If the correct value would cause overflow, plus or minus %HUGE_VAL
 * is returned (according to the sign of the value), and %ERANGE is
 * stored in %errno. If the correct value would cause underflow,
 * zero is returned and %ERANGE is stored in %errno.
 * If memory allocation fails, %ENOMEM is stored in %errno.
 *
 * This function resets %errno before calling strtod() so that
 * you can reliably detect overflow and underflow.
 *
 * Return value: the #gdouble value.
 **/

#ifndef PY_NO_SHORT_FLOAT_REPR

static double
_PyOS_ascii_strtod(const char *nptr, char **endptr)
{
    double result;
    _Py_SET_53BIT_PRECISION_HEADER;

    assert(nptr != NULL);
    /* Set errno to zero, so that we can distinguish zero results
       and underflows */
    errno = 0;

    _Py_SET_53BIT_PRECISION_START;
    result = _Py_dg_strtod(nptr, endptr);
    _Py_SET_53BIT_PRECISION_END;

    if (*endptr == nptr)
        /* string might represent an inf or nan */
        result = _Py_parse_inf_or_nan(nptr, endptr);

    return result;

}

#else

/*
   Use system strtod;  since strtod is locale aware, we may
   have to first fix the decimal separator.

   Note that unlike _Py_dg_strtod, the system strtod may not always give
   correctly rounded results.
*/

static double
_PyOS_ascii_strtod(const char *nptr, char **endptr)
{
    char *fail_pos;
    double val;
    struct lconv *locale_data;
    const char *decimal_point;
    size_t decimal_point_len;
    const char *p, *decimal_point_pos;
    const char *end = NULL; /* Silence gcc */
    const char *digits_pos = NULL;
    int negate = 0;

    assert(nptr != NULL);

    fail_pos = NULL;

    locale_data = localeconv();
    decimal_point = locale_data->decimal_point;
    decimal_point_len = strlen(decimal_point);

    assert(decimal_point_len != 0);

    decimal_point_pos = NULL;

    /* Parse infinities and nans */
    val = _Py_parse_inf_or_nan(nptr, endptr);
    if (*endptr != nptr)
        return val;

    /* Set errno to zero, so that we can distinguish zero results
       and underflows */
    errno = 0;

    /* We process the optional sign manually, then pass the remainder to
       the system strtod.  This ensures that the result of an underflow
       has the correct sign. (bug #1725)  */
    p = nptr;
    /* Process leading sign, if present */
    if (*p == '-') {
        negate = 1;
        p++;
    }
    else if (*p == '+') {
        p++;
    }

    /* Some platform strtods accept hex floats; Python shouldn't (at the
       moment), so we check explicitly for strings starting with '0x'. */
    if (*p == '0' && (*(p+1) == 'x' || *(p+1) == 'X'))
        goto invalid_string;

    /* Check that what's left begins with a digit or decimal point */
    if (!Py_ISDIGIT(*p) && *p != '.')
        goto invalid_string;

    digits_pos = p;
    if (decimal_point[0] != '.' ||
        decimal_point[1] != 0)
    {
        /* Look for a '.' in the input; if present, it'll need to be
           swapped for the current locale's decimal point before we
           call strtod.  On the other hand, if we find the current
           locale's decimal point then the input is invalid. */
        while (Py_ISDIGIT(*p))
            p++;

        if (*p == '.')
        {
            decimal_point_pos = p++;

            /* locate end of number */
            while (Py_ISDIGIT(*p))
                p++;

            if (*p == 'e' || *p == 'E')
                p++;
            if (*p == '+' || *p == '-')
                p++;
            while (Py_ISDIGIT(*p))
                p++;
            end = p;
        }
        else if (strncmp(p, decimal_point, decimal_point_len) == 0)
            /* Python bug #1417699 */
            goto invalid_string;
        /* For the other cases, we need not convert the decimal
           point */
    }

    if (decimal_point_pos) {
        char *copy, *c;
        /* Create a copy of the input, with the '.' converted to the
           locale-specific decimal point */
        copy = (char *)PyMem_MALLOC(end - digits_pos +
                                    1 + decimal_point_len);
        if (copy == NULL) {
            *endptr = (char *)nptr;
            errno = ENOMEM;
            return val;
        }

        c = copy;
        memcpy(c, digits_pos, decimal_point_pos - digits_pos);
        c += decimal_point_pos - digits_pos;
        memcpy(c, decimal_point, decimal_point_len);
        c += decimal_point_len;
        memcpy(c, decimal_point_pos + 1,
               end - (decimal_point_pos + 1));
        c += end - (decimal_point_pos + 1);
        *c = 0;

        val = strtod(copy, &fail_pos);

        if (fail_pos)
        {
            if (fail_pos > decimal_point_pos)
                fail_pos = (char *)digits_pos +
                    (fail_pos - copy) -
                    (decimal_point_len - 1);
            else
                fail_pos = (char *)digits_pos +
                    (fail_pos - copy);
        }

        PyMem_FREE(copy);

    }
    else {
        val = strtod(digits_pos, &fail_pos);
    }

    if (fail_pos == digits_pos)
        goto invalid_string;

    if (negate && fail_pos != nptr)
        val = -val;
    *endptr = fail_pos;

    return val;

  invalid_string:
    *endptr = (char*)nptr;
    errno = EINVAL;
    return -1.0;
}

#endif

/* PyOS_string_to_double converts a null-terminated byte string s (interpreted
   as a string of ASCII characters) to a float.  The string should not have
   leading or trailing whitespace.  The conversion is independent of the
   current locale.

   If endptr is NULL, try to convert the whole string.  Raise ValueError and
   return -1.0 if the string is not a valid representation of a floating-point
   number.

   If endptr is non-NULL, try to convert as much of the string as possible.
   If no initial segment of the string is the valid representation of a
   floating-point number then *endptr is set to point to the beginning of the
   string, -1.0 is returned and again ValueError is raised.

   On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine),
   if overflow_exception is NULL then +-Py_HUGE_VAL is returned, and no Python
   exception is raised.  Otherwise, overflow_exception should point to
   a Python exception, this exception will be raised, -1.0 will be returned,
   and *endptr will point just past the end of the converted value.

   If any other failure occurs (for example lack of memory), -1.0 is returned
   and the appropriate Python exception will have been set.
*/

double
PyOS_string_to_double(const char *s,
                      char **endptr,
                      PyObject *overflow_exception)
{
    double x, result=-1.0;
    char *fail_pos;

    errno = 0;
    PyFPE_START_PROTECT("PyOS_string_to_double", return -1.0)
    x = _PyOS_ascii_strtod(s, &fail_pos);
    PyFPE_END_PROTECT(x)

    if (errno == ENOMEM) {
        PyErr_NoMemory();
        fail_pos = (char *)s;
    }
    else if (!endptr && (fail_pos == s || *fail_pos != '\0'))
        PyErr_Format(PyExc_ValueError,
                      "could not convert string to float: "
                      "%.200s", s);
    else if (fail_pos == s)
        PyErr_Format(PyExc_ValueError,
                      "could not convert string to float: "
                      "%.200s", s);
    else if (errno == ERANGE && fabs(x) >= 1.0 && overflow_exception)
        PyErr_Format(overflow_exception,
                      "value too large to convert to float: "
                      "%.200s", s);
    else
        result = x;

    if (endptr != NULL)
        *endptr = fail_pos;
    return result;
}

#ifdef PY_NO_SHORT_FLOAT_REPR

/* Given a string that may have a decimal point in the current
   locale, change it back to a dot.  Since the string cannot get
   longer, no need for a maximum buffer size parameter. */
Py_LOCAL_INLINE(void)
change_decimal_from_locale_to_dot(char* buffer)
{
    struct lconv *locale_data = localeconv();
    const char *decimal_point = locale_data->decimal_point;

    if (decimal_point[0] != '.' || decimal_point[1] != 0) {
        size_t decimal_point_len = strlen(decimal_point);

        if (*buffer == '+' || *buffer == '-')
            buffer++;
        while (Py_ISDIGIT(*buffer))
            buffer++;
        if (strncmp(buffer, decimal_point, decimal_point_len) == 0) {
            *buffer = '.';
            buffer++;
            if (decimal_point_len > 1) {
                /* buffer needs to get smaller */
                size_t rest_len = strlen(buffer +
                                     (decimal_point_len - 1));
                memmove(buffer,
                    buffer + (decimal_point_len - 1),
                    rest_len);
                buffer[rest_len] = 0;
            }
        }
    }
}


/* From the C99 standard, section 7.19.6:
The exponent always contains at least two digits, and only as many more digits
as necessary to represent the exponent.
*/
#define MIN_EXPONENT_DIGITS 2

/* Ensure that any exponent, if present, is at least MIN_EXPONENT_DIGITS
   in length. */
Py_LOCAL_INLINE(void)
ensure_minimum_exponent_length(char* buffer, size_t buf_size)
{
    char *p = strpbrk(buffer, "eE");
    if (p && (*(p + 1) == '-' || *(p + 1) == '+')) {
        char *start = p + 2;
        int exponent_digit_cnt = 0;
        int leading_zero_cnt = 0;
        int in_leading_zeros = 1;
        int significant_digit_cnt;

        /* Skip over the exponent and the sign. */
        p += 2;

        /* Find the end of the exponent, keeping track of leading
           zeros. */
        while (*p && Py_ISDIGIT(*p)) {
            if (in_leading_zeros && *p == '0')
                ++leading_zero_cnt;
            if (*p != '0')
                in_leading_zeros = 0;
            ++p;
            ++exponent_digit_cnt;
        }

        significant_digit_cnt = exponent_digit_cnt - leading_zero_cnt;
        if (exponent_digit_cnt == MIN_EXPONENT_DIGITS) {
            /* If there are 2 exactly digits, we're done,
               regardless of what they contain */
        }
        else if (exponent_digit_cnt > MIN_EXPONENT_DIGITS) {
            int extra_zeros_cnt;

            /* There are more than 2 digits in the exponent.  See
               if we can delete some of the leading zeros */
            if (significant_digit_cnt < MIN_EXPONENT_DIGITS)
                significant_digit_cnt = MIN_EXPONENT_DIGITS;
            extra_zeros_cnt = exponent_digit_cnt -
                significant_digit_cnt;

            /* Delete extra_zeros_cnt worth of characters from the
               front of the exponent */
            assert(extra_zeros_cnt >= 0);

            /* Add one to significant_digit_cnt to copy the
               trailing 0 byte, thus setting the length */
            memmove(start,
                start + extra_zeros_cnt,
                significant_digit_cnt + 1);
        }
        else {
            /* If there are fewer than 2 digits, add zeros
               until there are 2, if there's enough room */
            int zeros = MIN_EXPONENT_DIGITS - exponent_digit_cnt;
            if (start + zeros + exponent_digit_cnt + 1
                  < buffer + buf_size) {
                memmove(start + zeros, start,
                    exponent_digit_cnt + 1);
                memset(start, '0', zeros);
            }
        }
    }
}

/* Remove trailing zeros after the decimal point from a numeric string; also
   remove the decimal point if all digits following it are zero.  The numeric
   string must end in '\0', and should not have any leading or trailing
   whitespace.  Assumes that the decimal point is '.'. */
Py_LOCAL_INLINE(void)
remove_trailing_zeros(char *buffer)
{
    char *old_fraction_end, *new_fraction_end, *end, *p;

    p = buffer;
    if (*p == '-' || *p == '+')
        /* Skip leading sign, if present */
        ++p;
    while (Py_ISDIGIT(*p))
        ++p;

    /* if there's no decimal point there's nothing to do */
    if (*p++ != '.')
        return;

    /* scan any digits after the point */
    while (Py_ISDIGIT(*p))
        ++p;
    old_fraction_end = p;

    /* scan up to ending '\0' */
    while (*p != '\0')
        p++;
    /* +1 to make sure that we move the null byte as well */
    end = p+1;

    /* scan back from fraction_end, looking for removable zeros */
    p = old_fraction_end;
    while (*(p-1) == '0')
        --p;
    /* and remove point if we've got that far */
    if (*(p-1) == '.')
        --p;
    new_fraction_end = p;

    memmove(new_fraction_end, old_fraction_end, end-old_fraction_end);
}

/* Ensure that buffer has a decimal point in it.  The decimal point will not
   be in the current locale, it will always be '.'. Don't add a decimal point
   if an exponent is present.  Also, convert to exponential notation where
   adding a '.0' would produce too many significant digits (see issue 5864).

   Returns a pointer to the fixed buffer, or NULL on failure.
*/
Py_LOCAL_INLINE(char *)
ensure_decimal_point(char* buffer, size_t buf_size, int precision)
{
    int digit_count, insert_count = 0, convert_to_exp = 0;
    char *chars_to_insert, *digits_start;

    /* search for the first non-digit character */
    char *p = buffer;
    if (*p == '-' || *p == '+')
        /* Skip leading sign, if present.  I think this could only
           ever be '-', but it can't hurt to check for both. */
        ++p;
    digits_start = p;
    while (*p && Py_ISDIGIT(*p))
        ++p;
    digit_count = Py_SAFE_DOWNCAST(p - digits_start, Py_ssize_t, int);

    if (*p == '.') {
        if (Py_ISDIGIT(*(p+1))) {
            /* Nothing to do, we already have a decimal
               point and a digit after it */
        }
        else {
            /* We have a decimal point, but no following
               digit.  Insert a zero after the decimal. */
            /* can't ever get here via PyOS_double_to_string */
            assert(precision == -1);
            ++p;
            chars_to_insert = "0";
            insert_count = 1;
        }
    }
    else if (!(*p == 'e' || *p == 'E')) {
        /* Don't add ".0" if we have an exponent. */
        if (digit_count == precision) {
            /* issue 5864: don't add a trailing .0 in the case
               where the '%g'-formatted result already has as many
               significant digits as were requested.  Switch to
               exponential notation instead. */
            convert_to_exp = 1;
            /* no exponent, no point, and we shouldn't land here
               for infs and nans, so we must be at the end of the
               string. */
            assert(*p == '\0');
        }
        else {
            assert(precision == -1 || digit_count < precision);
            chars_to_insert = ".0";
            insert_count = 2;
        }
    }
    if (insert_count) {
        size_t buf_len = strlen(buffer);
        if (buf_len + insert_count + 1 >= buf_size) {
            /* If there is not enough room in the buffer
               for the additional text, just skip it.  It's
               not worth generating an error over. */
        }
        else {
            memmove(p + insert_count, p,
                buffer + strlen(buffer) - p + 1);
            memcpy(p, chars_to_insert, insert_count);
        }
    }
    if (convert_to_exp) {
        int written;
        size_t buf_avail;
        p = digits_start;
        /* insert decimal point */
        assert(digit_count >= 1);
        memmove(p+2, p+1, digit_count); /* safe, but overwrites nul */
        p[1] = '.';
        p += digit_count+1;
        assert(p <= buf_size+buffer);
        buf_avail = buf_size+buffer-p;
        if (buf_avail == 0)
            return NULL;
        /* Add exponent.  It's okay to use lower case 'e': we only
           arrive here as a result of using the empty format code or
           repr/str builtins and those never want an upper case 'E' */
        written = PyOS_snprintf(p, buf_avail, "e%+.02d", digit_count-1);
        if (!(0 <= written &&
              written < Py_SAFE_DOWNCAST(buf_avail, size_t, int)))
            /* output truncated, or something else bad happened */
            return NULL;
        remove_trailing_zeros(buffer);
    }
    return buffer;
}

/* see FORMATBUFLEN in unicodeobject.c */
#define FLOAT_FORMATBUFLEN 120

/**
 * _PyOS_ascii_formatd:
 * @buffer: A buffer to place the resulting string in
 * @buf_size: The length of the buffer.
 * @format: The printf()-style format to use for the
 *          code to use for converting.
 * @d: The #gdouble to convert
 * @precision: The precision to use when formatting.
 *
 * Converts a #gdouble to a string, using the '.' as
 * decimal point. To format the number you pass in
 * a printf()-style format string. Allowed conversion
 * specifiers are 'e', 'E', 'f', 'F', 'g', 'G', and 'Z'.
 *
 * 'Z' is the same as 'g', except it always has a decimal and
 *     at least one digit after the decimal.
 *
 * Return value: The pointer to the buffer with the converted string.
 * On failure returns NULL but does not set any Python exception.
 **/
static char *
_PyOS_ascii_formatd(char       *buffer,
                   size_t      buf_size,
                   const char *format,
                   double      d,
                   int         precision)
{
    char format_char;
    size_t format_len = strlen(format);

    /* Issue 2264: code 'Z' requires copying the format.  'Z' is 'g', but
       also with at least one character past the decimal. */
    char tmp_format[FLOAT_FORMATBUFLEN];

    /* The last character in the format string must be the format char */
    format_char = format[format_len - 1];

    if (format[0] != '%')
        return NULL;

    /* I'm not sure why this test is here.  It's ensuring that the format
       string after the first character doesn't have a single quote, a
       lowercase l, or a percent. This is the reverse of the commented-out
       test about 10 lines ago. */
    if (strpbrk(format + 1, "'l%"))
        return NULL;

    /* Also curious about this function is that it accepts format strings
       like "%xg", which are invalid for floats.  In general, the
       interface to this function is not very good, but changing it is
       difficult because it's a public API. */

    if (!(format_char == 'e' || format_char == 'E' ||
          format_char == 'f' || format_char == 'F' ||
          format_char == 'g' || format_char == 'G' ||
          format_char == 'Z'))
        return NULL;

    /* Map 'Z' format_char to 'g', by copying the format string and
       replacing the final char with a 'g' */
    if (format_char == 'Z') {
        if (format_len + 1 >= sizeof(tmp_format)) {
            /* The format won't fit in our copy.  Error out.  In
               practice, this will never happen and will be
               detected by returning NULL */
            return NULL;
        }
        strcpy(tmp_format, format);
        tmp_format[format_len - 1] = 'g';
        format = tmp_format;
    }


    /* Have PyOS_snprintf do the hard work */
    PyOS_snprintf(buffer, buf_size, format, d);

    /* Do various fixups on the return string */

    /* Get the current locale, and find the decimal point string.
       Convert that string back to a dot. */
    change_decimal_from_locale_to_dot(buffer);

    /* If an exponent exists, ensure that the exponent is at least
       MIN_EXPONENT_DIGITS digits, providing the buffer is large enough
       for the extra zeros.  Also, if there are more than
       MIN_EXPONENT_DIGITS, remove as many zeros as possible until we get
       back to MIN_EXPONENT_DIGITS */
    ensure_minimum_exponent_length(buffer, buf_size);

    /* If format_char is 'Z', make sure we have at least one character
       after the decimal point (and make sure we have a decimal point);
       also switch to exponential notation in some edge cases where the
       extra character would produce more significant digits that we
       really want. */
    if (format_char == 'Z')
        buffer = ensure_decimal_point(buffer, buf_size, precision);

    return buffer;
}

/* The fallback code to use if _Py_dg_dtoa is not available. */

PyAPI_FUNC(char *) PyOS_double_to_string(double val,
                                         char format_code,
                                         int precision,
                                         int flags,
                                         int *type)
{
    char format[32];
    Py_ssize_t bufsize;
    char *buf;
    int t, exp;
    int upper = 0;

    /* Validate format_code, and map upper and lower case */
    switch (format_code) {
    case 'e':          /* exponent */
    case 'f':          /* fixed */
    case 'g':          /* general */
        break;
    case 'E':
        upper = 1;
        format_code = 'e';
        break;
    case 'F':
        upper = 1;
        format_code = 'f';
        break;
    case 'G':
        upper = 1;
        format_code = 'g';
        break;
    case 'r':          /* repr format */
        /* Supplied precision is unused, must be 0. */
        if (precision != 0) {
            PyErr_BadInternalCall();
            return NULL;
        }
        /* The repr() precision (17 significant decimal digits) is the
           minimal number that is guaranteed to have enough precision
           so that if the number is read back in the exact same binary
           value is recreated.  This is true for IEEE floating point
           by design, and also happens to work for all other modern
           hardware. */
        precision = 17;
        format_code = 'g';
        break;
    default:
        PyErr_BadInternalCall();
        return NULL;
    }

    /* Here's a quick-and-dirty calculation to figure out how big a buffer
       we need.  In general, for a finite float we need:

         1 byte for each digit of the decimal significand, and

         1 for a possible sign
         1 for a possible decimal point
         2 for a possible [eE][+-]
         1 for each digit of the exponent;  if we allow 19 digits
           total then we're safe up to exponents of 2**63.
         1 for the trailing nul byte

       This gives a total of 24 + the number of digits in the significand,
       and the number of digits in the significand is:

         for 'g' format: at most precision, except possibly
           when precision == 0, when it's 1.
         for 'e' format: precision+1
         for 'f' format: precision digits after the point, at least 1
           before.  To figure out how many digits appear before the point
           we have to examine the size of the number.  If fabs(val) < 1.0
           then there will be only one digit before the point.  If
           fabs(val) >= 1.0, then there are at most

         1+floor(log10(ceiling(fabs(val))))

           digits before the point (where the 'ceiling' allows for the
           possibility that the rounding rounds the integer part of val
           up).  A safe upper bound for the above quantity is
           1+floor(exp/3), where exp is the unique integer such that 0.5
           <= fabs(val)/2**exp < 1.0.  This exp can be obtained from
           frexp.

       So we allow room for precision+1 digits for all formats, plus an
       extra floor(exp/3) digits for 'f' format.

    */

    if (Py_IS_NAN(val) || Py_IS_INFINITY(val))
        /* 3 for 'inf'/'nan', 1 for sign, 1 for '\0' */
        bufsize = 5;
    else {
        bufsize = 25 + precision;
        if (format_code == 'f' && fabs(val) >= 1.0) {
            frexp(val, &exp);
            bufsize += exp/3;
        }
    }

    buf = PyMem_Malloc(bufsize);
    if (buf == NULL) {
        PyErr_NoMemory();
        return NULL;
    }

    /* Handle nan and inf. */
    if (Py_IS_NAN(val)) {
        strcpy(buf, "nan");
        t = Py_DTST_NAN;
    } else if (Py_IS_INFINITY(val)) {
        if (copysign(1., val) == 1.)
            strcpy(buf, "inf");
        else
            strcpy(buf, "-inf");
        t = Py_DTST_INFINITE;
    } else {
        t = Py_DTST_FINITE;
        if (flags & Py_DTSF_ADD_DOT_0)
            format_code = 'Z';

        PyOS_snprintf(format, sizeof(format), "%%%s.%i%c",
                      (flags & Py_DTSF_ALT ? "#" : ""), precision,
                      format_code);
        _PyOS_ascii_formatd(buf, bufsize, format, val, precision);
    }

    /* Add sign when requested.  It's convenient (esp. when formatting
     complex numbers) to include a sign even for inf and nan. */
    if (flags & Py_DTSF_SIGN && buf[0] != '-') {
        size_t len = strlen(buf);
        /* the bufsize calculations above should ensure that we've got
           space to add a sign */
        assert((size_t)bufsize >= len+2);
        memmove(buf+1, buf, len+1);
        buf[0] = '+';
    }
    if (upper) {
        /* Convert to upper case. */
        char *p1;
        for (p1 = buf; *p1; p1++)
            *p1 = Py_TOUPPER(*p1);
    }

    if (type)
        *type = t;
    return buf;
}

#else

/* _Py_dg_dtoa is available. */

/* I'm using a lookup table here so that I don't have to invent a non-locale
   specific way to convert to uppercase */
#define OFS_INF 0
#define OFS_NAN 1
#define OFS_E 2

/* The lengths of these are known to the code below, so don't change them */
static char *lc_float_strings[] = {
    "inf",
    "nan",
    "e",
};
static char *uc_float_strings[] = {
    "INF",
    "NAN",
    "E",
};


/* Convert a double d to a string, and return a PyMem_Malloc'd block of
   memory contain the resulting string.

   Arguments:
     d is the double to be converted
     format_code is one of 'e', 'f', 'g', 'r'.  'e', 'f' and 'g'
       correspond to '%e', '%f' and '%g';  'r' corresponds to repr.
     mode is one of '0', '2' or '3', and is completely determined by
       format_code: 'e' and 'g' use mode 2; 'f' mode 3, 'r' mode 0.
     precision is the desired precision
     always_add_sign is nonzero if a '+' sign should be included for positive
       numbers
     add_dot_0_if_integer is nonzero if integers in non-exponential form
       should have ".0" added.  Only applies to format codes 'r' and 'g'.
     use_alt_formatting is nonzero if alternative formatting should be
       used.  Only applies to format codes 'e', 'f' and 'g'.  For code 'g',
       at most one of use_alt_formatting and add_dot_0_if_integer should
       be nonzero.
     type, if non-NULL, will be set to one of these constants to identify
       the type of the 'd' argument:
     Py_DTST_FINITE
     Py_DTST_INFINITE
     Py_DTST_NAN

   Returns a PyMem_Malloc'd block of memory containing the resulting string,
    or NULL on error. If NULL is returned, the Python error has been set.
 */

static char *
format_float_short(double d, char format_code,
                   int mode, int precision,
                   int always_add_sign, int add_dot_0_if_integer,
                   int use_alt_formatting, char **float_strings, int *type)
{
    char *buf = NULL;
    char *p = NULL;
    Py_ssize_t bufsize = 0;
    char *digits, *digits_end;
    int decpt_as_int, sign, exp_len, exp = 0, use_exp = 0;
    Py_ssize_t decpt, digits_len, vdigits_start, vdigits_end;
    _Py_SET_53BIT_PRECISION_HEADER;

    /* _Py_dg_dtoa returns a digit string (no decimal point or exponent).
       Must be matched by a call to _Py_dg_freedtoa. */
    _Py_SET_53BIT_PRECISION_START;
    digits = _Py_dg_dtoa(d, mode, precision, &decpt_as_int, &sign,
                         &digits_end);
    _Py_SET_53BIT_PRECISION_END;

    decpt = (Py_ssize_t)decpt_as_int;
    if (digits == NULL) {
        /* The only failure mode is no memory. */
        PyErr_NoMemory();
        goto exit;
    }
    assert(digits_end != NULL && digits_end >= digits);
    digits_len = digits_end - digits;

    if (digits_len && !Py_ISDIGIT(digits[0])) {
        /* Infinities and nans here; adapt Gay's output,
           so convert Infinity to inf and NaN to nan, and
           ignore sign of nan. Then return. */

        /* ignore the actual sign of a nan */
        if (digits[0] == 'n' || digits[0] == 'N')
            sign = 0;

        /* We only need 5 bytes to hold the result "+inf\0" . */
        bufsize = 5; /* Used later in an assert. */
        buf = (char *)PyMem_Malloc(bufsize);
        if (buf == NULL) {
            PyErr_NoMemory();
            goto exit;
        }
        p = buf;

        if (sign == 1) {
            *p++ = '-';
        }
        else if (always_add_sign) {
            *p++ = '+';
        }
        if (digits[0] == 'i' || digits[0] == 'I') {
            strncpy(p, float_strings[OFS_INF], 3);
            p += 3;

            if (type)
                *type = Py_DTST_INFINITE;
        }
        else if (digits[0] == 'n' || digits[0] == 'N') {
            strncpy(p, float_strings[OFS_NAN], 3);
            p += 3;

            if (type)
                *type = Py_DTST_NAN;
        }
        else {
            /* shouldn't get here: Gay's code should always return
               something starting with a digit, an 'I',  or 'N' */
            strncpy(p, "ERR", 3);
            /* p += 3; */
            assert(0);
        }
        goto exit;
    }

    /* The result must be finite (not inf or nan). */
    if (type)
        *type = Py_DTST_FINITE;


    /* We got digits back, format them.  We may need to pad 'digits'
       either on the left or right (or both) with extra zeros, so in
       general the resulting string has the form

         [<sign>]<zeros><digits><zeros>[<exponent>]

       where either of the <zeros> pieces could be empty, and there's a
       decimal point that could appear either in <digits> or in the
       leading or trailing <zeros>.

       Imagine an infinite 'virtual' string vdigits, consisting of the
       string 'digits' (starting at index 0) padded on both the left and
       right with infinite strings of zeros.  We want to output a slice

         vdigits[vdigits_start : vdigits_end]

       of this virtual string.  Thus if vdigits_start < 0 then we'll end
       up producing some leading zeros; if vdigits_end > digits_len there
       will be trailing zeros in the output.  The next section of code
       determines whether to use an exponent or not, figures out the
       position 'decpt' of the decimal point, and computes 'vdigits_start'
       and 'vdigits_end'. */
    vdigits_end = digits_len;
    switch (format_code) {
    case 'e':
        use_exp = 1;
        vdigits_end = precision;
        break;
    case 'f':
        vdigits_end = decpt + precision;
        break;
    case 'g':
        if (decpt <= -4 || decpt >
            (add_dot_0_if_integer ? precision-1 : precision))
            use_exp = 1;
        if (use_alt_formatting)
            vdigits_end = precision;
        break;
    case 'r':
        /* convert to exponential format at 1e16.  We used to convert
           at 1e17, but that gives odd-looking results for some values
           when a 16-digit 'shortest' repr is padded with bogus zeros.
           For example, repr(2e16+8) would give 20000000000000010.0;
           the true value is 20000000000000008.0. */
        if (decpt <= -4 || decpt > 16)
            use_exp = 1;
        break;
    default:
        PyErr_BadInternalCall();
        goto exit;
    }

    /* if using an exponent, reset decimal point position to 1 and adjust
       exponent accordingly.*/
    if (use_exp) {
        exp = (int)decpt - 1;
        decpt = 1;
    }
    /* ensure vdigits_start < decpt <= vdigits_end, or vdigits_start <
       decpt < vdigits_end if add_dot_0_if_integer and no exponent */
    vdigits_start = decpt <= 0 ? decpt-1 : 0;
    if (!use_exp && add_dot_0_if_integer)
        vdigits_end = vdigits_end > decpt ? vdigits_end : decpt + 1;
    else
        vdigits_end = vdigits_end > decpt ? vdigits_end : decpt;

    /* double check inequalities */
    assert(vdigits_start <= 0 &&
           0 <= digits_len &&
           digits_len <= vdigits_end);
    /* decimal point should be in (vdigits_start, vdigits_end] */
    assert(vdigits_start < decpt && decpt <= vdigits_end);

    /* Compute an upper bound how much memory we need. This might be a few
       chars too long, but no big deal. */
    bufsize =
        /* sign, decimal point and trailing 0 byte */
        3 +

        /* total digit count (including zero padding on both sides) */
        (vdigits_end - vdigits_start) +

        /* exponent "e+100", max 3 numerical digits */
        (use_exp ? 5 : 0);

    /* Now allocate the memory and initialize p to point to the start of
       it. */
    buf = (char *)PyMem_Malloc(bufsize);
    if (buf == NULL) {
        PyErr_NoMemory();
        goto exit;
    }
    p = buf;

    /* Add a negative sign if negative, and a plus sign if non-negative
       and always_add_sign is true. */
    if (sign == 1)
        *p++ = '-';
    else if (always_add_sign)
        *p++ = '+';

    /* note that exactly one of the three 'if' conditions is true,
       so we include exactly one decimal point */
    /* Zero padding on left of digit string */
    if (decpt <= 0) {
        memset(p, '0', decpt-vdigits_start);
        p += decpt - vdigits_start;
        *p++ = '.';
        memset(p, '0', 0-decpt);
        p += 0-decpt;
    }
    else {
        memset(p, '0', 0-vdigits_start);
        p += 0 - vdigits_start;
    }

    /* Digits, with included decimal point */
    if (0 < decpt && decpt <= digits_len) {
        strncpy(p, digits, decpt-0);
        p += decpt-0;
        *p++ = '.';
        strncpy(p, digits+decpt, digits_len-decpt);
        p += digits_len-decpt;
    }
    else {
        strncpy(p, digits, digits_len);
        p += digits_len;
    }

    /* And zeros on the right */
    if (digits_len < decpt) {
        memset(p, '0', decpt-digits_len);
        p += decpt-digits_len;
        *p++ = '.';
        memset(p, '0', vdigits_end-decpt);
        p += vdigits_end-decpt;
    }
    else {
        memset(p, '0', vdigits_end-digits_len);
        p += vdigits_end-digits_len;
    }

    /* Delete a trailing decimal pt unless using alternative formatting. */
    if (p[-1] == '.' && !use_alt_formatting)
        p--;

    /* Now that we've done zero padding, add an exponent if needed. */
    if (use_exp) {
        *p++ = float_strings[OFS_E][0];
        exp_len = sprintf(p, "%+.02d", exp);
        p += exp_len;
    }
  exit:
    if (buf) {
        *p = '\0';
        /* It's too late if this fails, as we've already stepped on
           memory that isn't ours. But it's an okay debugging test. */
        assert(p-buf < bufsize);
    }
    if (digits)
        _Py_dg_freedtoa(digits);

    return buf;
}


PyAPI_FUNC(char *) PyOS_double_to_string(double val,
                                         char format_code,
                                         int precision,
                                         int flags,
                                         int *type)
{
    char **float_strings = lc_float_strings;
    int mode;

    /* Validate format_code, and map upper and lower case. Compute the
       mode and make any adjustments as needed. */
    switch (format_code) {
    /* exponent */
    case 'E':
        float_strings = uc_float_strings;
        format_code = 'e';
        /* Fall through. */
    case 'e':
        mode = 2;
        precision++;
        break;

    /* fixed */
    case 'F':
        float_strings = uc_float_strings;
        format_code = 'f';
        /* Fall through. */
    case 'f':
        mode = 3;
        break;

    /* general */
    case 'G':
        float_strings = uc_float_strings;
        format_code = 'g';
        /* Fall through. */
    case 'g':
        mode = 2;
        /* precision 0 makes no sense for 'g' format; interpret as 1 */
        if (precision == 0)
            precision = 1;
        break;

    /* repr format */
    case 'r':
        mode = 0;
        /* Supplied precision is unused, must be 0. */
        if (precision != 0) {
            PyErr_BadInternalCall();
            return NULL;
        }
        break;

    default:
        PyErr_BadInternalCall();
        return NULL;
    }

    return format_float_short(val, format_code, mode, precision,
                              flags & Py_DTSF_SIGN,
                              flags & Py_DTSF_ADD_DOT_0,
                              flags & Py_DTSF_ALT,
                              float_strings, type);
}
#endif /* ifdef PY_NO_SHORT_FLOAT_REPR */
7' href='#n4467'>4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673
/* Execute compiled code */

/* XXX TO DO:
   XXX speed up searching for keywords by using a dictionary
   XXX document it!
   */

/* enable more aggressive intra-module optimizations, where available */
/* affects both release and debug builds - see bpo-43271 */
#define PY_LOCAL_AGGRESSIVE

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_call.h"          // _PyObject_FastCallDictTstate()
#include "pycore_ceval.h"         // _PyEval_SignalAsyncExc()
#include "pycore_code.h"
#include "pycore_function.h"
#include "pycore_initconfig.h"    // _PyStatus_OK()
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_pyerrors.h"      // _PyErr_Fetch()
#include "pycore_pylifecycle.h"   // _PyErr_Print()
#include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_sysmodule.h"     // _PySys_Audit()
#include "pycore_tuple.h"         // _PyTuple_ITEMS()

#include "code.h"
#include "pycore_dict.h"
#include "dictobject.h"
#include "frameobject.h"
#include "pycore_frame.h"
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "structmember.h"         // struct PyMemberDef, T_OFFSET_EX

#include <ctype.h>
#include <stdbool.h>

#ifdef Py_DEBUG
/* For debugging the interpreter: */
#define LLTRACE  1      /* Low-level trace feature */
#define CHECKEXC 1      /* Double-check exception checking */
#endif

#if !defined(Py_BUILD_CORE)
#  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
#endif

_Py_IDENTIFIER(__name__);

/* Forward declarations */
static PyObject *trace_call_function(
    PyThreadState *tstate, PyObject *callable, PyObject **stack,
    Py_ssize_t oparg, PyObject *kwnames);
static PyObject * do_call_core(
    PyThreadState *tstate, PyObject *func,
    PyObject *callargs, PyObject *kwdict, int use_tracing);

#ifdef LLTRACE
static int lltrace;
static int prtrace(PyThreadState *, PyObject *, const char *);
static void lltrace_instruction(InterpreterFrame *frame, int opcode, int oparg)
{
    if (HAS_ARG(opcode)) {
        printf("%d: %d, %d\n",
                frame->f_lasti, opcode, oparg);
    }
    else {
        printf("%d: %d\n",
                frame->f_lasti, opcode);
    }
}
#endif
static int call_trace(Py_tracefunc, PyObject *,
                      PyThreadState *, InterpreterFrame *,
                      int, PyObject *);
static int call_trace_protected(Py_tracefunc, PyObject *,
                                PyThreadState *, InterpreterFrame *,
                                int, PyObject *);
static void call_exc_trace(Py_tracefunc, PyObject *,
                           PyThreadState *, InterpreterFrame *);
static int maybe_call_line_trace(Py_tracefunc, PyObject *,
                                 PyThreadState *, InterpreterFrame *, int);
static void maybe_dtrace_line(InterpreterFrame *, PyTraceInfo *, int);
static void dtrace_function_entry(InterpreterFrame *);
static void dtrace_function_return(InterpreterFrame *);

static PyObject * import_name(PyThreadState *, InterpreterFrame *,
                              PyObject *, PyObject *, PyObject *);
static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);
static int import_all_from(PyThreadState *, PyObject *, PyObject *);
static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);
static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);
static int check_except_type_valid(PyThreadState *tstate, PyObject* right);
static int check_except_star_type_valid(PyThreadState *tstate, PyObject* right);
static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);
static void format_awaitable_error(PyThreadState *, PyTypeObject *, int, int);
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static InterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
                        PyObject *locals, PyObject* const* args,
                        size_t argcount, PyObject *kwnames);
static void
_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame *frame);

#define NAME_ERROR_MSG \
    "name '%.200s' is not defined"
#define UNBOUNDLOCAL_ERROR_MSG \
    "cannot access local variable '%s' where it is not associated with a value"
#define UNBOUNDFREE_ERROR_MSG \
    "cannot access free variable '%s' where it is not associated with a" \
    " value in enclosing scope"

/* Dynamic execution profile */
#ifdef DYNAMIC_EXECUTION_PROFILE
#ifdef DXPAIRS
static long dxpairs[257][256];
#define dxp dxpairs[256]
#else
static long dxp[256];
#endif
#endif

#ifndef NDEBUG
/* Ensure that tstate is valid: sanity check for PyEval_AcquireThread() and
   PyEval_RestoreThread(). Detect if tstate memory was freed. It can happen
   when a thread continues to run after Python finalization, especially
   daemon threads. */
static int
is_tstate_valid(PyThreadState *tstate)
{
    assert(!_PyMem_IsPtrFreed(tstate));
    assert(!_PyMem_IsPtrFreed(tstate->interp));
    return 1;
}
#endif


/* This can set eval_breaker to 0 even though gil_drop_request became
   1.  We believe this is all right because the eval loop will release
   the GIL eventually anyway. */
static inline void
COMPUTE_EVAL_BREAKER(PyInterpreterState *interp,
                     struct _ceval_runtime_state *ceval,
                     struct _ceval_state *ceval2)
{
    _Py_atomic_store_relaxed(&ceval2->eval_breaker,
        _Py_atomic_load_relaxed(&ceval2->gil_drop_request)
        | (_Py_atomic_load_relaxed(&ceval->signals_pending)
           && _Py_ThreadCanHandleSignals(interp))
        | (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)
           && _Py_ThreadCanHandlePendingCalls())
        | ceval2->pending.async_exc);
}


static inline void
SET_GIL_DROP_REQUEST(PyInterpreterState *interp)
{
    struct _ceval_state *ceval2 = &interp->ceval;
    _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 1);
    _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
}


static inline void
RESET_GIL_DROP_REQUEST(PyInterpreterState *interp)
{
    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
    struct _ceval_state *ceval2 = &interp->ceval;
    _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 0);
    COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}


static inline void
SIGNAL_PENDING_CALLS(PyInterpreterState *interp)
{
    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
    struct _ceval_state *ceval2 = &interp->ceval;
    _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 1);
    COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}


static inline void
UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp)
{
    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
    struct _ceval_state *ceval2 = &interp->ceval;
    _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0);
    COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}


static inline void
SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp, int force)
{
    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
    struct _ceval_state *ceval2 = &interp->ceval;
    _Py_atomic_store_relaxed(&ceval->signals_pending, 1);
    if (force) {
        _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
    }
    else {
        /* eval_breaker is not set to 1 if thread_can_handle_signals() is false */
        COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
    }
}


static inline void
UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp)
{
    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
    struct _ceval_state *ceval2 = &interp->ceval;
    _Py_atomic_store_relaxed(&ceval->signals_pending, 0);
    COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}


static inline void
SIGNAL_ASYNC_EXC(PyInterpreterState *interp)
{
    struct _ceval_state *ceval2 = &interp->ceval;
    ceval2->pending.async_exc = 1;
    _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1);
}


static inline void
UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp)
{
    struct _ceval_runtime_state *ceval = &interp->runtime->ceval;
    struct _ceval_state *ceval2 = &interp->ceval;
    ceval2->pending.async_exc = 0;
    COMPUTE_EVAL_BREAKER(interp, ceval, ceval2);
}


#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif
#include "ceval_gil.h"

void _Py_NO_RETURN
_Py_FatalError_TstateNULL(const char *func)
{
    _Py_FatalErrorFunc(func,
                       "the function must be called with the GIL held, "
                       "but the GIL is released "
                       "(the current Python thread state is NULL)");
}

#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
int
_PyEval_ThreadsInitialized(PyInterpreterState *interp)
{
    return gil_created(&interp->ceval.gil);
}

int
PyEval_ThreadsInitialized(void)
{
    // Fatal error if there is no current interpreter
    PyInterpreterState *interp = PyInterpreterState_Get();
    return _PyEval_ThreadsInitialized(interp);
}
#else
int
_PyEval_ThreadsInitialized(_PyRuntimeState *runtime)
{
    return gil_created(&runtime->ceval.gil);
}

int
PyEval_ThreadsInitialized(void)
{
    _PyRuntimeState *runtime = &_PyRuntime;
    return _PyEval_ThreadsInitialized(runtime);
}
#endif

PyStatus
_PyEval_InitGIL(PyThreadState *tstate)
{
#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    if (!_Py_IsMainInterpreter(tstate->interp)) {
        /* Currently, the GIL is shared by all interpreters,
           and only the main interpreter is responsible to create
           and destroy it. */
        return _PyStatus_OK();
    }
#endif

#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
#else
    struct _gil_runtime_state *gil = &tstate->interp->runtime->ceval.gil;
#endif
    assert(!gil_created(gil));

    PyThread_init_thread();
    create_gil(gil);

    take_gil(tstate);

    assert(gil_created(gil));
    return _PyStatus_OK();
}

void
_PyEval_FiniGIL(PyInterpreterState *interp)
{
#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    if (!_Py_IsMainInterpreter(interp)) {
        /* Currently, the GIL is shared by all interpreters,
           and only the main interpreter is responsible to create
           and destroy it. */
        return;
    }
#endif

#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    struct _gil_runtime_state *gil = &interp->ceval.gil;
#else
    struct _gil_runtime_state *gil = &interp->runtime->ceval.gil;
#endif
    if (!gil_created(gil)) {
        /* First Py_InitializeFromConfig() call: the GIL doesn't exist
           yet: do nothing. */
        return;
    }

    destroy_gil(gil);
    assert(!gil_created(gil));
}

void
PyEval_InitThreads(void)
{
    /* Do nothing: kept for backward compatibility */
}

void
_PyEval_Fini(void)
{
#ifdef Py_STATS
    _Py_PrintSpecializationStats(1);
#endif
}

void
PyEval_AcquireLock(void)
{
    _PyRuntimeState *runtime = &_PyRuntime;
    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
    _Py_EnsureTstateNotNULL(tstate);

    take_gil(tstate);
}

void
PyEval_ReleaseLock(void)
{
    _PyRuntimeState *runtime = &_PyRuntime;
    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
    /* This function must succeed when the current thread state is NULL.
       We therefore avoid PyThreadState_Get() which dumps a fatal error
       in debug mode. */
    struct _ceval_runtime_state *ceval = &runtime->ceval;
    struct _ceval_state *ceval2 = &tstate->interp->ceval;
    drop_gil(ceval, ceval2, tstate);
}

void
_PyEval_ReleaseLock(PyThreadState *tstate)
{
    struct _ceval_runtime_state *ceval = &tstate->interp->runtime->ceval;
    struct _ceval_state *ceval2 = &tstate->interp->ceval;
    drop_gil(ceval, ceval2, tstate);
}

void
PyEval_AcquireThread(PyThreadState *tstate)
{
    _Py_EnsureTstateNotNULL(tstate);

    take_gil(tstate);

    struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    (void)_PyThreadState_Swap(gilstate, tstate);
#else
    if (_PyThreadState_Swap(gilstate, tstate) != NULL) {
        Py_FatalError("non-NULL old thread state");
    }
#endif
}

void
PyEval_ReleaseThread(PyThreadState *tstate)
{
    assert(is_tstate_valid(tstate));

    _PyRuntimeState *runtime = tstate->interp->runtime;
    PyThreadState *new_tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
    if (new_tstate != tstate) {
        Py_FatalError("wrong thread state");
    }
    struct _ceval_runtime_state *ceval = &runtime->ceval;
    struct _ceval_state *ceval2 = &tstate->interp->ceval;
    drop_gil(ceval, ceval2, tstate);
}

#ifdef HAVE_FORK
/* This function is called from PyOS_AfterFork_Child to destroy all threads
   which are not running in the child process, and clear internal locks
   which might be held by those threads. */
PyStatus
_PyEval_ReInitThreads(PyThreadState *tstate)
{
    _PyRuntimeState *runtime = tstate->interp->runtime;

#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    struct _gil_runtime_state *gil = &tstate->interp->ceval.gil;
#else
    struct _gil_runtime_state *gil = &runtime->ceval.gil;
#endif
    if (!gil_created(gil)) {
        return _PyStatus_OK();
    }
    recreate_gil(gil);

    take_gil(tstate);

    struct _pending_calls *pending = &tstate->interp->ceval.pending;
    if (_PyThread_at_fork_reinit(&pending->lock) < 0) {
        return _PyStatus_ERR("Can't reinitialize pending calls lock");
    }

    /* Destroy all threads except the current one */
    _PyThreadState_DeleteExcept(runtime, tstate);
    return _PyStatus_OK();
}
#endif

/* This function is used to signal that async exceptions are waiting to be
   raised. */

void
_PyEval_SignalAsyncExc(PyInterpreterState *interp)
{
    SIGNAL_ASYNC_EXC(interp);
}

PyThreadState *
PyEval_SaveThread(void)
{
    _PyRuntimeState *runtime = &_PyRuntime;
#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    PyThreadState *old_tstate = _PyThreadState_GET();
    PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, old_tstate);
#else
    PyThreadState *tstate = _PyThreadState_Swap(&runtime->gilstate, NULL);
#endif
    _Py_EnsureTstateNotNULL(tstate);

    struct _ceval_runtime_state *ceval = &runtime->ceval;
    struct _ceval_state *ceval2 = &tstate->interp->ceval;
#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    assert(gil_created(&ceval2->gil));
#else
    assert(gil_created(&ceval->gil));
#endif
    drop_gil(ceval, ceval2, tstate);
    return tstate;
}

void
PyEval_RestoreThread(PyThreadState *tstate)
{
    _Py_EnsureTstateNotNULL(tstate);

    take_gil(tstate);

    struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate;
    _PyThreadState_Swap(gilstate, tstate);
}


/* Mechanism whereby asynchronously executing callbacks (e.g. UNIX
   signal handlers or Mac I/O completion routines) can schedule calls
   to a function to be called synchronously.
   The synchronous function is called with one void* argument.
   It should return 0 for success or -1 for failure -- failure should
   be accompanied by an exception.

   If registry succeeds, the registry function returns 0; if it fails
   (e.g. due to too many pending calls) it returns -1 (without setting
   an exception condition).

   Note that because registry may occur from within signal handlers,
   or other asynchronous events, calling malloc() is unsafe!

   Any thread can schedule pending calls, but only the main thread
   will execute them.
   There is no facility to schedule calls to a particular thread, but
   that should be easy to change, should that ever be required.  In
   that case, the static variables here should go into the python
   threadstate.
*/

void
_PyEval_SignalReceived(PyInterpreterState *interp)
{
#ifdef MS_WINDOWS
    // bpo-42296: On Windows, _PyEval_SignalReceived() is called from a signal
    // handler which can run in a thread different than the Python thread, in
    // which case _Py_ThreadCanHandleSignals() is wrong. Ignore
    // _Py_ThreadCanHandleSignals() and always set eval_breaker to 1.
    //
    // The next eval_frame_handle_pending() call will call
    // _Py_ThreadCanHandleSignals() to recompute eval_breaker.
    int force = 1;
#else
    int force = 0;
#endif
    /* bpo-30703: Function called when the C signal handler of Python gets a
       signal. We cannot queue a callback using _PyEval_AddPendingCall() since
       that function is not async-signal-safe. */
    SIGNAL_PENDING_SIGNALS(interp, force);
}

/* Push one item onto the queue while holding the lock. */
static int
_push_pending_call(struct _pending_calls *pending,
                   int (*func)(void *), void *arg)
{
    int i = pending->last;
    int j = (i + 1) % NPENDINGCALLS;
    if (j == pending->first) {
        return -1; /* Queue full */
    }
    pending->calls[i].func = func;
    pending->calls[i].arg = arg;
    pending->last = j;
    return 0;
}

/* Pop one item off the queue while holding the lock. */
static void
_pop_pending_call(struct _pending_calls *pending,
                  int (**func)(void *), void **arg)
{
    int i = pending->first;
    if (i == pending->last) {
        return; /* Queue empty */
    }

    *func = pending->calls[i].func;
    *arg = pending->calls[i].arg;
    pending->first = (i + 1) % NPENDINGCALLS;
}

/* This implementation is thread-safe.  It allows
   scheduling to be made from any thread, and even from an executing
   callback.
 */

int
_PyEval_AddPendingCall(PyInterpreterState *interp,
                       int (*func)(void *), void *arg)
{
    struct _pending_calls *pending = &interp->ceval.pending;

    /* Ensure that _PyEval_InitPendingCalls() was called
       and that _PyEval_FiniPendingCalls() is not called yet. */
    assert(pending->lock != NULL);

    PyThread_acquire_lock(pending->lock, WAIT_LOCK);
    int result = _push_pending_call(pending, func, arg);
    PyThread_release_lock(pending->lock);

    /* signal main loop */
    SIGNAL_PENDING_CALLS(interp);
    return result;
}

int
Py_AddPendingCall(int (*func)(void *), void *arg)
{
    /* Best-effort to support subinterpreters and calls with the GIL released.

       First attempt _PyThreadState_GET() since it supports subinterpreters.

       If the GIL is released, _PyThreadState_GET() returns NULL . In this
       case, use PyGILState_GetThisThreadState() which works even if the GIL
       is released.

       Sadly, PyGILState_GetThisThreadState() doesn't support subinterpreters:
       see bpo-10915 and bpo-15751.

       Py_AddPendingCall() doesn't require the caller to hold the GIL. */
    PyThreadState *tstate = _PyThreadState_GET();
    if (tstate == NULL) {
        tstate = PyGILState_GetThisThreadState();
    }

    PyInterpreterState *interp;
    if (tstate != NULL) {
        interp = tstate->interp;
    }
    else {
        /* Last resort: use the main interpreter */
        interp = _PyRuntime.interpreters.main;
    }
    return _PyEval_AddPendingCall(interp, func, arg);
}

static int
handle_signals(PyThreadState *tstate)
{
    assert(is_tstate_valid(tstate));
    if (!_Py_ThreadCanHandleSignals(tstate->interp)) {
        return 0;
    }

    UNSIGNAL_PENDING_SIGNALS(tstate->interp);
    if (_PyErr_CheckSignalsTstate(tstate) < 0) {
        /* On failure, re-schedule a call to handle_signals(). */
        SIGNAL_PENDING_SIGNALS(tstate->interp, 0);
        return -1;
    }
    return 0;
}

static int
make_pending_calls(PyInterpreterState *interp)
{
    /* only execute pending calls on main thread */
    if (!_Py_ThreadCanHandlePendingCalls()) {
        return 0;
    }

    /* don't perform recursive pending calls */
    static int busy = 0;
    if (busy) {
        return 0;
    }
    busy = 1;

    /* unsignal before starting to call callbacks, so that any callback
       added in-between re-signals */
    UNSIGNAL_PENDING_CALLS(interp);
    int res = 0;

    /* perform a bounded number of calls, in case of recursion */
    struct _pending_calls *pending = &interp->ceval.pending;
    for (int i=0; i<NPENDINGCALLS; i++) {
        int (*func)(void *) = NULL;
        void *arg = NULL;

        /* pop one item off the queue while holding the lock */
        PyThread_acquire_lock(pending->lock, WAIT_LOCK);
        _pop_pending_call(pending, &func, &arg);
        PyThread_release_lock(pending->lock);

        /* having released the lock, perform the callback */
        if (func == NULL) {
            break;
        }
        res = func(arg);
        if (res) {
            goto error;
        }
    }

    busy = 0;
    return res;

error:
    busy = 0;
    SIGNAL_PENDING_CALLS(interp);
    return res;
}

void
_Py_FinishPendingCalls(PyThreadState *tstate)
{
    assert(PyGILState_Check());
    assert(is_tstate_valid(tstate));

    struct _pending_calls *pending = &tstate->interp->ceval.pending;

    if (!_Py_atomic_load_relaxed(&(pending->calls_to_do))) {
        return;
    }

    if (make_pending_calls(tstate->interp) < 0) {
        PyObject *exc, *val, *tb;
        _PyErr_Fetch(tstate, &exc, &val, &tb);
        PyErr_BadInternalCall();
        _PyErr_ChainExceptions(exc, val, tb);
        _PyErr_Print(tstate);
    }
}

/* Py_MakePendingCalls() is a simple wrapper for the sake
   of backward-compatibility. */
int
Py_MakePendingCalls(void)
{
    assert(PyGILState_Check());

    PyThreadState *tstate = _PyThreadState_GET();
    assert(is_tstate_valid(tstate));

    /* Python signal handler doesn't really queue a callback: it only signals
       that a signal was received, see _PyEval_SignalReceived(). */
    int res = handle_signals(tstate);
    if (res != 0) {
        return res;
    }

    res = make_pending_calls(tstate->interp);
    if (res != 0) {
        return res;
    }

    return 0;
}

/* The interpreter's recursion limit */

#ifndef Py_DEFAULT_RECURSION_LIMIT
#  define Py_DEFAULT_RECURSION_LIMIT 1000
#endif

void
_PyEval_InitRuntimeState(struct _ceval_runtime_state *ceval)
{
#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    _gil_initialize(&ceval->gil);
#endif
}

void
_PyEval_InitState(struct _ceval_state *ceval, PyThread_type_lock pending_lock)
{
    ceval->recursion_limit = Py_DEFAULT_RECURSION_LIMIT;

    struct _pending_calls *pending = &ceval->pending;
    assert(pending->lock == NULL);

    pending->lock = pending_lock;

#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
    _gil_initialize(&ceval->gil);
#endif
}

void
_PyEval_FiniState(struct _ceval_state *ceval)
{
    struct _pending_calls *pending = &ceval->pending;
    if (pending->lock != NULL) {
        PyThread_free_lock(pending->lock);
        pending->lock = NULL;
    }
}

int
Py_GetRecursionLimit(void)
{
    PyInterpreterState *interp = _PyInterpreterState_GET();
    return interp->ceval.recursion_limit;
}

void
Py_SetRecursionLimit(int new_limit)
{
    PyInterpreterState *interp = _PyInterpreterState_GET();
    interp->ceval.recursion_limit = new_limit;
    for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) {
        int depth = p->recursion_limit - p->recursion_remaining;
        p->recursion_limit = new_limit;
        p->recursion_remaining = new_limit - depth;
    }
}

/* The function _Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall()
   if the recursion_depth reaches recursion_limit. */
int
_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
{
    /* Check against global limit first. */
    int depth = tstate->recursion_limit - tstate->recursion_remaining;
    if (depth < tstate->interp->ceval.recursion_limit) {
        tstate->recursion_limit = tstate->interp->ceval.recursion_limit;
        tstate->recursion_remaining = tstate->recursion_limit - depth;
        assert(tstate->recursion_remaining > 0);
        return 0;
    }
#ifdef USE_STACKCHECK
    if (PyOS_CheckStack()) {
        ++tstate->recursion_remaining;
        _PyErr_SetString(tstate, PyExc_MemoryError, "Stack overflow");
        return -1;
    }
#endif
    if (tstate->recursion_headroom) {
        if (tstate->recursion_remaining < -50) {
            /* Overflowing while handling an overflow. Give up. */
            Py_FatalError("Cannot recover from stack overflow.");
        }
    }
    else {
        if (tstate->recursion_remaining <= 0) {
            tstate->recursion_headroom++;
            _PyErr_Format(tstate, PyExc_RecursionError,
                        "maximum recursion depth exceeded%s",
                        where);
            tstate->recursion_headroom--;
            ++tstate->recursion_remaining;
            return -1;
        }
    }
    return 0;
}


static const binaryfunc binary_ops[] = {
    [NB_ADD] = PyNumber_Add,
    [NB_AND] = PyNumber_And,
    [NB_FLOOR_DIVIDE] = PyNumber_FloorDivide,
    [NB_LSHIFT] = PyNumber_Lshift,
    [NB_MATRIX_MULTIPLY] = PyNumber_MatrixMultiply,
    [NB_MULTIPLY] = PyNumber_Multiply,
    [NB_REMAINDER] = PyNumber_Remainder,
    [NB_OR] = PyNumber_Or,
    [NB_POWER] = _PyNumber_PowerNoMod,
    [NB_RSHIFT] = PyNumber_Rshift,
    [NB_SUBTRACT] = PyNumber_Subtract,
    [NB_TRUE_DIVIDE] = PyNumber_TrueDivide,
    [NB_XOR] = PyNumber_Xor,
    [NB_INPLACE_ADD] = PyNumber_InPlaceAdd,
    [NB_INPLACE_AND] = PyNumber_InPlaceAnd,
    [NB_INPLACE_FLOOR_DIVIDE] = PyNumber_InPlaceFloorDivide,
    [NB_INPLACE_LSHIFT] = PyNumber_InPlaceLshift,
    [NB_INPLACE_MATRIX_MULTIPLY] = PyNumber_InPlaceMatrixMultiply,
    [NB_INPLACE_MULTIPLY] = PyNumber_InPlaceMultiply,
    [NB_INPLACE_REMAINDER] = PyNumber_InPlaceRemainder,
    [NB_INPLACE_OR] = PyNumber_InPlaceOr,
    [NB_INPLACE_POWER] = _PyNumber_InPlacePowerNoMod,
    [NB_INPLACE_RSHIFT] = PyNumber_InPlaceRshift,
    [NB_INPLACE_SUBTRACT] = PyNumber_InPlaceSubtract,
    [NB_INPLACE_TRUE_DIVIDE] = PyNumber_InPlaceTrueDivide,
    [NB_INPLACE_XOR] = PyNumber_InPlaceXor,
};


// PEP 634: Structural Pattern Matching


// Return a tuple of values corresponding to keys, with error checks for
// duplicate/missing keys.
static PyObject*
match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys)
{
    assert(PyTuple_CheckExact(keys));
    Py_ssize_t nkeys = PyTuple_GET_SIZE(keys);
    if (!nkeys) {
        // No keys means no items.
        return PyTuple_New(0);
    }
    PyObject *seen = NULL;
    PyObject *dummy = NULL;
    PyObject *values = NULL;
    PyObject *get_name = NULL;
    PyObject *get = NULL;
    // We use the two argument form of map.get(key, default) for two reasons:
    // - Atomically check for a key and get its value without error handling.
    // - Don't cause key creation or resizing in dict subclasses like
    //   collections.defaultdict that define __missing__ (or similar).
    _Py_IDENTIFIER(get);
    get_name = _PyUnicode_FromId(&PyId_get); // borrowed
    if (get_name == NULL) {
        return NULL;
    }
    int meth_found = _PyObject_GetMethod(map, get_name, &get);
    if (get == NULL) {
        goto fail;
    }
    seen = PySet_New(NULL);
    if (seen == NULL) {
        goto fail;
    }
    // dummy = object()
    dummy = _PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type);
    if (dummy == NULL) {
        goto fail;
    }
    values = PyTuple_New(nkeys);
    if (values == NULL) {
        goto fail;
    }
    for (Py_ssize_t i = 0; i < nkeys; i++) {
        PyObject *key = PyTuple_GET_ITEM(keys, i);
        if (PySet_Contains(seen, key) || PySet_Add(seen, key)) {
            if (!_PyErr_Occurred(tstate)) {
                // Seen it before!
                _PyErr_Format(tstate, PyExc_ValueError,
                              "mapping pattern checks duplicate key (%R)", key);
            }
            goto fail;
        }
        PyObject *args[] = { map, key, dummy };
        PyObject *value = NULL;
        if (meth_found) {
            value = PyObject_Vectorcall(get, args, 3, NULL);
        }
        else {
            value = PyObject_Vectorcall(get, &args[1], 2, NULL);
        }
        if (value == NULL) {
            goto fail;
        }
        if (value == dummy) {
            // key not in map!
            Py_DECREF(value);
            Py_DECREF(values);
            // Return None:
            Py_INCREF(Py_None);
            values = Py_None;
            goto done;
        }
        PyTuple_SET_ITEM(values, i, value);
    }
    // Success:
done:
    Py_DECREF(get);
    Py_DECREF(seen);
    Py_DECREF(dummy);
    return values;
fail:
    Py_XDECREF(get);
    Py_XDECREF(seen);
    Py_XDECREF(dummy);
    Py_XDECREF(values);
    return NULL;
}

// Extract a named attribute from the subject, with additional bookkeeping to
// raise TypeErrors for repeated lookups. On failure, return NULL (with no
// error set). Use _PyErr_Occurred(tstate) to disambiguate.
static PyObject*
match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,
                 PyObject *name, PyObject *seen)
{
    assert(PyUnicode_CheckExact(name));
    assert(PySet_CheckExact(seen));
    if (PySet_Contains(seen, name) || PySet_Add(seen, name)) {
        if (!_PyErr_Occurred(tstate)) {
            // Seen it before!
            _PyErr_Format(tstate, PyExc_TypeError,
                          "%s() got multiple sub-patterns for attribute %R",
                          ((PyTypeObject*)type)->tp_name, name);
        }
        return NULL;
    }
    PyObject *attr = PyObject_GetAttr(subject, name);
    if (attr == NULL && _PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
        _PyErr_Clear(tstate);
    }
    return attr;
}

// On success (match), return a tuple of extracted attributes. On failure (no
// match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.
static PyObject*
match_class(PyThreadState *tstate, PyObject *subject, PyObject *type,
            Py_ssize_t nargs, PyObject *kwargs)
{
    if (!PyType_Check(type)) {
        const char *e = "called match pattern must be a type";
        _PyErr_Format(tstate, PyExc_TypeError, e);
        return NULL;
    }
    assert(PyTuple_CheckExact(kwargs));
    // First, an isinstance check:
    if (PyObject_IsInstance(subject, type) <= 0) {
        return NULL;
    }
    // So far so good:
    PyObject *seen = PySet_New(NULL);
    if (seen == NULL) {
        return NULL;
    }
    PyObject *attrs = PyList_New(0);
    if (attrs == NULL) {
        Py_DECREF(seen);
        return NULL;
    }
    // NOTE: From this point on, goto fail on failure:
    PyObject *match_args = NULL;
    // First, the positional subpatterns:
    if (nargs) {
        int match_self = 0;
        match_args = PyObject_GetAttrString(type, "__match_args__");
        if (match_args) {
            if (!PyTuple_CheckExact(match_args)) {
                const char *e = "%s.__match_args__ must be a tuple (got %s)";
                _PyErr_Format(tstate, PyExc_TypeError, e,
                              ((PyTypeObject *)type)->tp_name,
                              Py_TYPE(match_args)->tp_name);
                goto fail;
            }
        }
        else if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
            _PyErr_Clear(tstate);
            // _Py_TPFLAGS_MATCH_SELF is only acknowledged if the type does not
            // define __match_args__. This is natural behavior for subclasses:
            // it's as if __match_args__ is some "magic" value that is lost as
            // soon as they redefine it.
            match_args = PyTuple_New(0);
            match_self = PyType_HasFeature((PyTypeObject*)type,
                                            _Py_TPFLAGS_MATCH_SELF);
        }
        else {
            goto fail;
        }
        assert(PyTuple_CheckExact(match_args));
        Py_ssize_t allowed = match_self ? 1 : PyTuple_GET_SIZE(match_args);
        if (allowed < nargs) {
            const char *plural = (allowed == 1) ? "" : "s";
            _PyErr_Format(tstate, PyExc_TypeError,
                          "%s() accepts %d positional sub-pattern%s (%d given)",
                          ((PyTypeObject*)type)->tp_name,
                          allowed, plural, nargs);
            goto fail;
        }
        if (match_self) {
            // Easy. Copy the subject itself, and move on to kwargs.
            PyList_Append(attrs, subject);
        }
        else {
            for (Py_ssize_t i = 0; i < nargs; i++) {
                PyObject *name = PyTuple_GET_ITEM(match_args, i);
                if (!PyUnicode_CheckExact(name)) {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "__match_args__ elements must be strings "
                                  "(got %s)", Py_TYPE(name)->tp_name);
                    goto fail;
                }
                PyObject *attr = match_class_attr(tstate, subject, type, name,
                                                  seen);
                if (attr == NULL) {
                    goto fail;
                }
                PyList_Append(attrs, attr);
                Py_DECREF(attr);
            }
        }
        Py_CLEAR(match_args);
    }
    // Finally, the keyword subpatterns:
    for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwargs); i++) {
        PyObject *name = PyTuple_GET_ITEM(kwargs, i);
        PyObject *attr = match_class_attr(tstate, subject, type, name, seen);
        if (attr == NULL) {
            goto fail;
        }
        PyList_Append(attrs, attr);
        Py_DECREF(attr);
    }
    Py_SETREF(attrs, PyList_AsTuple(attrs));
    Py_DECREF(seen);
    return attrs;
fail:
    // We really don't care whether an error was raised or not... that's our
    // caller's problem. All we know is that the match failed.
    Py_XDECREF(match_args);
    Py_DECREF(seen);
    Py_DECREF(attrs);
    return NULL;
}


static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);
static PyObject *do_reraise_star(PyObject *excs, PyObject *orig);
static int exception_group_match(
    PyObject* exc_value, PyObject *match_type,
    PyObject **match, PyObject **rest);

static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);

PyObject *
PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
{
    PyThreadState *tstate = _PyThreadState_GET();
    if (locals == NULL) {
        locals = globals;
    }
    PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref
    if (builtins == NULL) {
        return NULL;
    }
    PyFrameConstructor desc = {
        .fc_globals = globals,
        .fc_builtins = builtins,
        .fc_name = ((PyCodeObject *)co)->co_name,
        .fc_qualname = ((PyCodeObject *)co)->co_name,
        .fc_code = co,
        .fc_defaults = NULL,
        .fc_kwdefaults = NULL,
        .fc_closure = NULL
    };
    PyFunctionObject *func = _PyFunction_FromConstructor(&desc);
    if (func == NULL) {
        return NULL;
    }
    PyObject *res = _PyEval_Vector(tstate, func, locals, NULL, 0, NULL);
    Py_DECREF(func);
    return res;
}


/* Interpreter main loop */

PyObject *
PyEval_EvalFrame(PyFrameObject *f)
{
    /* Function kept for backward compatibility */
    PyThreadState *tstate = _PyThreadState_GET();
    return _PyEval_EvalFrame(tstate, f->f_frame, 0);
}

PyObject *
PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
{
    PyThreadState *tstate = _PyThreadState_GET();
    return _PyEval_EvalFrame(tstate, f->f_frame, throwflag);
}


/* Handle signals, pending calls, GIL drop request
   and asynchronous exception */
static int
eval_frame_handle_pending(PyThreadState *tstate)
{
    _PyRuntimeState * const runtime = &_PyRuntime;
    struct _ceval_runtime_state *ceval = &runtime->ceval;

    /* Pending signals */
    if (_Py_atomic_load_relaxed(&ceval->signals_pending)) {
        if (handle_signals(tstate) != 0) {
            return -1;
        }
    }

    /* Pending calls */
    struct _ceval_state *ceval2 = &tstate->interp->ceval;
    if (_Py_atomic_load_relaxed(&ceval2->pending.calls_to_do)) {
        if (make_pending_calls(tstate->interp) != 0) {
            return -1;
        }
    }

    /* GIL drop request */
    if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) {
        /* Give another thread a chance */
        if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) {
            Py_FatalError("tstate mix-up");
        }
        drop_gil(ceval, ceval2, tstate);

        /* Other threads may run now */

        take_gil(tstate);

#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
        (void)_PyThreadState_Swap(&runtime->gilstate, tstate);
#else
        if (_PyThreadState_Swap(&runtime->gilstate, tstate) != NULL) {
            Py_FatalError("orphan tstate");
        }
#endif
    }

    /* Check for asynchronous exception. */
    if (tstate->async_exc != NULL) {
        PyObject *exc = tstate->async_exc;
        tstate->async_exc = NULL;
        UNSIGNAL_ASYNC_EXC(tstate->interp);
        _PyErr_SetNone(tstate, exc);
        Py_DECREF(exc);
        return -1;
    }

#ifdef MS_WINDOWS
    // bpo-42296: On Windows, _PyEval_SignalReceived() can be called in a
    // different thread than the Python thread, in which case
    // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the
    // current Python thread with the correct _Py_ThreadCanHandleSignals()
    // value. It prevents to interrupt the eval loop at every instruction if
    // the current Python thread cannot handle signals (if
    // _Py_ThreadCanHandleSignals() is false).
    COMPUTE_EVAL_BREAKER(tstate->interp, ceval, ceval2);
#endif

    return 0;
}


/* Computed GOTOs, or
       the-optimization-commonly-but-improperly-known-as-"threaded code"
   using gcc's labels-as-values extension
   (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html).

   The traditional bytecode evaluation loop uses a "switch" statement, which
   decent compilers will optimize as a single indirect branch instruction
   combined with a lookup table of jump addresses. However, since the
   indirect jump instruction is shared by all opcodes, the CPU will have a
   hard time making the right prediction for where to jump next (actually,
   it will be always wrong except in the uncommon case of a sequence of
   several identical opcodes).

   "Threaded code" in contrast, uses an explicit jump table and an explicit
   indirect jump instruction at the end of each opcode. Since the jump
   instruction is at a different address for each opcode, the CPU will make a
   separate prediction for each of these instructions, which is equivalent to
   predicting the second opcode of each opcode pair. These predictions have
   a much better chance to turn out valid, especially in small bytecode loops.

   A mispredicted branch on a modern CPU flushes the whole pipeline and
   can cost several CPU cycles (depending on the pipeline depth),
   and potentially many more instructions (depending on the pipeline width).
   A correctly predicted branch, however, is nearly free.

   At the time of this writing, the "threaded code" version is up to 15-20%
   faster than the normal "switch" version, depending on the compiler and the
   CPU architecture.

   We disable the optimization if DYNAMIC_EXECUTION_PROFILE is defined,
   because it would render the measurements invalid.


   NOTE: care must be taken that the compiler doesn't try to "optimize" the
   indirect jumps by sharing them between all opcodes. Such optimizations
   can be disabled on gcc by using the -fno-gcse flag (or possibly
   -fno-crossjumping).
*/

/* Use macros rather than inline functions, to make it as clear as possible
 * to the C compiler that the tracing check is a simple test then branch.
 * We want to be sure that the compiler knows this before it generates
 * the CFG.
 */
#ifdef LLTRACE
#define LLTRACE_INSTR() if (lltrace) { lltrace_instruction(frame, opcode, oparg); }
#else
#define LLTRACE_INSTR() ((void)0)
#endif

#ifdef WITH_DTRACE
#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0)
#else
#define OR_DTRACE_LINE
#endif

#ifdef DYNAMIC_EXECUTION_PROFILE
#undef USE_COMPUTED_GOTOS
#define USE_COMPUTED_GOTOS 0
#endif

#ifdef HAVE_COMPUTED_GOTOS
    #ifndef USE_COMPUTED_GOTOS
    #define USE_COMPUTED_GOTOS 1
    #endif
#else
    #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS
    #error "Computed gotos are not supported on this compiler."
    #endif
    #undef USE_COMPUTED_GOTOS
    #define USE_COMPUTED_GOTOS 0
#endif

#ifdef Py_STATS
#define INSTRUCTION_START(op) frame->f_lasti = INSTR_OFFSET(); next_instr++; OPCODE_EXE_INC(op);
#else
#define INSTRUCTION_START(op) frame->f_lasti = INSTR_OFFSET(); next_instr++
#endif

#if USE_COMPUTED_GOTOS
#define TARGET(op) TARGET_##op: INSTRUCTION_START(op);
#define DISPATCH_GOTO() goto *opcode_targets[opcode]
#else
#define TARGET(op) case op: INSTRUCTION_START(op);
#define DISPATCH_GOTO() goto dispatch_opcode
#endif

/* RECORD_DXPROFILE()  records the dxprofile information, if enabled. Normally a no-op */
#ifdef DYNAMIC_EXECUTION_PROFILE
#ifdef DXPAIRS
#define RECORD_DXPROFILE() \
    do { \
        dxpairs[lastopcode][opcode]++; \
        lastopcode = opcode; \
        dxp[opcode]++; \
    } while (0)
#else
  #define RECORD_DXPROFILE() \
    do { \
        dxp[opcode]++; \
    } while (0)
#endif
#else
#define RECORD_DXPROFILE() ((void)0)
#endif

/* PRE_DISPATCH_GOTO() does lltrace and dxprofile if either is enabled. Normally a no-op */
#ifndef LLTRACE
#ifndef DYNAMIC_EXECUTION_PROFILE
#define PRE_DISPATCH_GOTO() ((void)0)
#endif
#endif
#ifndef PRE_DISPATCH_GOTO
#define PRE_DISPATCH_GOTO() do { LLTRACE_INSTR(); RECORD_DXPROFILE(); } while (0)
#endif

#define NOTRACE_DISPATCH() \
    { \
        NEXTOPARG(); \
        PRE_DISPATCH_GOTO(); \
        DISPATCH_GOTO(); \
    }

/* Do interpreter dispatch accounting for tracing and instrumentation */
#define DISPATCH() \
    { \
        NEXTOPARG(); \
        PRE_DISPATCH_GOTO(); \
        assert(cframe.use_tracing == 0 || cframe.use_tracing == 255); \
        opcode |= cframe.use_tracing OR_DTRACE_LINE; \
        DISPATCH_GOTO(); \
    }

#define CHECK_EVAL_BREAKER() \
    if (_Py_atomic_load_relaxed(eval_breaker)) { \
        goto check_eval_breaker; \
    }


/* Tuple access macros */

#ifndef Py_DEBUG
#define GETITEM(v, i) PyTuple_GET_ITEM((PyTupleObject *)(v), (i))
#else
#define GETITEM(v, i) PyTuple_GetItem((v), (i))
#endif

/* Code access macros */

/* The integer overflow is checked by an assertion below. */
#define INSTR_OFFSET() ((int)(next_instr - first_instr))
#define NEXTOPARG()  do { \
        _Py_CODEUNIT word = *next_instr; \
        opcode = _Py_OPCODE(word); \
        oparg = _Py_OPARG(word); \
    } while (0)
#define JUMPTO(x)       (next_instr = first_instr + (x))
#define JUMPBY(x)       (next_instr += (x))

/* Get opcode and oparg from original instructions, not quickened form. */
#define TRACING_NEXTOPARG() do { \
        _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \
        opcode = _Py_OPCODE(word); \
        oparg = _Py_OPARG(word); \
    } while (0)

/* OpCode prediction macros
    Some opcodes tend to come in pairs thus making it possible to
    predict the second code when the first is run.  For example,
    COMPARE_OP is often followed by POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE.

    Verifying the prediction costs a single high-speed test of a register
    variable against a constant.  If the pairing was good, then the
    processor's own internal branch predication has a high likelihood of
    success, resulting in a nearly zero-overhead transition to the
    next opcode.  A successful prediction saves a trip through the eval-loop
    including its unpredictable switch-case branch.  Combined with the
    processor's internal branch prediction, a successful PREDICT has the
    effect of making the two opcodes run as if they were a single new opcode
    with the bodies combined.

    If collecting opcode statistics, your choices are to either keep the
    predictions turned-on and interpret the results as if some opcodes
    had been combined or turn-off predictions so that the opcode frequency
    counter updates for both opcodes.

    Opcode prediction is disabled with threaded code, since the latter allows
    the CPU to record separate branch prediction information for each
    opcode.

*/

#define PREDICT_ID(op)          PRED_##op

#if defined(DYNAMIC_EXECUTION_PROFILE) || USE_COMPUTED_GOTOS
#define PREDICT(op)             if (0) goto PREDICT_ID(op)
#else
#define PREDICT(op) \
    do { \
        _Py_CODEUNIT word = *next_instr; \
        opcode = _Py_OPCODE(word) | cframe.use_tracing OR_DTRACE_LINE; \
        if (opcode == op) { \
            oparg = _Py_OPARG(word); \
            INSTRUCTION_START(op); \
            goto PREDICT_ID(op); \
        } \
    } while(0)
#endif
#define PREDICTED(op)           PREDICT_ID(op):


/* Stack manipulation macros */

/* The stack can grow at most MAXINT deep, as co_nlocals and
   co_stacksize are ints. */
#define STACK_LEVEL()     ((int)(stack_pointer - _PyFrame_Stackbase(frame)))
#define EMPTY()           (STACK_LEVEL() == 0)
#define TOP()             (stack_pointer[-1])
#define SECOND()          (stack_pointer[-2])
#define THIRD()           (stack_pointer[-3])
#define FOURTH()          (stack_pointer[-4])
#define PEEK(n)           (stack_pointer[-(n)])
#define SET_TOP(v)        (stack_pointer[-1] = (v))
#define SET_SECOND(v)     (stack_pointer[-2] = (v))
#define SET_THIRD(v)      (stack_pointer[-3] = (v))
#define SET_FOURTH(v)     (stack_pointer[-4] = (v))
#define BASIC_STACKADJ(n) (stack_pointer += n)
#define BASIC_PUSH(v)     (*stack_pointer++ = (v))
#define BASIC_POP()       (*--stack_pointer)

#ifdef LLTRACE
#define PUSH(v)         { (void)(BASIC_PUSH(v), \
                          lltrace && prtrace(tstate, TOP(), "push")); \
                          assert(STACK_LEVEL() <= frame->f_code->co_stacksize); }
#define POP()           ((void)(lltrace && prtrace(tstate, TOP(), "pop")), \
                         BASIC_POP())
#define STACK_GROW(n)   do { \
                          assert(n >= 0); \
                          (void)(BASIC_STACKADJ(n), \
                          lltrace && prtrace(tstate, TOP(), "stackadj")); \
                          assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
                        } while (0)
#define STACK_SHRINK(n) do { \
                            assert(n >= 0); \
                            (void)(lltrace && prtrace(tstate, TOP(), "stackadj")); \
                            (void)(BASIC_STACKADJ(-(n))); \
                            assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
                        } while (0)
#define EXT_POP(STACK_POINTER) ((void)(lltrace && \
                                prtrace(tstate, (STACK_POINTER)[-1], "ext_pop")), \
                                *--(STACK_POINTER))
#else
#define PUSH(v)                BASIC_PUSH(v)
#define POP()                  BASIC_POP()
#define STACK_GROW(n)          BASIC_STACKADJ(n)
#define STACK_SHRINK(n)        BASIC_STACKADJ(-(n))
#define EXT_POP(STACK_POINTER) (*--(STACK_POINTER))
#endif

/* Local variable macros */

#define GETLOCAL(i)     (frame->localsplus[i])

/* The SETLOCAL() macro must not DECREF the local variable in-place and
   then store the new value; it must copy the old value to a temporary
   value, then store the new value, and then DECREF the temporary value.
   This is because it is possible that during the DECREF the frame is
   accessed by other code (e.g. a __del__ method or gc.collect()) and the
   variable would be pointing to already-freed memory. */
#define SETLOCAL(i, value)      do { PyObject *tmp = GETLOCAL(i); \
                                     GETLOCAL(i) = value; \
                                     Py_XDECREF(tmp); } while (0)

#define JUMP_TO_INSTRUCTION(op) goto PREDICT_ID(op)

#define GET_CACHE() \
    _GetSpecializedCacheEntryForInstruction(first_instr, INSTR_OFFSET(), oparg)


#define DEOPT_IF(cond, instname) if (cond) { goto instname ## _miss; }

#define UPDATE_PREV_INSTR_OPARG(instr, oparg) ((uint8_t*)(instr))[-1] = (oparg)


#define GLOBALS() frame->f_globals
#define BUILTINS() frame->f_builtins
#define LOCALS() frame->f_locals

/* Shared opcode macros */

// shared by LOAD_ATTR_MODULE and LOAD_METHOD_MODULE
#define LOAD_MODULE_ATTR_OR_METHOD(attr_or_method) \
    SpecializedCacheEntry *caches = GET_CACHE(); \
    _PyAdaptiveEntry *cache0 = &caches[0].adaptive; \
    _PyAttrCache *cache1 = &caches[-1].attr; \
    DEOPT_IF(!PyModule_CheckExact(owner), LOAD_##attr_or_method); \
    PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; \
    assert(dict != NULL); \
    DEOPT_IF(dict->ma_keys->dk_version != cache1->dk_version_or_hint, \
        LOAD_##attr_or_method); \
    assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); \
    assert(cache0->index < dict->ma_keys->dk_nentries); \
    PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + cache0->index; \
    res = ep->me_value; \
    DEOPT_IF(res == NULL, LOAD_##attr_or_method); \
    STAT_INC(LOAD_##attr_or_method, hit); \
    Py_INCREF(res);

#define TRACE_FUNCTION_EXIT() \
    if (cframe.use_tracing) { \
        if (trace_function_exit(tstate, frame, retval)) { \
            Py_DECREF(retval); \
            goto exit_unwind; \
        } \
    }

#define DTRACE_FUNCTION_EXIT() \
    if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \
        dtrace_function_return(frame); \
    }

#define TRACE_FUNCTION_UNWIND()  \
    if (cframe.use_tracing) { \
        /* Since we are already unwinding, \
         * we dont't care if this raises */ \
        trace_function_exit(tstate, frame, NULL); \
    }

#define TRACE_FUNCTION_ENTRY() \
    if (cframe.use_tracing) { \
        if (trace_function_entry(tstate, frame)) { \
            goto exit_unwind; \
        } \
    }

#define DTRACE_FUNCTION_ENTRY()  \
    if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \
        dtrace_function_entry(frame); \
    }



static int
trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
{
    if (tstate->c_tracefunc != NULL) {
        /* tstate->c_tracefunc, if defined, is a
            function that will be called on *every* entry
            to a code block.  Its return value, if not
            None, is a function that will be called at
            the start of each executed line of code.
            (Actually, the function must return itself
            in order to continue tracing.)  The trace
            functions are called with three arguments:
            a pointer to the current frame, a string
            indicating why the function is called, and
            an argument which depends on the situation.
            The global trace function is also called
            whenever an exception is detected. */
        if (call_trace_protected(tstate->c_tracefunc,
                                    tstate->c_traceobj,
                                    tstate, frame,
                                    PyTrace_CALL, Py_None)) {
            /* Trace function raised an error */
            return -1;
        }
    }
    if (tstate->c_profilefunc != NULL) {
        /* Similar for c_profilefunc, except it needn't
            return itself and isn't called for "line" events */
        if (call_trace_protected(tstate->c_profilefunc,
                                    tstate->c_profileobj,
                                    tstate, frame,
                                    PyTrace_CALL, Py_None)) {
            /* Profile function raised an error */
            return -1;
        }
    }
    return 0;
}

static int
trace_function_exit(PyThreadState *tstate, InterpreterFrame *frame, PyObject *retval)
{
    if (tstate->c_tracefunc) {
        if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
                                    tstate, frame, PyTrace_RETURN, retval)) {
            return -1;
        }
    }
    if (tstate->c_profilefunc) {
        if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
                                    tstate, frame, PyTrace_RETURN, retval)) {
            return -1;
        }
    }
    return 0;
}

static PyObject *
make_coro(PyThreadState *tstate, PyFunctionObject *func,
          PyObject *locals,
          PyObject* const* args, size_t argcount,
          PyObject *kwnames);

static int
skip_backwards_over_extended_args(PyCodeObject *code, int offset)
{
    _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code);
    while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) {
        offset--;
    }
    return offset;
}

static InterpreterFrame *
pop_frame(PyThreadState *tstate, InterpreterFrame *frame)
{
    InterpreterFrame *prev_frame = frame->previous;
    _PyEvalFrameClearAndPop(tstate, frame);
    return prev_frame;
}

PyObject* _Py_HOT_FUNCTION
_PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
{
    _Py_EnsureTstateNotNULL(tstate);

#if USE_COMPUTED_GOTOS
/* Import the static jump table */
#include "opcode_targets.h"
#endif

#ifdef DXPAIRS
    int lastopcode = 0;
#endif
    int opcode;        /* Current opcode */
    int oparg;         /* Current opcode argument, if any */
    _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;

    CFrame cframe;

    /* Variables used for making calls */
    PyObject *kwnames;
    int nargs;
    /*
     * It is only between a PRECALL_METHOD instruction and the following instruction,
     * that these two values can be anything other than their defaults. */
    int postcall_shrink = 1;
    int extra_args = 0;
#define RESET_STACK_ADJUST_FOR_CALLS \
    do { \
        postcall_shrink = 1; \
        extra_args = 0; \
    } while (0)
#define STACK_ADJUST_IS_RESET \
    (postcall_shrink == 1 && extra_args == 0)

    /* WARNING: Because the CFrame lives on the C stack,
     * but can be accessed from a heap allocated object (tstate)
     * strict stack discipline must be maintained.
     */
    CFrame *prev_cframe = tstate->cframe;
    cframe.use_tracing = prev_cframe->use_tracing;
    cframe.previous = prev_cframe;
    tstate->cframe = &cframe;

    assert(frame->depth == 0);
    /* Push frame */
    frame->previous = prev_cframe->current_frame;
    cframe.current_frame = frame;

    /* support for generator.throw() */
    if (throwflag) {
        if (_Py_EnterRecursiveCall(tstate, "")) {
            tstate->recursion_remaining--;
            goto exit_unwind;
        }
        TRACE_FUNCTION_ENTRY();
        DTRACE_FUNCTION_ENTRY();
        goto resume_with_error;
    }

    /* Local "register" variables.
     * These are cached values from the frame and code object.  */

    PyObject *names;
    PyObject *consts;
    _Py_CODEUNIT *first_instr;
    _Py_CODEUNIT *next_instr;
    PyObject **stack_pointer;

/* Sets the above local variables from the frame */
#define SET_LOCALS_FROM_FRAME() \
    { \
        PyCodeObject *co = frame->f_code; \
        names = co->co_names; \
        consts = co->co_consts; \
        first_instr = co->co_firstinstr; \
    } \
    assert(frame->f_lasti >= -1); \
    next_instr = first_instr + frame->f_lasti + 1; \
    stack_pointer = _PyFrame_GetStackPointer(frame); \
    /* Set stackdepth to -1. \
        Update when returning or calling trace function. \
        Having stackdepth <= 0 ensures that invalid \
        values are not visible to the cycle GC. \
        We choose -1 rather than 0 to assist debugging. \
        */ \
    frame->stacktop = -1;


start_frame:
    if (_Py_EnterRecursiveCall(tstate, "")) {
        tstate->recursion_remaining--;
        goto exit_unwind;
    }

    assert(tstate->cframe == &cframe);
    assert(frame == cframe.current_frame);

    TRACE_FUNCTION_ENTRY();
    DTRACE_FUNCTION_ENTRY();

    if (_Py_IncrementCountAndMaybeQuicken(frame->f_code) < 0) {
        goto exit_unwind;
    }
    frame->f_state = FRAME_EXECUTING;

resume_frame:
    SET_LOCALS_FROM_FRAME();

#ifdef LLTRACE
    _Py_IDENTIFIER(__ltrace__);
    {
        int r = _PyDict_ContainsId(GLOBALS(), &PyId___ltrace__);
        if (r < 0) {
            goto exit_unwind;
        }
        lltrace = r;
    }
#endif

#ifdef Py_DEBUG
    /* _PyEval_EvalFrameDefault() must not be called with an exception set,
       because it can clear it (directly or indirectly) and so the
       caller loses its exception */
    assert(!_PyErr_Occurred(tstate));
#endif

check_eval_breaker:
    {
        assert(STACK_LEVEL() >= 0); /* else underflow */
        assert(STACK_LEVEL() <= frame->f_code->co_stacksize);  /* else overflow */
        assert(!_PyErr_Occurred(tstate));

        /* Do periodic things.  Doing this every time through
           the loop would add too much overhead, so we do it
           only every Nth instruction.  We also do it if
           ``pending.calls_to_do'' is set, i.e. when an asynchronous
           event needs attention (e.g. a signal handler or
           async I/O handler); see Py_AddPendingCall() and
           Py_MakePendingCalls() above. */

        if (_Py_atomic_load_relaxed(eval_breaker)) {
            opcode = _Py_OPCODE(*next_instr);
            if (opcode != BEFORE_ASYNC_WITH &&
                opcode != SEND &&
                _Py_OPCODE(next_instr[-1]) != SEND) {
                /* Few cases where we skip running signal handlers and other
                   pending calls:
                   - If we're about to enter the 'with:'. It will prevent
                     emitting a resource warning in the common idiom
                     'with open(path) as file:'.
                   - If we're about to enter the 'async with:'.
                   - If we're about to enter the 'try:' of a try/finally (not
                     *very* useful, but might help in some cases and it's
                     traditional)
                   - If we're resuming a chain of nested 'yield from' or
                     'await' calls, then each frame is parked with YIELD_FROM
                     as its next opcode. If the user hit control-C we want to
                     wait until we've reached the innermost frame before
                     running the signal handler and raising KeyboardInterrupt
                     (see bpo-30039).
                */
                if (eval_frame_handle_pending(tstate) != 0) {
                    goto error;
                }
             }
        }

    DISPATCH();

    /* Start instructions */
#if USE_COMPUTED_GOTOS
    {
#else
    dispatch_opcode:
        switch (opcode) {
#endif

        /* BEWARE!
           It is essential that any operation that fails must goto error
           and that all operation that succeed call DISPATCH() ! */

        TARGET(NOP) {
            DISPATCH();
        }

        TARGET(LOAD_CLOSURE) {
            /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
            PyObject *value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
            DISPATCH();
        }

        TARGET(LOAD_FAST) {
            PyObject *value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
            DISPATCH();
        }

        TARGET(LOAD_CONST) {
            PREDICTED(LOAD_CONST);
            PyObject *value = GETITEM(consts, oparg);
            Py_INCREF(value);
            PUSH(value);
            DISPATCH();
        }

        TARGET(STORE_FAST) {
            PREDICTED(STORE_FAST);
            PyObject *value = POP();
            SETLOCAL(oparg, value);
            DISPATCH();
        }

        TARGET(LOAD_FAST__LOAD_FAST) {
            PyObject *value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            NEXTOPARG();
            next_instr++;
            Py_INCREF(value);
            PUSH(value);
            value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
            NOTRACE_DISPATCH();
        }

        TARGET(LOAD_FAST__LOAD_CONST) {
            PyObject *value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            NEXTOPARG();
            next_instr++;
            Py_INCREF(value);
            PUSH(value);
            value = GETITEM(consts, oparg);
            Py_INCREF(value);
            PUSH(value);
            NOTRACE_DISPATCH();
        }

        TARGET(STORE_FAST__LOAD_FAST) {
            PyObject *value = POP();
            SETLOCAL(oparg, value);
            NEXTOPARG();
            next_instr++;
            value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
            NOTRACE_DISPATCH();
        }

        TARGET(STORE_FAST__STORE_FAST) {
            PyObject *value = POP();
            SETLOCAL(oparg, value);
            NEXTOPARG();
            next_instr++;
            value = POP();
            SETLOCAL(oparg, value);
            NOTRACE_DISPATCH();
        }

        TARGET(LOAD_CONST__LOAD_FAST) {
            PyObject *value = GETITEM(consts, oparg);
            NEXTOPARG();
            next_instr++;
            Py_INCREF(value);
            PUSH(value);
            value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
            NOTRACE_DISPATCH();
        }

        TARGET(POP_TOP) {
            PyObject *value = POP();
            Py_DECREF(value);
            DISPATCH();
        }

        TARGET(ROT_TWO) {
            PyObject *top = TOP();
            PyObject *second = SECOND();
            SET_TOP(second);
            SET_SECOND(top);
            DISPATCH();
        }

        TARGET(ROT_THREE) {
            PyObject *top = TOP();
            PyObject *second = SECOND();
            PyObject *third = THIRD();
            SET_TOP(second);
            SET_SECOND(third);
            SET_THIRD(top);
            DISPATCH();
        }

        TARGET(ROT_FOUR) {
            PyObject *top = TOP();
            PyObject *second = SECOND();
            PyObject *third = THIRD();
            PyObject *fourth = FOURTH();
            SET_TOP(second);
            SET_SECOND(third);
            SET_THIRD(fourth);
            SET_FOURTH(top);
            DISPATCH();
        }

        TARGET(DUP_TOP) {
            PyObject *top = TOP();
            Py_INCREF(top);
            PUSH(top);
            DISPATCH();
        }

        TARGET(DUP_TOP_TWO) {
            PyObject *top = TOP();
            PyObject *second = SECOND();
            Py_INCREF(top);
            Py_INCREF(second);
            STACK_GROW(2);
            SET_TOP(top);
            SET_SECOND(second);
            DISPATCH();
        }

        TARGET(UNARY_POSITIVE) {
            PyObject *value = TOP();
            PyObject *res = PyNumber_Positive(value);
            Py_DECREF(value);
            SET_TOP(res);
            if (res == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(UNARY_NEGATIVE) {
            PyObject *value = TOP();
            PyObject *res = PyNumber_Negative(value);
            Py_DECREF(value);
            SET_TOP(res);
            if (res == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(UNARY_NOT) {
            PyObject *value = TOP();
            int err = PyObject_IsTrue(value);
            Py_DECREF(value);
            if (err == 0) {
                Py_INCREF(Py_True);
                SET_TOP(Py_True);
                DISPATCH();
            }
            else if (err > 0) {
                Py_INCREF(Py_False);
                SET_TOP(Py_False);
                DISPATCH();
            }
            STACK_SHRINK(1);
            goto error;
        }

        TARGET(UNARY_INVERT) {
            PyObject *value = TOP();
            PyObject *res = PyNumber_Invert(value);
            Py_DECREF(value);
            SET_TOP(res);
            if (res == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(BINARY_OP_MULTIPLY_INT) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
            SET_SECOND(prod);
            Py_DECREF(right);
            Py_DECREF(left);
            STACK_SHRINK(1);
            if (prod == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_MULTIPLY_FLOAT) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            double dprod = ((PyFloatObject *)left)->ob_fval *
                ((PyFloatObject *)right)->ob_fval;
            PyObject *prod = PyFloat_FromDouble(dprod);
            SET_SECOND(prod);
            Py_DECREF(right);
            Py_DECREF(left);
            STACK_SHRINK(1);
            if (prod == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_SUBTRACT_INT) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
            SET_SECOND(sub);
            Py_DECREF(right);
            Py_DECREF(left);
            STACK_SHRINK(1);
            if (sub == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_SUBTRACT_FLOAT) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval;
            PyObject *sub = PyFloat_FromDouble(dsub);
            SET_SECOND(sub);
            Py_DECREF(right);
            Py_DECREF(left);
            STACK_SHRINK(1);
            if (sub == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_ADD_UNICODE) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *res = PyUnicode_Concat(left, right);
            STACK_SHRINK(1);
            SET_TOP(res);
            Py_DECREF(left);
            Py_DECREF(right);
            if (TOP() == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_INPLACE_ADD_UNICODE) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            DEOPT_IF(Py_REFCNT(left) != 2, BINARY_OP);
            int next_oparg = _Py_OPARG(*next_instr);
            assert(_Py_OPCODE(*next_instr) == STORE_FAST);
            /* In the common case, there are 2 references to the value
            * stored in 'variable' when the v = v + ... is performed: one
            * on the value stack (in 'v') and one still stored in the
            * 'variable'.  We try to delete the variable now to reduce
            * the refcnt to 1.
            */
            PyObject *var = GETLOCAL(next_oparg);
            DEOPT_IF(var != left, BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            GETLOCAL(next_oparg) = NULL;
            Py_DECREF(left);
            STACK_SHRINK(1);
            PyUnicode_Append(&TOP(), right);
            Py_DECREF(right);
            if (TOP() == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_ADD_FLOAT) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            double dsum = ((PyFloatObject *)left)->ob_fval +
                ((PyFloatObject *)right)->ob_fval;
            PyObject *sum = PyFloat_FromDouble(dsum);
            SET_SECOND(sum);
            Py_DECREF(right);
            Py_DECREF(left);
            STACK_SHRINK(1);
            if (sum == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_ADD_INT) {
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
            SET_SECOND(sum);
            Py_DECREF(right);
            Py_DECREF(left);
            STACK_SHRINK(1);
            if (sum == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR) {
            PREDICTED(BINARY_SUBSCR);
            PyObject *sub = POP();
            PyObject *container = TOP();
            PyObject *res = PyObject_GetItem(container, sub);
            Py_DECREF(container);
            Py_DECREF(sub);
            SET_TOP(res);
            if (res == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_ADAPTIVE) {
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *sub = TOP();
                PyObject *container = SECOND();
                next_instr--;
                if (_Py_Specialize_BinarySubscr(container, sub, next_instr, cache) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(BINARY_SUBSCR, deferred);
                cache->adaptive.counter--;
                assert(cache->adaptive.original_oparg == 0);
                /* No need to set oparg here; it isn't used by BINARY_SUBSCR */
                JUMP_TO_INSTRUCTION(BINARY_SUBSCR);
            }
        }

        TARGET(BINARY_SUBSCR_LIST_INT) {
            PyObject *sub = TOP();
            PyObject *list = SECOND();
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);

            // Deopt unless 0 <= sub < PyList_Size(list)
            Py_ssize_t signed_magnitude = Py_SIZE(sub);
            DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
            assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
            Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
            DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res = PyList_GET_ITEM(list, index);
            assert(res != NULL);
            Py_INCREF(res);
            STACK_SHRINK(1);
            Py_DECREF(sub);
            SET_TOP(res);
            Py_DECREF(list);
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_TUPLE_INT) {
            PyObject *sub = TOP();
            PyObject *tuple = SECOND();
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);

            // Deopt unless 0 <= sub < PyTuple_Size(list)
            Py_ssize_t signed_magnitude = Py_SIZE(sub);
            DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
            assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
            Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
            DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res = PyTuple_GET_ITEM(tuple, index);
            assert(res != NULL);
            Py_INCREF(res);
            STACK_SHRINK(1);
            Py_DECREF(sub);
            SET_TOP(res);
            Py_DECREF(tuple);
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_DICT) {
            PyObject *dict = SECOND();
            DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *sub = TOP();
            PyObject *res = PyDict_GetItemWithError(dict, sub);
            if (res == NULL) {
                goto binary_subscr_dict_error;
            }
            Py_INCREF(res);
            STACK_SHRINK(1);
            Py_DECREF(sub);
            SET_TOP(res);
            Py_DECREF(dict);
            DISPATCH();
        }

        TARGET(BINARY_SUBSCR_GETITEM) {
            PyObject *sub = TOP();
            PyObject *container = SECOND();
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyObjectCache *cache1 = &caches[-1].obj;
            PyFunctionObject *getitem = (PyFunctionObject *)cache1->obj;
            DEOPT_IF(Py_TYPE(container)->tp_version_tag != cache0->version, BINARY_SUBSCR);
            DEOPT_IF(getitem->func_version != cache0->index, BINARY_SUBSCR);
            PyCodeObject *code = (PyCodeObject *)getitem->func_code;
            size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
            assert(code->co_argcount == 2);
            InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
            if (new_frame == NULL) {
                goto error;
            }
            _PyFrame_InitializeSpecials(new_frame, getitem,
                                        NULL, code->co_nlocalsplus);
            STACK_SHRINK(2);
            new_frame->localsplus[0] = container;
            new_frame->localsplus[1] = sub;
            for (int i = 2; i < code->co_nlocalsplus; i++) {
                new_frame->localsplus[i] = NULL;
            }
            _PyFrame_SetStackPointer(frame, stack_pointer);
            new_frame->previous = frame;
            frame = cframe.current_frame = new_frame;
            new_frame->depth = frame->depth + 1;
            goto start_frame;
        }

        TARGET(LIST_APPEND) {
            PyObject *v = POP();
            PyObject *list = PEEK(oparg);
            int err;
            err = PyList_Append(list, v);
            Py_DECREF(v);
            if (err != 0)
                goto error;
            PREDICT(JUMP_ABSOLUTE);
            DISPATCH();
        }

        TARGET(SET_ADD) {
            PyObject *v = POP();
            PyObject *set = PEEK(oparg);
            int err;
            err = PySet_Add(set, v);
            Py_DECREF(v);
            if (err != 0)
                goto error;
            PREDICT(JUMP_ABSOLUTE);
            DISPATCH();
        }

        TARGET(STORE_SUBSCR) {
            PREDICTED(STORE_SUBSCR);
            PyObject *sub = TOP();
            PyObject *container = SECOND();
            PyObject *v = THIRD();
            int err;
            STACK_SHRINK(3);
            /* container[sub] = v */
            err = PyObject_SetItem(container, sub, v);
            Py_DECREF(v);
            Py_DECREF(container);
            Py_DECREF(sub);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(STORE_SUBSCR_ADAPTIVE) {
            if (oparg == 0) {
                PyObject *sub = TOP();
                PyObject *container = SECOND();
                next_instr--;
                if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(STORE_SUBSCR, deferred);
                // oparg is the adaptive cache counter
                UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1);
                JUMP_TO_INSTRUCTION(STORE_SUBSCR);
            }
        }

        TARGET(STORE_SUBSCR_LIST_INT) {
            PyObject *sub = TOP();
            PyObject *list = SECOND();
            PyObject *value = THIRD();
            DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
            DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);

            // Ensure nonnegative, zero-or-one-digit ints.
            DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR);
            Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
            // Ensure index < len(list)
            DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
            STAT_INC(STORE_SUBSCR, hit);

            PyObject *old_value = PyList_GET_ITEM(list, index);
            PyList_SET_ITEM(list, index, value);
            STACK_SHRINK(3);
            assert(old_value != NULL);
            Py_DECREF(old_value);
            Py_DECREF(sub);
            Py_DECREF(list);
            DISPATCH();
        }

        TARGET(STORE_SUBSCR_DICT) {
            PyObject *sub = TOP();
            PyObject *dict = SECOND();
            PyObject *value = THIRD();
            DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
            STACK_SHRINK(3);
            STAT_INC(STORE_SUBSCR, hit);
            int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
            Py_DECREF(dict);
            if (err != 0) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(DELETE_SUBSCR) {
            PyObject *sub = TOP();
            PyObject *container = SECOND();
            int err;
            STACK_SHRINK(2);
            /* del container[sub] */
            err = PyObject_DelItem(container, sub);
            Py_DECREF(container);
            Py_DECREF(sub);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(PRINT_EXPR) {
            _Py_IDENTIFIER(displayhook);
            PyObject *value = POP();
            PyObject *hook = _PySys_GetObjectId(&PyId_displayhook);
            PyObject *res;
            if (hook == NULL) {
                _PyErr_SetString(tstate, PyExc_RuntimeError,
                                 "lost sys.displayhook");
                Py_DECREF(value);
                goto error;
            }
            res = PyObject_CallOneArg(hook, value);
            Py_DECREF(value);
            if (res == NULL)
                goto error;
            Py_DECREF(res);
            DISPATCH();
        }

        TARGET(RAISE_VARARGS) {
            PyObject *cause = NULL, *exc = NULL;
            switch (oparg) {
            case 2:
                cause = POP(); /* cause */
                /* fall through */
            case 1:
                exc = POP(); /* exc */
                /* fall through */
            case 0:
                if (do_raise(tstate, exc, cause)) {
                    goto exception_unwind;
                }
                break;
            default:
                _PyErr_SetString(tstate, PyExc_SystemError,
                                 "bad RAISE_VARARGS oparg");
                break;
            }
            goto error;
        }

        TARGET(RETURN_VALUE) {
            PyObject *retval = POP();
            assert(EMPTY());
            frame->f_state = FRAME_RETURNED;
            _PyFrame_SetStackPointer(frame, stack_pointer);
            TRACE_FUNCTION_EXIT();
            DTRACE_FUNCTION_EXIT();
            _Py_LeaveRecursiveCall(tstate);
            if (frame->depth) {
                frame = cframe.current_frame = pop_frame(tstate, frame);
                _PyFrame_StackPush(frame, retval);
                goto resume_frame;
            }
            /* Restore previous cframe and return. */
            tstate->cframe = cframe.previous;
            tstate->cframe->use_tracing = cframe.use_tracing;
            assert(tstate->cframe->current_frame == frame->previous);
            assert(!_PyErr_Occurred(tstate));
            return retval;
        }

        TARGET(GET_AITER) {
            unaryfunc getter = NULL;
            PyObject *iter = NULL;
            PyObject *obj = TOP();
            PyTypeObject *type = Py_TYPE(obj);

            if (type->tp_as_async != NULL) {
                getter = type->tp_as_async->am_aiter;
            }

            if (getter != NULL) {
                iter = (*getter)(obj);
                Py_DECREF(obj);
                if (iter == NULL) {
                    SET_TOP(NULL);
                    goto error;
                }
            }
            else {
                SET_TOP(NULL);
                _PyErr_Format(tstate, PyExc_TypeError,
                              "'async for' requires an object with "
                              "__aiter__ method, got %.100s",
                              type->tp_name);
                Py_DECREF(obj);
                goto error;
            }

            if (Py_TYPE(iter)->tp_as_async == NULL ||
                    Py_TYPE(iter)->tp_as_async->am_anext == NULL) {

                SET_TOP(NULL);
                _PyErr_Format(tstate, PyExc_TypeError,
                              "'async for' received an object from __aiter__ "
                              "that does not implement __anext__: %.100s",
                              Py_TYPE(iter)->tp_name);
                Py_DECREF(iter);
                goto error;
            }

            SET_TOP(iter);
            DISPATCH();
        }

        TARGET(GET_ANEXT) {
            unaryfunc getter = NULL;
            PyObject *next_iter = NULL;
            PyObject *awaitable = NULL;
            PyObject *aiter = TOP();
            PyTypeObject *type = Py_TYPE(aiter);

            if (PyAsyncGen_CheckExact(aiter)) {
                awaitable = type->tp_as_async->am_anext(aiter);
                if (awaitable == NULL) {
                    goto error;
                }
            } else {
                if (type->tp_as_async != NULL){
                    getter = type->tp_as_async->am_anext;
                }

                if (getter != NULL) {
                    next_iter = (*getter)(aiter);
                    if (next_iter == NULL) {
                        goto error;
                    }
                }
                else {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "'async for' requires an iterator with "
                                  "__anext__ method, got %.100s",
                                  type->tp_name);
                    goto error;
                }

                awaitable = _PyCoro_GetAwaitableIter(next_iter);
                if (awaitable == NULL) {
                    _PyErr_FormatFromCause(
                        PyExc_TypeError,
                        "'async for' received an invalid object "
                        "from __anext__: %.100s",
                        Py_TYPE(next_iter)->tp_name);

                    Py_DECREF(next_iter);
                    goto error;
                } else {
                    Py_DECREF(next_iter);
                }
            }

            PUSH(awaitable);
            PREDICT(LOAD_CONST);
            DISPATCH();
        }

        TARGET(GET_AWAITABLE) {
            PREDICTED(GET_AWAITABLE);
            PyObject *iterable = TOP();
            PyObject *iter = _PyCoro_GetAwaitableIter(iterable);

            if (iter == NULL) {
                int opcode_at_minus_3 = 0;
                if ((next_instr - first_instr) > 2) {
                    opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]);
                }
                format_awaitable_error(tstate, Py_TYPE(iterable),
                                       opcode_at_minus_3,
                                       _Py_OPCODE(next_instr[-2]));
            }

            Py_DECREF(iterable);

            if (iter != NULL && PyCoro_CheckExact(iter)) {
                PyObject *yf = _PyGen_yf((PyGenObject*)iter);
                if (yf != NULL) {
                    /* `iter` is a coroutine object that is being
                       awaited, `yf` is a pointer to the current awaitable
                       being awaited on. */
                    Py_DECREF(yf);
                    Py_CLEAR(iter);
                    _PyErr_SetString(tstate, PyExc_RuntimeError,
                                     "coroutine is being awaited already");
                    /* The code below jumps to `error` if `iter` is NULL. */
                }
            }

            SET_TOP(iter); /* Even if it's NULL */

            if (iter == NULL) {
                goto error;
            }

            PREDICT(LOAD_CONST);
            DISPATCH();
        }

        TARGET(SEND) {
            assert(frame->depth == 0);
            assert(STACK_LEVEL() >= 2);
            PyObject *v = POP();
            PyObject *receiver = TOP();
            PySendResult gen_status;
            PyObject *retval;
            if (tstate->c_tracefunc == NULL) {
                gen_status = PyIter_Send(receiver, v, &retval);
            } else {
                _Py_IDENTIFIER(send);
                if (Py_IsNone(v) && PyIter_Check(receiver)) {
                    retval = Py_TYPE(receiver)->tp_iternext(receiver);
                }
                else {
                    retval = _PyObject_CallMethodIdOneArg(receiver, &PyId_send, v);
                }
                if (retval == NULL) {
                    if (tstate->c_tracefunc != NULL
                            && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
                        call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
                    if (_PyGen_FetchStopIterationValue(&retval) == 0) {
                        gen_status = PYGEN_RETURN;
                    }
                    else {
                        gen_status = PYGEN_ERROR;
                    }
                }
                else {
                    gen_status = PYGEN_NEXT;
                }
            }
            Py_DECREF(v);
            if (gen_status == PYGEN_ERROR) {
                assert (retval == NULL);
                goto error;
            }
            if (gen_status == PYGEN_RETURN) {
                assert (retval != NULL);
                Py_DECREF(receiver);
                SET_TOP(retval);
                JUMPBY(oparg);
                DISPATCH();
            }
            assert (gen_status == PYGEN_NEXT);
            assert (retval != NULL);
            frame->f_state = FRAME_SUSPENDED;
            _PyFrame_SetStackPointer(frame, stack_pointer);
            TRACE_FUNCTION_EXIT();
            DTRACE_FUNCTION_EXIT();
            _Py_LeaveRecursiveCall(tstate);
            /* Restore previous cframe and return. */
            tstate->cframe = cframe.previous;
            tstate->cframe->use_tracing = cframe.use_tracing;
            assert(tstate->cframe->current_frame == frame->previous);
            assert(!_PyErr_Occurred(tstate));
            return retval;
        }

        TARGET(YIELD_VALUE) {
            assert(frame->depth == 0);
            PyObject *retval = POP();

            if (frame->f_code->co_flags & CO_ASYNC_GENERATOR) {
                PyObject *w = _PyAsyncGenValueWrapperNew(retval);
                Py_DECREF(retval);
                if (w == NULL) {
                    retval = NULL;
                    goto error;
                }
                retval = w;
            }
            frame->f_state = FRAME_SUSPENDED;
            _PyFrame_SetStackPointer(frame, stack_pointer);
            TRACE_FUNCTION_EXIT();
            DTRACE_FUNCTION_EXIT();
            _Py_LeaveRecursiveCall(tstate);
            /* Restore previous cframe and return. */
            tstate->cframe = cframe.previous;
            tstate->cframe->use_tracing = cframe.use_tracing;
            assert(tstate->cframe->current_frame == frame->previous);
            assert(!_PyErr_Occurred(tstate));
            return retval;
        }

        TARGET(GEN_START) {
            PyObject *none = POP();
            assert(none == Py_None);
            assert(oparg < 3);
            Py_DECREF(none);
            DISPATCH();
        }

        TARGET(POP_EXCEPT) {
            _PyErr_StackItem *exc_info = tstate->exc_info;
            PyObject *value = exc_info->exc_value;
            exc_info->exc_value = POP();
            Py_XDECREF(value);
            DISPATCH();
        }

        TARGET(POP_EXCEPT_AND_RERAISE) {
            PyObject *lasti = PEEK(2);
            if (PyLong_Check(lasti)) {
                frame->f_lasti = PyLong_AsLong(lasti);
                assert(!_PyErr_Occurred(tstate));
            }
            else {
                _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
                goto error;
            }
            PyObject *value = POP();
            assert(value);
            assert(PyExceptionInstance_Check(value));
            PyObject *type = Py_NewRef(PyExceptionInstance_Class(value));
            PyObject *traceback = PyException_GetTraceback(value);
            Py_DECREF(POP()); /* lasti */
            _PyErr_Restore(tstate, type, value, traceback);

            _PyErr_StackItem *exc_info = tstate->exc_info;
            value = exc_info->exc_value;
            exc_info->exc_value = POP();
            Py_XDECREF(value);
            goto exception_unwind;
        }

        TARGET(RERAISE) {
            if (oparg) {
                PyObject *lasti = PEEK(oparg + 1);
                if (PyLong_Check(lasti)) {
                    frame->f_lasti = PyLong_AsLong(lasti);
                    assert(!_PyErr_Occurred(tstate));
                }
                else {
                    assert(PyLong_Check(lasti));
                    _PyErr_SetString(tstate, PyExc_SystemError, "lasti is not an int");
                    goto error;
                }
            }
            PyObject *val = POP();
            assert(val && PyExceptionInstance_Check(val));
            PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val));
            PyObject *tb = PyException_GetTraceback(val);
            _PyErr_Restore(tstate, exc, val, tb);
            goto exception_unwind;
        }

        TARGET(PREP_RERAISE_STAR) {
            PyObject *excs = POP();
            assert(PyList_Check(excs));
            PyObject *orig = POP();

            PyObject *val = do_reraise_star(excs, orig);
            Py_DECREF(excs);
            Py_DECREF(orig);

            if (val == NULL) {
                goto error;
            }

            PyObject *lasti_unused = Py_NewRef(_PyLong_GetZero());
            PUSH(lasti_unused);
            PUSH(val);
            DISPATCH();
        }

        TARGET(END_ASYNC_FOR) {
            PyObject *val = POP();
            assert(val && PyExceptionInstance_Check(val));
            if (PyErr_GivenExceptionMatches(val, PyExc_StopAsyncIteration)) {
                Py_DECREF(val);
                Py_DECREF(POP());
                DISPATCH();
            }
            else {
                PyObject *exc = Py_NewRef(PyExceptionInstance_Class(val));
                PyObject *tb = PyException_GetTraceback(val);
                _PyErr_Restore(tstate, exc, val, tb);
                goto exception_unwind;
            }
        }

        TARGET(LOAD_ASSERTION_ERROR) {
            PyObject *value = PyExc_AssertionError;
            Py_INCREF(value);
            PUSH(value);
            DISPATCH();
        }

        TARGET(LOAD_BUILD_CLASS) {
            _Py_IDENTIFIER(__build_class__);

            PyObject *bc;
            if (PyDict_CheckExact(BUILTINS())) {
                bc = _PyDict_GetItemIdWithError(BUILTINS(), &PyId___build_class__);
                if (bc == NULL) {
                    if (!_PyErr_Occurred(tstate)) {
                        _PyErr_SetString(tstate, PyExc_NameError,
                                         "__build_class__ not found");
                    }
                    goto error;
                }
                Py_INCREF(bc);
            }
            else {
                PyObject *build_class_str = _PyUnicode_FromId(&PyId___build_class__);
                if (build_class_str == NULL)
                    goto error;
                bc = PyObject_GetItem(BUILTINS(), build_class_str);
                if (bc == NULL) {
                    if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
                        _PyErr_SetString(tstate, PyExc_NameError,
                                         "__build_class__ not found");
                    goto error;
                }
            }
            PUSH(bc);
            DISPATCH();
        }

        TARGET(STORE_NAME) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *v = POP();
            PyObject *ns = LOCALS();
            int err;
            if (ns == NULL) {
                _PyErr_Format(tstate, PyExc_SystemError,
                              "no locals found when storing %R", name);
                Py_DECREF(v);
                goto error;
            }
            if (PyDict_CheckExact(ns))
                err = PyDict_SetItem(ns, name, v);
            else
                err = PyObject_SetItem(ns, name, v);
            Py_DECREF(v);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(DELETE_NAME) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *ns = LOCALS();
            int err;
            if (ns == NULL) {
                _PyErr_Format(tstate, PyExc_SystemError,
                              "no locals when deleting %R", name);
                goto error;
            }
            err = PyObject_DelItem(ns, name);
            if (err != 0) {
                format_exc_check_arg(tstate, PyExc_NameError,
                                     NAME_ERROR_MSG,
                                     name);
                goto error;
            }
            DISPATCH();
        }

        TARGET(UNPACK_SEQUENCE) {
            PREDICTED(UNPACK_SEQUENCE);
            PyObject *seq = POP(), *item, **items;
            if (PyTuple_CheckExact(seq) &&
                PyTuple_GET_SIZE(seq) == oparg) {
                items = ((PyTupleObject *)seq)->ob_item;
                while (oparg--) {
                    item = items[oparg];
                    Py_INCREF(item);
                    PUSH(item);
                }
            } else if (PyList_CheckExact(seq) &&
                       PyList_GET_SIZE(seq) == oparg) {
                items = ((PyListObject *)seq)->ob_item;
                while (oparg--) {
                    item = items[oparg];
                    Py_INCREF(item);
                    PUSH(item);
                }
            } else if (unpack_iterable(tstate, seq, oparg, -1,
                                       stack_pointer + oparg)) {
                STACK_GROW(oparg);
            } else {
                /* unpack_iterable() raised an exception */
                Py_DECREF(seq);
                goto error;
            }
            Py_DECREF(seq);
            DISPATCH();
        }

        TARGET(UNPACK_EX) {
            int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
            PyObject *seq = POP();

            if (unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8,
                                stack_pointer + totalargs)) {
                stack_pointer += totalargs;
            } else {
                Py_DECREF(seq);
                goto error;
            }
            Py_DECREF(seq);
            DISPATCH();
        }

        TARGET(STORE_ATTR) {
            PREDICTED(STORE_ATTR);
            PyObject *name = GETITEM(names, oparg);
            PyObject *owner = TOP();
            PyObject *v = SECOND();
            int err;
            STACK_SHRINK(2);
            err = PyObject_SetAttr(owner, name, v);
            Py_DECREF(v);
            Py_DECREF(owner);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(DELETE_ATTR) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *owner = POP();
            int err;
            err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
            Py_DECREF(owner);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(STORE_GLOBAL) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *v = POP();
            int err;
            err = PyDict_SetItem(GLOBALS(), name, v);
            Py_DECREF(v);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(DELETE_GLOBAL) {
            PyObject *name = GETITEM(names, oparg);
            int err;
            err = PyDict_DelItem(GLOBALS(), name);
            if (err != 0) {
                if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                    format_exc_check_arg(tstate, PyExc_NameError,
                                         NAME_ERROR_MSG, name);
                }
                goto error;
            }
            DISPATCH();
        }

        TARGET(LOAD_NAME) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *locals = LOCALS();
            PyObject *v;
            if (locals == NULL) {
                _PyErr_Format(tstate, PyExc_SystemError,
                              "no locals when loading %R", name);
                goto error;
            }
            if (PyDict_CheckExact(locals)) {
                v = PyDict_GetItemWithError(locals, name);
                if (v != NULL) {
                    Py_INCREF(v);
                }
                else if (_PyErr_Occurred(tstate)) {
                    goto error;
                }
            }
            else {
                v = PyObject_GetItem(locals, name);
                if (v == NULL) {
                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
                        goto error;
                    _PyErr_Clear(tstate);
                }
            }
            if (v == NULL) {
                v = PyDict_GetItemWithError(GLOBALS(), name);
                if (v != NULL) {
                    Py_INCREF(v);
                }
                else if (_PyErr_Occurred(tstate)) {
                    goto error;
                }
                else {
                    if (PyDict_CheckExact(BUILTINS())) {
                        v = PyDict_GetItemWithError(BUILTINS(), name);
                        if (v == NULL) {
                            if (!_PyErr_Occurred(tstate)) {
                                format_exc_check_arg(
                                        tstate, PyExc_NameError,
                                        NAME_ERROR_MSG, name);
                            }
                            goto error;
                        }
                        Py_INCREF(v);
                    }
                    else {
                        v = PyObject_GetItem(BUILTINS(), name);
                        if (v == NULL) {
                            if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                                format_exc_check_arg(
                                            tstate, PyExc_NameError,
                                            NAME_ERROR_MSG, name);
                            }
                            goto error;
                        }
                    }
                }
            }
            PUSH(v);
            DISPATCH();
        }

        TARGET(LOAD_GLOBAL) {
            PREDICTED(LOAD_GLOBAL);
            PyObject *name = GETITEM(names, oparg);
            PyObject *v;
            if (PyDict_CheckExact(GLOBALS())
                && PyDict_CheckExact(BUILTINS()))
            {
                v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
                                       (PyDictObject *)BUILTINS(),
                                       name);
                if (v == NULL) {
                    if (!_PyErr_Occurred(tstate)) {
                        /* _PyDict_LoadGlobal() returns NULL without raising
                         * an exception if the key doesn't exist */
                        format_exc_check_arg(tstate, PyExc_NameError,
                                             NAME_ERROR_MSG, name);
                    }
                    goto error;
                }
                Py_INCREF(v);
            }
            else {
                /* Slow-path if globals or builtins is not a dict */

                /* namespace 1: globals */
                name = GETITEM(names, oparg);
                v = PyObject_GetItem(GLOBALS(), name);
                if (v == NULL) {
                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                        goto error;
                    }
                    _PyErr_Clear(tstate);

                    /* namespace 2: builtins */
                    v = PyObject_GetItem(BUILTINS(), name);
                    if (v == NULL) {
                        if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                            format_exc_check_arg(
                                        tstate, PyExc_NameError,
                                        NAME_ERROR_MSG, name);
                        }
                        goto error;
                    }
                }
            }
            PUSH(v);
            DISPATCH();
        }

        TARGET(LOAD_GLOBAL_ADAPTIVE) {
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *name = GETITEM(names, cache->adaptive.original_oparg);
                next_instr--;
                if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name, cache) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(LOAD_GLOBAL, deferred);
                cache->adaptive.counter--;
                oparg = cache->adaptive.original_oparg;
                JUMP_TO_INSTRUCTION(LOAD_GLOBAL);
            }
        }

        TARGET(LOAD_GLOBAL_MODULE) {
            assert(cframe.use_tracing == 0);
            DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
            PyDictObject *dict = (PyDictObject *)GLOBALS();
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyLoadGlobalCache *cache1 = &caches[-1].load_global;
            DEOPT_IF(dict->ma_keys->dk_version != cache1->module_keys_version, LOAD_GLOBAL);
            PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + cache0->index;
            PyObject *res = ep->me_value;
            DEOPT_IF(res == NULL, LOAD_GLOBAL);
            STAT_INC(LOAD_GLOBAL, hit);
            Py_INCREF(res);
            PUSH(res);
            DISPATCH();
        }

        TARGET(LOAD_GLOBAL_BUILTIN) {
            assert(cframe.use_tracing == 0);
            DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
            DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL);
            PyDictObject *mdict = (PyDictObject *)GLOBALS();
            PyDictObject *bdict = (PyDictObject *)BUILTINS();
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyLoadGlobalCache *cache1 = &caches[-1].load_global;
            DEOPT_IF(mdict->ma_keys->dk_version != cache1->module_keys_version, LOAD_GLOBAL);
            DEOPT_IF(bdict->ma_keys->dk_version != cache1->builtin_keys_version, LOAD_GLOBAL);
            PyDictKeyEntry *ep = DK_ENTRIES(bdict->ma_keys) + cache0->index;
            PyObject *res = ep->me_value;
            DEOPT_IF(res == NULL, LOAD_GLOBAL);
            STAT_INC(LOAD_GLOBAL, hit);
            Py_INCREF(res);
            PUSH(res);
            DISPATCH();
        }

        TARGET(DELETE_FAST) {
            PyObject *v = GETLOCAL(oparg);
            if (v != NULL) {
                SETLOCAL(oparg, NULL);
                DISPATCH();
            }
            format_exc_check_arg(
                tstate, PyExc_UnboundLocalError,
                UNBOUNDLOCAL_ERROR_MSG,
                PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
                );
            goto error;
        }

        TARGET(MAKE_CELL) {
            // "initial" is probably NULL but not if it's an arg (or set
            // via PyFrame_LocalsToFast() before MAKE_CELL has run).
            PyObject *initial = GETLOCAL(oparg);
            PyObject *cell = PyCell_New(initial);
            if (cell == NULL) {
                goto error;
            }
            SETLOCAL(oparg, cell);
            DISPATCH();
        }

        TARGET(DELETE_DEREF) {
            PyObject *cell = GETLOCAL(oparg);
            PyObject *oldobj = PyCell_GET(cell);
            if (oldobj != NULL) {
                PyCell_SET(cell, NULL);
                Py_DECREF(oldobj);
                DISPATCH();
            }
            format_exc_unbound(tstate, frame->f_code, oparg);
            goto error;
        }

        TARGET(LOAD_CLASSDEREF) {
            PyObject *name, *value, *locals = LOCALS();
            assert(locals);
            assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus);
            name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg);
            if (PyDict_CheckExact(locals)) {
                value = PyDict_GetItemWithError(locals, name);
                if (value != NULL) {
                    Py_INCREF(value);
                }
                else if (_PyErr_Occurred(tstate)) {
                    goto error;
                }
            }
            else {
                value = PyObject_GetItem(locals, name);
                if (value == NULL) {
                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                        goto error;
                    }
                    _PyErr_Clear(tstate);
                }
            }
            if (!value) {
                PyObject *cell = GETLOCAL(oparg);
                value = PyCell_GET(cell);
                if (value == NULL) {
                    format_exc_unbound(tstate, frame->f_code, oparg);
                    goto error;
                }
                Py_INCREF(value);
            }
            PUSH(value);
            DISPATCH();
        }

        TARGET(LOAD_DEREF) {
            PyObject *cell = GETLOCAL(oparg);
            PyObject *value = PyCell_GET(cell);
            if (value == NULL) {
                format_exc_unbound(tstate, frame->f_code, oparg);
                goto error;
            }
            Py_INCREF(value);
            PUSH(value);
            DISPATCH();
        }

        TARGET(STORE_DEREF) {
            PyObject *v = POP();
            PyObject *cell = GETLOCAL(oparg);
            PyObject *oldobj = PyCell_GET(cell);
            PyCell_SET(cell, v);
            Py_XDECREF(oldobj);
            DISPATCH();
        }

        TARGET(COPY_FREE_VARS) {
            /* Copy closure variables to free variables */
            PyCodeObject *co = frame->f_code;
            PyObject *closure = frame->f_func->func_closure;
            int offset = co->co_nlocals + co->co_nplaincellvars;
            assert(oparg == co->co_nfreevars);
            for (int i = 0; i < oparg; ++i) {
                PyObject *o = PyTuple_GET_ITEM(closure, i);
                Py_INCREF(o);
                frame->localsplus[offset + i] = o;
            }
            DISPATCH();
        }

        TARGET(BUILD_STRING) {
            PyObject *str;
            PyObject *empty = PyUnicode_New(0, 0);
            if (empty == NULL) {
                goto error;
            }
            str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg);
            Py_DECREF(empty);
            if (str == NULL)
                goto error;
            while (--oparg >= 0) {
                PyObject *item = POP();
                Py_DECREF(item);
            }
            PUSH(str);
            DISPATCH();
        }

        TARGET(BUILD_TUPLE) {
            PyObject *tup = PyTuple_New(oparg);
            if (tup == NULL)
                goto error;
            while (--oparg >= 0) {
                PyObject *item = POP();
                PyTuple_SET_ITEM(tup, oparg, item);
            }
            PUSH(tup);
            DISPATCH();
        }

        TARGET(BUILD_LIST) {
            PyObject *list =  PyList_New(oparg);
            if (list == NULL)
                goto error;
            while (--oparg >= 0) {
                PyObject *item = POP();
                PyList_SET_ITEM(list, oparg, item);
            }
            PUSH(list);
            DISPATCH();
        }

        TARGET(LIST_TO_TUPLE) {
            PyObject *list = POP();
            PyObject *tuple = PyList_AsTuple(list);
            Py_DECREF(list);
            if (tuple == NULL) {
                goto error;
            }
            PUSH(tuple);
            DISPATCH();
        }

        TARGET(LIST_EXTEND) {
            PyObject *iterable = POP();
            PyObject *list = PEEK(oparg);
            PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable);
            if (none_val == NULL) {
                if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
                   (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable)))
                {
                    _PyErr_Clear(tstate);
                    _PyErr_Format(tstate, PyExc_TypeError,
                          "Value after * must be an iterable, not %.200s",
                          Py_TYPE(iterable)->tp_name);
                }
                Py_DECREF(iterable);
                goto error;
            }
            Py_DECREF(none_val);
            Py_DECREF(iterable);
            DISPATCH();
        }

        TARGET(SET_UPDATE) {
            PyObject *iterable = POP();
            PyObject *set = PEEK(oparg);
            int err = _PySet_Update(set, iterable);
            Py_DECREF(iterable);
            if (err < 0) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BUILD_SET) {
            PyObject *set = PySet_New(NULL);
            int err = 0;
            int i;
            if (set == NULL)
                goto error;
            for (i = oparg; i > 0; i--) {
                PyObject *item = PEEK(i);
                if (err == 0)
                    err = PySet_Add(set, item);
                Py_DECREF(item);
            }
            STACK_SHRINK(oparg);
            if (err != 0) {
                Py_DECREF(set);
                goto error;
            }
            PUSH(set);
            DISPATCH();
        }

        TARGET(BUILD_MAP) {
            Py_ssize_t i;
            PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg);
            if (map == NULL)
                goto error;
            for (i = oparg; i > 0; i--) {
                int err;
                PyObject *key = PEEK(2*i);
                PyObject *value = PEEK(2*i - 1);
                err = PyDict_SetItem(map, key, value);
                if (err != 0) {
                    Py_DECREF(map);
                    goto error;
                }
            }

            while (oparg--) {
                Py_DECREF(POP());
                Py_DECREF(POP());
            }
            PUSH(map);
            DISPATCH();
        }

        TARGET(SETUP_ANNOTATIONS) {
            _Py_IDENTIFIER(__annotations__);
            int err;
            PyObject *ann_dict;
            if (LOCALS() == NULL) {
                _PyErr_Format(tstate, PyExc_SystemError,
                              "no locals found when setting up annotations");
                goto error;
            }
            /* check if __annotations__ in locals()... */
            if (PyDict_CheckExact(LOCALS())) {
                ann_dict = _PyDict_GetItemIdWithError(LOCALS(),
                                             &PyId___annotations__);
                if (ann_dict == NULL) {
                    if (_PyErr_Occurred(tstate)) {
                        goto error;
                    }
                    /* ...if not, create a new one */
                    ann_dict = PyDict_New();
                    if (ann_dict == NULL) {
                        goto error;
                    }
                    err = _PyDict_SetItemId(LOCALS(),
                                            &PyId___annotations__, ann_dict);
                    Py_DECREF(ann_dict);
                    if (err != 0) {
                        goto error;
                    }
                }
            }
            else {
                /* do the same if locals() is not a dict */
                PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__);
                if (ann_str == NULL) {
                    goto error;
                }
                ann_dict = PyObject_GetItem(LOCALS(), ann_str);
                if (ann_dict == NULL) {
                    if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                        goto error;
                    }
                    _PyErr_Clear(tstate);
                    ann_dict = PyDict_New();
                    if (ann_dict == NULL) {
                        goto error;
                    }
                    err = PyObject_SetItem(LOCALS(), ann_str, ann_dict);
                    Py_DECREF(ann_dict);
                    if (err != 0) {
                        goto error;
                    }
                }
                else {
                    Py_DECREF(ann_dict);
                }
            }
            DISPATCH();
        }

        TARGET(BUILD_CONST_KEY_MAP) {
            Py_ssize_t i;
            PyObject *map;
            PyObject *keys = TOP();
            if (!PyTuple_CheckExact(keys) ||
                PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) {
                _PyErr_SetString(tstate, PyExc_SystemError,
                                 "bad BUILD_CONST_KEY_MAP keys argument");
                goto error;
            }
            map = _PyDict_NewPresized((Py_ssize_t)oparg);
            if (map == NULL) {
                goto error;
            }
            for (i = oparg; i > 0; i--) {
                int err;
                PyObject *key = PyTuple_GET_ITEM(keys, oparg - i);
                PyObject *value = PEEK(i + 1);
                err = PyDict_SetItem(map, key, value);
                if (err != 0) {
                    Py_DECREF(map);
                    goto error;
                }
            }

            Py_DECREF(POP());
            while (oparg--) {
                Py_DECREF(POP());
            }
            PUSH(map);
            DISPATCH();
        }

        TARGET(DICT_UPDATE) {
            PyObject *update = POP();
            PyObject *dict = PEEK(oparg);
            if (PyDict_Update(dict, update) < 0) {
                if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                    "'%.200s' object is not a mapping",
                                    Py_TYPE(update)->tp_name);
                }
                Py_DECREF(update);
                goto error;
            }
            Py_DECREF(update);
            DISPATCH();
        }

        TARGET(DICT_MERGE) {
            PyObject *update = POP();
            PyObject *dict = PEEK(oparg);

            if (_PyDict_MergeEx(dict, update, 2) < 0) {
                format_kwargs_error(tstate, PEEK(2 + oparg), update);
                Py_DECREF(update);
                goto error;
            }
            Py_DECREF(update);
            PREDICT(CALL_FUNCTION_EX);
            DISPATCH();
        }

        TARGET(MAP_ADD) {
            PyObject *value = TOP();
            PyObject *key = SECOND();
            PyObject *map;
            STACK_SHRINK(2);
            map = PEEK(oparg);                      /* dict */
            assert(PyDict_CheckExact(map));
            /* map[key] = value */
            if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) {
                goto error;
            }
            PREDICT(JUMP_ABSOLUTE);
            DISPATCH();
        }

        TARGET(LOAD_ATTR) {
            PREDICTED(LOAD_ATTR);
            PyObject *name = GETITEM(names, oparg);
            PyObject *owner = TOP();
            PyObject *res = PyObject_GetAttr(owner, name);
            if (res == NULL) {
                goto error;
            }
            Py_DECREF(owner);
            SET_TOP(res);
            DISPATCH();
        }

        TARGET(LOAD_ATTR_ADAPTIVE) {
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *owner = TOP();
                PyObject *name = GETITEM(names, cache->adaptive.original_oparg);
                next_instr--;
                if (_Py_Specialize_LoadAttr(owner, next_instr, name, cache) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(LOAD_ATTR, deferred);
                cache->adaptive.counter--;
                oparg = cache->adaptive.original_oparg;
                JUMP_TO_INSTRUCTION(LOAD_ATTR);
            }
        }

        TARGET(LOAD_ATTR_INSTANCE_VALUE) {
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyObject *res;
            PyTypeObject *tp = Py_TYPE(owner);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyAttrCache *cache1 = &caches[-1].attr;
            assert(cache1->tp_version != 0);
            DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR);
            assert(tp->tp_dictoffset < 0);
            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
            PyDictValues *values = *_PyObject_ValuesPointer(owner);
            DEOPT_IF(values == NULL, LOAD_ATTR);
            res = values->values[cache0->index];
            DEOPT_IF(res == NULL, LOAD_ATTR);
            STAT_INC(LOAD_ATTR, hit);
            Py_INCREF(res);
            SET_TOP(res);
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(LOAD_ATTR_MODULE) {
            assert(cframe.use_tracing == 0);
            // shared with LOAD_METHOD_MODULE
            PyObject *owner = TOP();
            PyObject *res;
            LOAD_MODULE_ATTR_OR_METHOD(ATTR);
            SET_TOP(res);
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(LOAD_ATTR_WITH_HINT) {
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyObject *res;
            PyTypeObject *tp = Py_TYPE(owner);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyAttrCache *cache1 = &caches[-1].attr;
            assert(cache1->tp_version != 0);
            DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR);
            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
            PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
            DEOPT_IF(dict == NULL, LOAD_ATTR);
            assert(PyDict_CheckExact((PyObject *)dict));
            PyObject *name = GETITEM(names, cache0->original_oparg);
            uint32_t hint = cache1->dk_version_or_hint;
            DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
            PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
            DEOPT_IF(ep->me_key != name, LOAD_ATTR);
            res = ep->me_value;
            DEOPT_IF(res == NULL, LOAD_ATTR);
            STAT_INC(LOAD_ATTR, hit);
            Py_INCREF(res);
            SET_TOP(res);
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(LOAD_ATTR_SLOT) {
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyObject *res;
            PyTypeObject *tp = Py_TYPE(owner);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyAttrCache *cache1 = &caches[-1].attr;
            assert(cache1->tp_version != 0);
            DEOPT_IF(tp->tp_version_tag != cache1->tp_version, LOAD_ATTR);
            char *addr = (char *)owner + cache0->index;
            res = *(PyObject **)addr;
            DEOPT_IF(res == NULL, LOAD_ATTR);
            STAT_INC(LOAD_ATTR, hit);
            Py_INCREF(res);
            SET_TOP(res);
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(STORE_ATTR_ADAPTIVE) {
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *owner = TOP();
                PyObject *name = GETITEM(names, cache->adaptive.original_oparg);
                next_instr--;
                if (_Py_Specialize_StoreAttr(owner, next_instr, name, cache) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(STORE_ATTR, deferred);
                cache->adaptive.counter--;
                oparg = cache->adaptive.original_oparg;
                JUMP_TO_INSTRUCTION(STORE_ATTR);
            }
        }

        TARGET(STORE_ATTR_INSTANCE_VALUE) {
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyTypeObject *tp = Py_TYPE(owner);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyAttrCache *cache1 = &caches[-1].attr;
            assert(cache1->tp_version != 0);
            DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR);
            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
            PyDictValues *values = *_PyObject_ValuesPointer(owner);
            DEOPT_IF(values == NULL, STORE_ATTR);
            STAT_INC(STORE_ATTR, hit);
            int index = cache0->index;
            STACK_SHRINK(1);
            PyObject *value = POP();
            PyObject *old_value = values->values[index];
            values->values[index] = value;
            if (old_value == NULL) {
                assert(index < 16);
                values->mv_order = (values->mv_order << 4) | index;
            }
            else {
                Py_DECREF(old_value);
            }
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(STORE_ATTR_WITH_HINT) {
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyTypeObject *tp = Py_TYPE(owner);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyAttrCache *cache1 = &caches[-1].attr;
            assert(cache1->tp_version != 0);
            DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR);
            assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
            PyDictObject *dict = *(PyDictObject **)_PyObject_ManagedDictPointer(owner);
            DEOPT_IF(dict == NULL, STORE_ATTR);
            assert(PyDict_CheckExact((PyObject *)dict));
            PyObject *name = GETITEM(names, cache0->original_oparg);
            uint32_t hint = cache1->dk_version_or_hint;
            DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR);
            PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint;
            DEOPT_IF(ep->me_key != name, STORE_ATTR);
            PyObject *old_value = ep->me_value;
            DEOPT_IF(old_value == NULL, STORE_ATTR);
            STAT_INC(STORE_ATTR, hit);
            STACK_SHRINK(1);
            PyObject *value = POP();
            ep->me_value = value;
            Py_DECREF(old_value);
            /* Ensure dict is GC tracked if it needs to be */
            if (!_PyObject_GC_IS_TRACKED(dict) && _PyObject_GC_MAY_BE_TRACKED(value)) {
                _PyObject_GC_TRACK(dict);
            }
            /* PEP 509 */
            dict->ma_version_tag = DICT_NEXT_VERSION();
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(STORE_ATTR_SLOT) {
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyTypeObject *tp = Py_TYPE(owner);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            _PyAttrCache *cache1 = &caches[-1].attr;
            assert(cache1->tp_version != 0);
            DEOPT_IF(tp->tp_version_tag != cache1->tp_version, STORE_ATTR);
            char *addr = (char *)owner + cache0->index;
            STAT_INC(STORE_ATTR, hit);
            STACK_SHRINK(1);
            PyObject *value = POP();
            PyObject *old_value = *(PyObject **)addr;
            *(PyObject **)addr = value;
            Py_XDECREF(old_value);
            Py_DECREF(owner);
            DISPATCH();
        }

        TARGET(COMPARE_OP) {
            PREDICTED(COMPARE_OP);
            assert(oparg <= Py_GE);
            PyObject *right = POP();
            PyObject *left = TOP();
            PyObject *res = PyObject_RichCompare(left, right, oparg);
            SET_TOP(res);
            Py_DECREF(left);
            Py_DECREF(right);
            if (res == NULL)
                goto error;
            PREDICT(POP_JUMP_IF_FALSE);
            PREDICT(POP_JUMP_IF_TRUE);
            DISPATCH();
        }

        TARGET(COMPARE_OP_ADAPTIVE) {
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *right = TOP();
                PyObject *left = SECOND();
                next_instr--;
                _Py_Specialize_CompareOp(left, right, next_instr, cache);
                DISPATCH();
            }
            else {
                STAT_INC(COMPARE_OP, deferred);
                cache->adaptive.counter--;
                oparg = cache->adaptive.original_oparg;
                JUMP_TO_INSTRUCTION(COMPARE_OP);
            }
        }

        TARGET(COMPARE_OP_FLOAT_JUMP) {
            assert(cframe.use_tracing == 0);
            // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
            SpecializedCacheEntry *caches = GET_CACHE();
            int when_to_jump_mask = caches[0].adaptive.index;
            PyObject *right = TOP();
            PyObject *left = SECOND();
            DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
            DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
            double dleft = PyFloat_AS_DOUBLE(left);
            double dright = PyFloat_AS_DOUBLE(right);
            int sign = (dleft > dright) - (dleft < dright);
            DEOPT_IF(isnan(dleft), COMPARE_OP);
            DEOPT_IF(isnan(dright), COMPARE_OP);
            STAT_INC(COMPARE_OP, hit);
            NEXTOPARG();
            STACK_SHRINK(2);
            Py_DECREF(left);
            Py_DECREF(right);
            assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
            int jump = (1 << (sign + 1)) & when_to_jump_mask;
            if (!jump) {
                next_instr++;
                NOTRACE_DISPATCH();
            }
            else {
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
                NOTRACE_DISPATCH();
            }
        }

        TARGET(COMPARE_OP_INT_JUMP) {
            assert(cframe.use_tracing == 0);
            // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
            SpecializedCacheEntry *caches = GET_CACHE();
            int when_to_jump_mask = caches[0].adaptive.index;
            PyObject *right = TOP();
            PyObject *left = SECOND();
            DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
            DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
            DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
            DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
            STAT_INC(COMPARE_OP, hit);
            assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
            Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
            Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
            int sign = (ileft > iright) - (ileft < iright);
            NEXTOPARG();
            STACK_SHRINK(2);
            Py_DECREF(left);
            Py_DECREF(right);
            assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
            int jump = (1 << (sign + 1)) & when_to_jump_mask;
            if (!jump) {
                next_instr++;
                NOTRACE_DISPATCH();
            }
            else {
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
                NOTRACE_DISPATCH();
            }
        }

        TARGET(COMPARE_OP_STR_JUMP) {
            assert(cframe.use_tracing == 0);
            // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
            SpecializedCacheEntry *caches = GET_CACHE();
            int invert = caches[0].adaptive.index;
            PyObject *right = TOP();
            PyObject *left = SECOND();
            DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
            DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
            STAT_INC(COMPARE_OP, hit);
            int res = _PyUnicode_Equal(left, right);
            if (res < 0) {
                goto error;
            }
            assert(caches[0].adaptive.original_oparg == Py_EQ ||
                   caches[0].adaptive.original_oparg == Py_NE);
            NEXTOPARG();
            assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
            STACK_SHRINK(2);
            Py_DECREF(left);
            Py_DECREF(right);
            assert(res == 0 || res == 1);
            assert(invert == 0 || invert == 1);
            int jump = res ^ invert;
            if (!jump) {
                next_instr++;
                NOTRACE_DISPATCH();
            }
            else {
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
                NOTRACE_DISPATCH();
            }
        }

        TARGET(IS_OP) {
            PyObject *right = POP();
            PyObject *left = TOP();
            int res = Py_Is(left, right) ^ oparg;
            PyObject *b = res ? Py_True : Py_False;
            Py_INCREF(b);
            SET_TOP(b);
            Py_DECREF(left);
            Py_DECREF(right);
            PREDICT(POP_JUMP_IF_FALSE);
            PREDICT(POP_JUMP_IF_TRUE);
            DISPATCH();
        }

        TARGET(CONTAINS_OP) {
            PyObject *right = POP();
            PyObject *left = POP();
            int res = PySequence_Contains(right, left);
            Py_DECREF(left);
            Py_DECREF(right);
            if (res < 0) {
                goto error;
            }
            PyObject *b = (res^oparg) ? Py_True : Py_False;
            Py_INCREF(b);
            PUSH(b);
            PREDICT(POP_JUMP_IF_FALSE);
            PREDICT(POP_JUMP_IF_TRUE);
            DISPATCH();
        }

        TARGET(JUMP_IF_NOT_EG_MATCH) {
            PyObject *match_type = POP();
            if (check_except_star_type_valid(tstate, match_type) < 0) {
                Py_DECREF(match_type);
                goto error;
            }

            PyObject *exc_value = TOP();
            PyObject *match = NULL, *rest = NULL;
            int res = exception_group_match(exc_value, match_type,
                                            &match, &rest);
            Py_DECREF(match_type);
            if (res < 0) {
                goto error;
            }

            if (match == NULL || rest == NULL) {
                assert(match == NULL);
                assert(rest == NULL);
                goto error;
            }

            if (Py_IsNone(match)) {
                Py_DECREF(match);
                Py_XDECREF(rest);
                /* no match - jump to target */
                JUMPTO(oparg);
            }
            else {

                /* Total or partial match - update the stack from
                 * [val]
                 * to
                 * [rest, match]
                 * (rest can be Py_None)
                 */

                PyObject *exc = TOP();

                SET_TOP(rest);
                PUSH(match);

                PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL);

                Py_DECREF(exc);

            }

            DISPATCH();
        }

        TARGET(JUMP_IF_NOT_EXC_MATCH) {
            PyObject *right = POP();
            PyObject *left = TOP();
            assert(PyExceptionInstance_Check(left));
            if (check_except_type_valid(tstate, right) < 0) {
                 Py_DECREF(right);
                 goto error;
            }

            int res = PyErr_GivenExceptionMatches(left, right);
            Py_DECREF(right);
            if (res == 0) {
                JUMPTO(oparg);
            }
            DISPATCH();
        }

        TARGET(IMPORT_NAME) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *fromlist = POP();
            PyObject *level = TOP();
            PyObject *res;
            res = import_name(tstate, frame, name, fromlist, level);
            Py_DECREF(level);
            Py_DECREF(fromlist);
            SET_TOP(res);
            if (res == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(IMPORT_STAR) {
            PyObject *from = POP(), *locals;
            int err;
            if (_PyFrame_FastToLocalsWithError(frame) < 0) {
                Py_DECREF(from);
                goto error;
            }

            locals = LOCALS();
            if (locals == NULL) {
                _PyErr_SetString(tstate, PyExc_SystemError,
                                 "no locals found during 'import *'");
                Py_DECREF(from);
                goto error;
            }
            err = import_all_from(tstate, locals, from);
            _PyFrame_LocalsToFast(frame, 0);
            Py_DECREF(from);
            if (err != 0)
                goto error;
            DISPATCH();
        }

        TARGET(IMPORT_FROM) {
            PyObject *name = GETITEM(names, oparg);
            PyObject *from = TOP();
            PyObject *res;
            res = import_from(tstate, from, name);
            PUSH(res);
            if (res == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(JUMP_FORWARD) {
            JUMPBY(oparg);
            DISPATCH();
        }

        TARGET(POP_JUMP_IF_FALSE) {
            PREDICTED(POP_JUMP_IF_FALSE);
            PyObject *cond = POP();
            int err;
            if (Py_IsTrue(cond)) {
                Py_DECREF(cond);
                DISPATCH();
            }
            if (Py_IsFalse(cond)) {
                Py_DECREF(cond);
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
                DISPATCH();
            }
            err = PyObject_IsTrue(cond);
            Py_DECREF(cond);
            if (err > 0)
                ;
            else if (err == 0) {
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
            }
            else
                goto error;
            DISPATCH();
        }

        TARGET(POP_JUMP_IF_TRUE) {
            PREDICTED(POP_JUMP_IF_TRUE);
            PyObject *cond = POP();
            int err;
            if (Py_IsFalse(cond)) {
                Py_DECREF(cond);
                DISPATCH();
            }
            if (Py_IsTrue(cond)) {
                Py_DECREF(cond);
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
                DISPATCH();
            }
            err = PyObject_IsTrue(cond);
            Py_DECREF(cond);
            if (err > 0) {
                JUMPTO(oparg);
                CHECK_EVAL_BREAKER();
            }
            else if (err == 0)
                ;
            else
                goto error;
            DISPATCH();
        }

        TARGET(JUMP_IF_FALSE_OR_POP) {
            PyObject *cond = TOP();
            int err;
            if (Py_IsTrue(cond)) {
                STACK_SHRINK(1);
                Py_DECREF(cond);
                DISPATCH();
            }
            if (Py_IsFalse(cond)) {
                JUMPTO(oparg);
                DISPATCH();
            }
            err = PyObject_IsTrue(cond);
            if (err > 0) {
                STACK_SHRINK(1);
                Py_DECREF(cond);
            }
            else if (err == 0)
                JUMPTO(oparg);
            else
                goto error;
            DISPATCH();
        }

        TARGET(JUMP_IF_TRUE_OR_POP) {
            PyObject *cond = TOP();
            int err;
            if (Py_IsFalse(cond)) {
                STACK_SHRINK(1);
                Py_DECREF(cond);
                DISPATCH();
            }
            if (Py_IsTrue(cond)) {
                JUMPTO(oparg);
                DISPATCH();
            }
            err = PyObject_IsTrue(cond);
            if (err > 0) {
                JUMPTO(oparg);
            }
            else if (err == 0) {
                STACK_SHRINK(1);
                Py_DECREF(cond);
            }
            else
                goto error;
            DISPATCH();
        }

        TARGET(JUMP_ABSOLUTE) {
            PREDICTED(JUMP_ABSOLUTE);
            assert(oparg < INSTR_OFFSET());
            int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
            if (err) {
                if (err < 0) {
                    goto error;
                }
                /* Update first_instr and next_instr to point to newly quickened code */
                int nexti = INSTR_OFFSET();
                first_instr = frame->f_code->co_firstinstr;
                next_instr = first_instr + nexti;
            }
            JUMPTO(oparg);
            CHECK_EVAL_BREAKER();
            DISPATCH();
        }

        TARGET(JUMP_ABSOLUTE_QUICK) {
            assert(oparg < INSTR_OFFSET());
            JUMPTO(oparg);
            CHECK_EVAL_BREAKER();
            DISPATCH();
        }

        TARGET(GET_LEN) {
            // PUSH(len(TOS))
            Py_ssize_t len_i = PyObject_Length(TOP());
            if (len_i < 0) {
                goto error;
            }
            PyObject *len_o = PyLong_FromSsize_t(len_i);
            if (len_o == NULL) {
                goto error;
            }
            PUSH(len_o);
            DISPATCH();
        }

        TARGET(MATCH_CLASS) {
            // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
            // None on failure.
            PyObject *names = POP();
            PyObject *type = POP();
            PyObject *subject = TOP();
            assert(PyTuple_CheckExact(names));
            PyObject *attrs = match_class(tstate, subject, type, oparg, names);
            Py_DECREF(names);
            Py_DECREF(type);
            if (attrs) {
                // Success!
                assert(PyTuple_CheckExact(attrs));
                SET_TOP(attrs);
            }
            else if (_PyErr_Occurred(tstate)) {
                // Error!
                goto error;
            }
            else {
                // Failure!
                Py_INCREF(Py_None);
                SET_TOP(Py_None);
            }
            Py_DECREF(subject);
            DISPATCH();
        }

        TARGET(MATCH_MAPPING) {
            PyObject *subject = TOP();
            int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
            PyObject *res = match ? Py_True : Py_False;
            Py_INCREF(res);
            PUSH(res);
            PREDICT(POP_JUMP_IF_FALSE);
            DISPATCH();
        }

        TARGET(MATCH_SEQUENCE) {
            PyObject *subject = TOP();
            int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
            PyObject *res = match ? Py_True : Py_False;
            Py_INCREF(res);
            PUSH(res);
            PREDICT(POP_JUMP_IF_FALSE);
            DISPATCH();
        }

        TARGET(MATCH_KEYS) {
            // On successful match, PUSH(values). Otherwise, PUSH(None).
            PyObject *keys = TOP();
            PyObject *subject = SECOND();
            PyObject *values_or_none = match_keys(tstate, subject, keys);
            if (values_or_none == NULL) {
                goto error;
            }
            PUSH(values_or_none);
            DISPATCH();
        }

        TARGET(GET_ITER) {
            /* before: [obj]; after [getiter(obj)] */
            PyObject *iterable = TOP();
            PyObject *iter = PyObject_GetIter(iterable);
            Py_DECREF(iterable);
            SET_TOP(iter);
            if (iter == NULL)
                goto error;
            PREDICT(FOR_ITER);
            PREDICT(CALL_NO_KW);
            DISPATCH();
        }

        TARGET(GET_YIELD_FROM_ITER) {
            /* before: [obj]; after [getiter(obj)] */
            PyObject *iterable = TOP();
            PyObject *iter;
            if (PyCoro_CheckExact(iterable)) {
                /* `iterable` is a coroutine */
                if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
                    /* and it is used in a 'yield from' expression of a
                       regular generator. */
                    Py_DECREF(iterable);
                    SET_TOP(NULL);
                    _PyErr_SetString(tstate, PyExc_TypeError,
                                     "cannot 'yield from' a coroutine object "
                                     "in a non-coroutine generator");
                    goto error;
                }
            }
            else if (!PyGen_CheckExact(iterable)) {
                /* `iterable` is not a generator. */
                iter = PyObject_GetIter(iterable);
                Py_DECREF(iterable);
                SET_TOP(iter);
                if (iter == NULL)
                    goto error;
            }
            PREDICT(LOAD_CONST);
            DISPATCH();
        }

        TARGET(FOR_ITER) {
            PREDICTED(FOR_ITER);
            /* before: [iter]; after: [iter, iter()] *or* [] */
            PyObject *iter = TOP();
            PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
            if (next != NULL) {
                PUSH(next);
                PREDICT(STORE_FAST);
                PREDICT(UNPACK_SEQUENCE);
                DISPATCH();
            }
            if (_PyErr_Occurred(tstate)) {
                if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
                    goto error;
                }
                else if (tstate->c_tracefunc != NULL) {
                    call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
                }
                _PyErr_Clear(tstate);
            }
            /* iterator ended normally */
            STACK_SHRINK(1);
            Py_DECREF(iter);
            JUMPBY(oparg);
            DISPATCH();
        }

        TARGET(BEFORE_ASYNC_WITH) {
            _Py_IDENTIFIER(__aenter__);
            _Py_IDENTIFIER(__aexit__);
            PyObject *mgr = TOP();
            PyObject *res;
            PyObject *enter = _PyObject_LookupSpecial(mgr, &PyId___aenter__);
            if (enter == NULL) {
                if (!_PyErr_Occurred(tstate)) {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "'%.200s' object does not support the "
                                  "asynchronous context manager protocol",
                                  Py_TYPE(mgr)->tp_name);
                }
                goto error;
            }
            PyObject *exit = _PyObject_LookupSpecial(mgr, &PyId___aexit__);
            if (exit == NULL) {
                if (!_PyErr_Occurred(tstate)) {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "'%.200s' object does not support the "
                                  "asynchronous context manager protocol "
                                  "(missed __aexit__ method)",
                                  Py_TYPE(mgr)->tp_name);
                }
                Py_DECREF(enter);
                goto error;
            }
            SET_TOP(exit);
            Py_DECREF(mgr);
            res = _PyObject_CallNoArgs(enter);
            Py_DECREF(enter);
            if (res == NULL)
                goto error;
            PUSH(res);
            PREDICT(GET_AWAITABLE);
            DISPATCH();
        }

        TARGET(BEFORE_WITH) {
            _Py_IDENTIFIER(__enter__);
            _Py_IDENTIFIER(__exit__);
            PyObject *mgr = TOP();
            PyObject *res;
            PyObject *enter = _PyObject_LookupSpecial(mgr, &PyId___enter__);
            if (enter == NULL) {
                if (!_PyErr_Occurred(tstate)) {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "'%.200s' object does not support the "
                                  "context manager protocol",
                                  Py_TYPE(mgr)->tp_name);
                }
                goto error;
            }
            PyObject *exit = _PyObject_LookupSpecial(mgr, &PyId___exit__);
            if (exit == NULL) {
                if (!_PyErr_Occurred(tstate)) {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "'%.200s' object does not support the "
                                  "context manager protocol "
                                  "(missed __exit__ method)",
                                  Py_TYPE(mgr)->tp_name);
                }
                Py_DECREF(enter);
                goto error;
            }
            SET_TOP(exit);
            Py_DECREF(mgr);
            res = _PyObject_CallNoArgs(enter);
            Py_DECREF(enter);
            if (res == NULL) {
                goto error;
            }
            PUSH(res);
            DISPATCH();
        }

        TARGET(WITH_EXCEPT_START) {
            /* At the top of the stack are 4 values:
               - TOP = exc_info()
               - SECOND = previous exception
               - THIRD: lasti of exception in exc_info()
               - FOURTH: the context.__exit__ bound method
               We call FOURTH(type(TOP), TOP, GetTraceback(TOP)).
               Then we push the __exit__ return value.
            */
            PyObject *exit_func;
            PyObject *exc, *val, *tb, *res;

            val = TOP();
            assert(val && PyExceptionInstance_Check(val));
            exc = PyExceptionInstance_Class(val);
            tb = PyException_GetTraceback(val);
            Py_XDECREF(tb);
            assert(PyLong_Check(PEEK(3)));
            exit_func = PEEK(4);
            PyObject *stack[4] = {NULL, exc, val, tb};
            res = PyObject_Vectorcall(exit_func, stack + 1,
                    3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
            if (res == NULL)
                goto error;

            PUSH(res);
            DISPATCH();
        }

        TARGET(PUSH_EXC_INFO) {
            PyObject *value = TOP();

            _PyErr_StackItem *exc_info = tstate->exc_info;
            if (exc_info->exc_value != NULL) {
                SET_TOP(exc_info->exc_value);
            }
            else {
                Py_INCREF(Py_None);
                SET_TOP(Py_None);
            }

            Py_INCREF(value);
            PUSH(value);
            assert(PyExceptionInstance_Check(value));
            exc_info->exc_value = value;

            DISPATCH();
        }

        TARGET(LOAD_METHOD) {
            PREDICTED(LOAD_METHOD);
            /* Designed to work in tandem with CALL_METHOD. */
            PyObject *name = GETITEM(names, oparg);
            PyObject *obj = TOP();
            PyObject *meth = NULL;

            int meth_found = _PyObject_GetMethod(obj, name, &meth);

            if (meth == NULL) {
                /* Most likely attribute wasn't found. */
                goto error;
            }

            if (meth_found) {
                /* We can bypass temporary bound method object.
                   meth is unbound method and obj is self.

                   meth | self | arg1 | ... | argN
                 */
                SET_TOP(meth);
                PUSH(obj);  // self
            }
            else {
                /* meth is not an unbound method (but a regular attr, or
                   something was returned by a descriptor protocol).  Set
                   the second element of the stack to NULL, to signal
                   CALL_METHOD that it's not a method call.

                   NULL | meth | arg1 | ... | argN
                */
                SET_TOP(NULL);
                Py_DECREF(obj);
                PUSH(meth);
            }
            DISPATCH();
        }

        TARGET(LOAD_METHOD_ADAPTIVE) {
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *owner = TOP();
                PyObject *name = GETITEM(names, cache->adaptive.original_oparg);
                next_instr--;
                if (_Py_Specialize_LoadMethod(owner, next_instr, name, cache) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(LOAD_METHOD, deferred);
                cache->adaptive.counter--;
                oparg = cache->adaptive.original_oparg;
                JUMP_TO_INSTRUCTION(LOAD_METHOD);
            }
        }

        TARGET(LOAD_METHOD_CACHED) {
            /* LOAD_METHOD, with cached method object */
            assert(cframe.use_tracing == 0);
            PyObject *self = TOP();
            PyTypeObject *self_cls = Py_TYPE(self);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAttrCache *cache1 = &caches[-1].attr;
            _PyObjectCache *cache2 = &caches[-2].obj;

            DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD);
            assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
            PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self);
            DEOPT_IF(dict != NULL, LOAD_METHOD);
            DEOPT_IF(((PyHeapTypeObject *)self_cls)->ht_cached_keys->dk_version != cache1->dk_version_or_hint, LOAD_METHOD);
            STAT_INC(LOAD_METHOD, hit);
            PyObject *res = cache2->obj;
            assert(res != NULL);
            assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
            Py_INCREF(res);
            SET_TOP(res);
            PUSH(self);
            DISPATCH();
        }

        TARGET(LOAD_METHOD_NO_DICT) {
            PyObject *self = TOP();
            PyTypeObject *self_cls = Py_TYPE(self);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAttrCache *cache1 = &caches[-1].attr;
            _PyObjectCache *cache2 = &caches[-2].obj;
            DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD);
            assert(self_cls->tp_dictoffset == 0);
            STAT_INC(LOAD_METHOD, hit);
            PyObject *res = cache2->obj;
            assert(res != NULL);
            assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
            Py_INCREF(res);
            SET_TOP(res);
            PUSH(self);
            DISPATCH();
        }

        TARGET(LOAD_METHOD_MODULE) {
            /* LOAD_METHOD, for module methods */
            assert(cframe.use_tracing == 0);
            PyObject *owner = TOP();
            PyObject *res;
            LOAD_MODULE_ATTR_OR_METHOD(METHOD);
            SET_TOP(NULL);
            Py_DECREF(owner);
            PUSH(res);
            DISPATCH();
        }

        TARGET(LOAD_METHOD_CLASS) {
            /* LOAD_METHOD, for class methods */
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAttrCache *cache1 = &caches[-1].attr;
            _PyObjectCache *cache2 = &caches[-2].obj;

            PyObject *cls = TOP();
            DEOPT_IF(!PyType_Check(cls), LOAD_METHOD);
            DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != cache1->tp_version,
                LOAD_METHOD);
            assert(cache1->tp_version != 0);

            STAT_INC(LOAD_METHOD, hit);
            PyObject *res = cache2->obj;
            assert(res != NULL);
            Py_INCREF(res);
            SET_TOP(NULL);
            Py_DECREF(cls);
            PUSH(res);
            DISPATCH();
        }

        TARGET(PRECALL_METHOD) {
            /* Designed to work in tamdem with LOAD_METHOD. */
            /* `meth` is NULL when LOAD_METHOD thinks that it's not
                a method call.

                Stack layout:

                       ... | NULL | callable | arg1 | ... | argN
                                                            ^- TOP()
                                               ^- (-oparg)
                                    ^- (-oparg-1)
                             ^- (-oparg-2)

                `callable` will be POPed by call_function.
                NULL will will be POPed manually later.
                If `meth` isn't NULL, it's a method call.  Stack layout:

                     ... | method | self | arg1 | ... | argN
                                                        ^- TOP()
                                           ^- (-oparg)
                                    ^- (-oparg-1)
                           ^- (-oparg-2)

               `self` and `method` will be POPed by call_function.
               We'll be passing `oparg + 1` to call_function, to
               make it accept the `self` as a first argument.
            */
            int is_method = (PEEK(oparg + 2) != NULL);
            extra_args = is_method;
            postcall_shrink = 2-is_method;
            DISPATCH();
        }

        TARGET(CALL_KW) {
            kwnames = POP();
            oparg += extra_args;
            extra_args = 0;
            nargs = oparg - (int)PyTuple_GET_SIZE(kwnames);
            goto call_function;
        }

        TARGET(CALL_NO_KW) {
            PyObject *function;
            PREDICTED(CALL_NO_KW);
            kwnames = NULL;
            oparg += extra_args;
            nargs = oparg;
        call_function:
            function = PEEK(oparg + 1);
            if (Py_TYPE(function) == &PyMethod_Type) {
                PyObject *meth = ((PyMethodObject *)function)->im_func;
                PyObject *self = ((PyMethodObject *)function)->im_self;
                Py_INCREF(meth);
                Py_INCREF(self);
                PEEK(oparg + 1) = self;
                Py_DECREF(function);
                function = meth;
                oparg++;
                nargs++;
                assert(postcall_shrink >= 1);
                postcall_shrink--;
            }
            // Check if the call can be inlined or not
            if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) {
                int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
                int is_generator = code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR);
                if (!is_generator) {
                    PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
                    STACK_SHRINK(oparg);
                    InterpreterFrame *new_frame = _PyEvalFramePushAndInit(
                        tstate, (PyFunctionObject *)function, locals,
                        stack_pointer, nargs, kwnames
                    );
                    STACK_SHRINK(postcall_shrink);
                    RESET_STACK_ADJUST_FOR_CALLS;
                    // The frame has stolen all the arguments from the stack,
                    // so there is no need to clean them up.
                    Py_XDECREF(kwnames);
                    Py_DECREF(function);
                    if (new_frame == NULL) {
                        goto error;
                    }
                    _PyFrame_SetStackPointer(frame, stack_pointer);
                    new_frame->previous = frame;
                    cframe.current_frame = frame = new_frame;
                    new_frame->depth = frame->depth + 1;
                    goto start_frame;
                }
            }
            /* Callable is not a normal Python function */
            PyObject *res;
            if (cframe.use_tracing) {
                res = trace_call_function(tstate, function, stack_pointer-oparg, nargs, kwnames);
            }
            else {
                res = PyObject_Vectorcall(function, stack_pointer-oparg,
                                          nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
            }
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
            Py_DECREF(function);
            Py_XDECREF(kwnames);
            /* Clear the stack */
            STACK_SHRINK(oparg);
            for (int i = 0; i < oparg; i++) {
                Py_DECREF(stack_pointer[i]);
            }
            STACK_SHRINK(postcall_shrink);
            RESET_STACK_ADJUST_FOR_CALLS;
            PUSH(res);
            if (res == NULL) {
                goto error;
            }
            CHECK_EVAL_BREAKER();
            DISPATCH();
        }

        TARGET(CALL_NO_KW_ADAPTIVE) {
            SpecializedCacheEntry *cache = GET_CACHE();
            oparg = cache->adaptive.original_oparg;
            if (cache->adaptive.counter == 0) {
                next_instr--;
                int nargs = oparg+extra_args;
                if (_Py_Specialize_CallNoKw(
                    PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) {
                    goto error;
                }
                DISPATCH();
            }
            else {
                STAT_INC(CALL_NO_KW, deferred);
                cache->adaptive.counter--;
                kwnames = NULL;
                oparg += extra_args;
                nargs = oparg;
                goto call_function;
            }
        }

        TARGET(CALL_NO_KW_PY_SIMPLE) {
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            int argcount = cache0->original_oparg + extra_args;
            DEOPT_IF(argcount != cache0->index, CALL_NO_KW);
            _PyCallCache *cache1 = &caches[-1].call;
            PyObject *callable = PEEK(argcount+1);
            DEOPT_IF(!PyFunction_Check(callable), CALL_NO_KW);
            PyFunctionObject *func = (PyFunctionObject *)callable;
            DEOPT_IF(func->func_version != cache1->func_version, CALL_NO_KW);
            /* PEP 523 */
            DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);
            PyCodeObject *code = (PyCodeObject *)func->func_code;
            size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
            InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
            if (new_frame == NULL) {
                RESET_STACK_ADJUST_FOR_CALLS;
                goto error;
            }
            _PyFrame_InitializeSpecials(new_frame, func,
                                        NULL, code->co_nlocalsplus);
            STACK_SHRINK(argcount);
            for (int i = 0; i < argcount; i++) {
                new_frame->localsplus[i] = stack_pointer[i];
            }
            int deflen = cache1->defaults_len;
            for (int i = 0; i < deflen; i++) {
                PyObject *def = PyTuple_GET_ITEM(func->func_defaults, cache1->defaults_start+i);
                Py_INCREF(def);
                new_frame->localsplus[argcount+i] = def;
            }
            for (int i = argcount+deflen; i < code->co_nlocalsplus; i++) {
                new_frame->localsplus[i] = NULL;
            }
            STACK_SHRINK(postcall_shrink);
            RESET_STACK_ADJUST_FOR_CALLS;
            Py_DECREF(func);
            _PyFrame_SetStackPointer(frame, stack_pointer);
            new_frame->previous = frame;
            frame = cframe.current_frame = new_frame;
            new_frame->depth = frame->depth + 1;
            goto start_frame;
        }

        TARGET(CALL_NO_KW_TYPE_1) {
            assert(STACK_ADJUST_IS_RESET);
            assert(GET_CACHE()->adaptive.original_oparg == 1);
            PyObject *obj = TOP();
            PyObject *callable = SECOND();
            DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL_NO_KW);
            PyObject *res = Py_NewRef(Py_TYPE(obj));
            STACK_SHRINK(1);
            Py_DECREF(callable);
            Py_DECREF(obj);
            SET_TOP(res);
            DISPATCH();
        }

        TARGET(CALL_NO_KW_BUILTIN_CLASS_1) {
            assert(STACK_ADJUST_IS_RESET);
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            assert(cache0->original_oparg == 1);
            PyObject *callable = SECOND();
            PyObject *arg = TOP();
            DEOPT_IF(!PyType_Check(callable), CALL_NO_KW);
            PyTypeObject *tp = (PyTypeObject *)callable;
            DEOPT_IF(tp->tp_version_tag != cache0->version, CALL_NO_KW);
            STACK_SHRINK(1);
            PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, 1, NULL);
            SET_TOP(res);
            Py_DECREF(tp);
            Py_DECREF(arg);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_NO_KW_BUILTIN_O) {
            assert(cframe.use_tracing == 0);
            assert(STACK_ADJUST_IS_RESET);
            /* Builtin METH_O functions */

            PyObject *callable = SECOND();
            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
            DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);

            PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
            // This is slower but CPython promises to check all non-vectorcall
            // function calls.
            if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
                goto error;
            }
            PyObject *arg = POP();
            PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg);
            _Py_LeaveRecursiveCall(tstate);
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));

            /* Clear the stack of the function object. */
            Py_DECREF(arg);
            Py_DECREF(callable);
            SET_TOP(res);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_NO_KW_BUILTIN_FAST) {
            assert(cframe.use_tracing == 0);
            assert(STACK_ADJUST_IS_RESET);
            /* Builtin METH_FASTCALL functions, without keywords */
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            int nargs = cache0->original_oparg;
            PyObject **pfunc = &PEEK(nargs + 1);
            PyObject *callable = *pfunc;
            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
            DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
                CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);

            PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
            /* res = func(self, args, nargs) */
            PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
                PyCFunction_GET_SELF(callable),
                &PEEK(nargs),
                nargs);
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));

            /* Clear the stack of the function object. */
            while (stack_pointer > pfunc) {
                PyObject *x = POP();
                Py_DECREF(x);
            }
            PUSH(res);
            if (res == NULL) {
                /* Not deopting because this doesn't mean our optimization was
                   wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
                   'invalid'). In those cases an exception is set, so we must
                   handle it.
                */
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_NO_KW_LEN) {
            assert(cframe.use_tracing == 0);
            assert(STACK_ADJUST_IS_RESET);
            /* len(o) */
            SpecializedCacheEntry *caches = GET_CACHE();
            assert(caches[0].adaptive.original_oparg == 1);
            _PyObjectCache *cache1 = &caches[-1].obj;

            PyObject *callable = SECOND();
            DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);

            Py_ssize_t len_i = PyObject_Length(TOP());
            if (len_i < 0) {
                goto error;
            }
            PyObject *res = PyLong_FromSsize_t(len_i);
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));

            /* Clear the stack of the function object. */
            Py_DECREF(POP());
            Py_DECREF(callable);
            SET_TOP(res);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_NO_KW_ISINSTANCE) {
            assert(cframe.use_tracing == 0);
            assert(STACK_ADJUST_IS_RESET);
            /* isinstance(o, o2) */
            SpecializedCacheEntry *caches = GET_CACHE();
            assert(caches[0].adaptive.original_oparg == 2);
            _PyObjectCache *cache1 = &caches[-1].obj;

            PyObject *callable = THIRD();
            DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);

            int retval = PyObject_IsInstance(SECOND(), TOP());
            if (retval < 0) {
                goto error;
            }
            PyObject *res = PyBool_FromLong(retval);
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));

            /* Clear the stack of the function object. */
            Py_DECREF(POP());
            Py_DECREF(POP());
            Py_DECREF(callable);
            SET_TOP(res);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_NO_KW_LIST_APPEND) {
            assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
            assert(GET_CACHE()->adaptive.original_oparg == 1);
            DEOPT_IF(extra_args == 0, CALL_NO_KW);
            PyObject *list = SECOND();
            DEOPT_IF(!PyList_CheckExact(list), CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);
            assert(extra_args == 1);
            extra_args = 0;
            assert(STACK_ADJUST_IS_RESET);
            PyObject *arg = TOP();
            int err = PyList_Append(list, arg);
            if (err) {
                goto error;
            }
            PyObject *callable = THIRD();
            Py_DECREF(arg);
            Py_DECREF(list);
            Py_INCREF(Py_None);
            STACK_SHRINK(2);
            SET_TOP(Py_None);
            Py_DECREF(callable);
            DISPATCH();
        }

        TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
            assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
            assert(GET_CACHE()->adaptive.original_oparg == 1);
            DEOPT_IF(extra_args == 0, CALL_NO_KW);
            assert(extra_args == 1);
            PyObject *callable = THIRD();
            DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW);
            DEOPT_IF(((PyMethodDescrObject *)callable)->d_method->ml_flags != METH_O, CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);
            assert(extra_args == 1);
            extra_args = 0;
            assert(STACK_ADJUST_IS_RESET);
            PyCFunction cfunc = ((PyMethodDescrObject *)callable)->d_method->ml_meth;
            // This is slower but CPython promises to check all non-vectorcall
            // function calls.
            if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
                goto error;
            }
            PyObject *arg = POP();
            PyObject *self = POP();
            PyObject *res = cfunc(self, arg);
            _Py_LeaveRecursiveCall(tstate);
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
            Py_DECREF(self);
            Py_DECREF(arg);
            SET_TOP(res);
            Py_DECREF(callable);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
            assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
            /* Builtin METH_FASTCALL methods, without keywords */
            SpecializedCacheEntry *caches = GET_CACHE();
            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
            DEOPT_IF(extra_args == 0, CALL_NO_KW);
            assert(extra_args == 1);
            int nargs = cache0->original_oparg;
            PyObject *callable = PEEK(nargs + 2);
            DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type),  CALL_NO_KW);
            PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
            DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL_NO_KW);
            STAT_INC(CALL_NO_KW, hit);
            assert(extra_args == 1);
            extra_args = 0;
            assert(STACK_ADJUST_IS_RESET);
            _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
            PyObject *self = PEEK(nargs+1);
            PyObject *res = cfunc(self, &PEEK(nargs), nargs);
            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
            /* Clear the stack of the arguments. */
            STACK_SHRINK(nargs+1);
            for (int i = 0; i <= nargs; i++) {
                Py_DECREF(stack_pointer[i]);
            }
            SET_TOP(res);
            Py_DECREF(callable);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(CALL_FUNCTION_EX) {
            PREDICTED(CALL_FUNCTION_EX);
            PyObject *func, *callargs, *kwargs = NULL, *result;
            if (oparg & 0x01) {
                kwargs = POP();
                if (!PyDict_CheckExact(kwargs)) {
                    PyObject *d = PyDict_New();
                    if (d == NULL)
                        goto error;
                    if (_PyDict_MergeEx(d, kwargs, 2) < 0) {
                        Py_DECREF(d);
                        format_kwargs_error(tstate, SECOND(), kwargs);
                        Py_DECREF(kwargs);
                        goto error;
                    }
                    Py_DECREF(kwargs);
                    kwargs = d;
                }
                assert(PyDict_CheckExact(kwargs));
            }
            callargs = POP();
            func = TOP();
            if (!PyTuple_CheckExact(callargs)) {
                if (check_args_iterable(tstate, func, callargs) < 0) {
                    Py_DECREF(callargs);
                    goto error;
                }
                Py_SETREF(callargs, PySequence_Tuple(callargs));
                if (callargs == NULL) {
                    goto error;
                }
            }
            assert(PyTuple_CheckExact(callargs));

            result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing);
            Py_DECREF(func);
            Py_DECREF(callargs);
            Py_XDECREF(kwargs);

            SET_TOP(result);
            if (result == NULL) {
                goto error;
            }
            CHECK_EVAL_BREAKER();
            DISPATCH();
        }

        TARGET(MAKE_FUNCTION) {
            PyObject *codeobj = POP();
            PyFunctionObject *func = (PyFunctionObject *)
                PyFunction_New(codeobj, GLOBALS());

            Py_DECREF(codeobj);
            if (func == NULL) {
                goto error;
            }

            if (oparg & 0x08) {
                assert(PyTuple_CheckExact(TOP()));
                func->func_closure = POP();
            }
            if (oparg & 0x04) {
                assert(PyTuple_CheckExact(TOP()));
                func->func_annotations = POP();
            }
            if (oparg & 0x02) {
                assert(PyDict_CheckExact(TOP()));
                func->func_kwdefaults = POP();
            }
            if (oparg & 0x01) {
                assert(PyTuple_CheckExact(TOP()));
                func->func_defaults = POP();
            }

            PUSH((PyObject *)func);
            DISPATCH();
        }

        TARGET(BUILD_SLICE) {
            PyObject *start, *stop, *step, *slice;
            if (oparg == 3)
                step = POP();
            else
                step = NULL;
            stop = POP();
            start = TOP();
            slice = PySlice_New(start, stop, step);
            Py_DECREF(start);
            Py_DECREF(stop);
            Py_XDECREF(step);
            SET_TOP(slice);
            if (slice == NULL)
                goto error;
            DISPATCH();
        }

        TARGET(FORMAT_VALUE) {
            /* Handles f-string value formatting. */
            PyObject *result;
            PyObject *fmt_spec;
            PyObject *value;
            PyObject *(*conv_fn)(PyObject *);
            int which_conversion = oparg & FVC_MASK;
            int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC;

            fmt_spec = have_fmt_spec ? POP() : NULL;
            value = POP();

            /* See if any conversion is specified. */
            switch (which_conversion) {
            case FVC_NONE:  conv_fn = NULL;           break;
            case FVC_STR:   conv_fn = PyObject_Str;   break;
            case FVC_REPR:  conv_fn = PyObject_Repr;  break;
            case FVC_ASCII: conv_fn = PyObject_ASCII; break;
            default:
                _PyErr_Format(tstate, PyExc_SystemError,
                              "unexpected conversion flag %d",
                              which_conversion);
                goto error;
            }

            /* If there's a conversion function, call it and replace
               value with that result. Otherwise, just use value,
               without conversion. */
            if (conv_fn != NULL) {
                result = conv_fn(value);
                Py_DECREF(value);
                if (result == NULL) {
                    Py_XDECREF(fmt_spec);
                    goto error;
                }
                value = result;
            }

            /* If value is a unicode object, and there's no fmt_spec,
               then we know the result of format(value) is value
               itself. In that case, skip calling format(). I plan to
               move this optimization in to PyObject_Format()
               itself. */
            if (PyUnicode_CheckExact(value) && fmt_spec == NULL) {
                /* Do nothing, just transfer ownership to result. */
                result = value;
            } else {
                /* Actually call format(). */
                result = PyObject_Format(value, fmt_spec);
                Py_DECREF(value);
                Py_XDECREF(fmt_spec);
                if (result == NULL) {
                    goto error;
                }
            }

            PUSH(result);
            DISPATCH();
        }

        TARGET(ROT_N) {
            PyObject *top = TOP();
            memmove(&PEEK(oparg - 1), &PEEK(oparg),
                    sizeof(PyObject*) * (oparg - 1));
            PEEK(oparg) = top;
            DISPATCH();
        }

        TARGET(COPY) {
            assert(oparg != 0);
            PyObject *peek = PEEK(oparg);
            Py_INCREF(peek);
            PUSH(peek);
            DISPATCH();
        }

        TARGET(BINARY_OP) {
            PREDICTED(BINARY_OP);
            PyObject *rhs = POP();
            PyObject *lhs = TOP();
            assert(0 <= oparg);
            assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
            assert(binary_ops[oparg]);
            PyObject *res = binary_ops[oparg](lhs, rhs);
            Py_DECREF(lhs);
            Py_DECREF(rhs);
            SET_TOP(res);
            if (res == NULL) {
                goto error;
            }
            DISPATCH();
        }

        TARGET(BINARY_OP_ADAPTIVE) {
            assert(cframe.use_tracing == 0);
            SpecializedCacheEntry *cache = GET_CACHE();
            if (cache->adaptive.counter == 0) {
                PyObject *lhs = SECOND();
                PyObject *rhs = TOP();
                next_instr--;
                _Py_Specialize_BinaryOp(lhs, rhs, next_instr, cache);
                DISPATCH();
            }
            else {
                STAT_INC(BINARY_OP, deferred);
                cache->adaptive.counter--;
                oparg = cache->adaptive.original_oparg;
                JUMP_TO_INSTRUCTION(BINARY_OP);
            }
        }

        TARGET(EXTENDED_ARG) {
            int oldoparg = oparg;
            NEXTOPARG();
            oparg |= oldoparg << 8;
            PRE_DISPATCH_GOTO();
            DISPATCH_GOTO();
        }

#if USE_COMPUTED_GOTOS
        TARGET_DO_TRACING: {
#else
        case DO_TRACING: {
#endif
            int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti);
            frame->f_lasti = INSTR_OFFSET();
            TRACING_NEXTOPARG();
            if (PyDTrace_LINE_ENABLED()) {
                maybe_dtrace_line(frame, &tstate->trace_info, instr_prev);
            }
            /* line-by-line tracing support */

            if (cframe.use_tracing &&
                tstate->c_tracefunc != NULL && !tstate->tracing) {
                int err;
                /* see maybe_call_line_trace()
                for expository comments */
                _PyFrame_SetStackPointer(frame, stack_pointer);

                err = maybe_call_line_trace(tstate->c_tracefunc,
                                            tstate->c_traceobj,
                                            tstate, frame, instr_prev);
                if (err) {
                    /* trace function raised an exception */
                    next_instr++;
                    goto error;
                }
                /* Reload possibly changed frame fields */
                JUMPTO(frame->f_lasti);

                stack_pointer = _PyFrame_GetStackPointer(frame);
                frame->stacktop = -1;
                TRACING_NEXTOPARG();
            }
            PRE_DISPATCH_GOTO();
            DISPATCH_GOTO();
        }


#if USE_COMPUTED_GOTOS
        _unknown_opcode:
#else
        default:
#endif
            fprintf(stderr,
                "XXX lineno: %d, opcode: %d\n",
                PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT)),
                opcode);
            _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode");
            goto error;

        } /* End instructions */

        /* This should never be reached. Every opcode should end with DISPATCH()
           or goto error. */
        Py_UNREACHABLE();

/* Specialization misses */

#define MISS_WITH_CACHE(opname) \
opname ## _miss: \
    { \
        STAT_INC(opname, miss); \
        _PyAdaptiveEntry *cache = &GET_CACHE()->adaptive; \
        cache->counter--; \
        if (cache->counter == 0) { \
            next_instr[-1] = _Py_MAKECODEUNIT(opname ## _ADAPTIVE, _Py_OPARG(next_instr[-1])); \
            STAT_INC(opname, deopt); \
            cache_backoff(cache); \
        } \
        oparg = cache->original_oparg; \
        JUMP_TO_INSTRUCTION(opname); \
    }

#define MISS_WITH_OPARG_COUNTER(opname) \
opname ## _miss: \
    { \
        STAT_INC(opname, miss); \
        uint8_t oparg = _Py_OPARG(next_instr[-1])-1; \
        UPDATE_PREV_INSTR_OPARG(next_instr, oparg); \
        assert(_Py_OPARG(next_instr[-1]) == oparg); \
        if (oparg == 0) /* too many cache misses */ { \
            oparg = ADAPTIVE_CACHE_BACKOFF; \
            next_instr[-1] = _Py_MAKECODEUNIT(opname ## _ADAPTIVE, oparg); \
            STAT_INC(opname, deopt); \
        } \
        JUMP_TO_INSTRUCTION(opname); \
    }

MISS_WITH_CACHE(LOAD_ATTR)
MISS_WITH_CACHE(STORE_ATTR)
MISS_WITH_CACHE(LOAD_GLOBAL)
MISS_WITH_CACHE(LOAD_METHOD)
MISS_WITH_CACHE(CALL_NO_KW)
MISS_WITH_CACHE(BINARY_OP)
MISS_WITH_CACHE(COMPARE_OP)
MISS_WITH_CACHE(BINARY_SUBSCR)
MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)

binary_subscr_dict_error:
        {
            PyObject *sub = POP();
            if (!_PyErr_Occurred(tstate)) {
                _PyErr_SetKeyError(sub);
            }
            Py_DECREF(sub);
            goto error;
        }

unbound_local_error:
        {
            format_exc_check_arg(tstate, PyExc_UnboundLocalError,
                UNBOUNDLOCAL_ERROR_MSG,
                PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
            );
            goto error;
        }

error:
        /* Double-check exception status. */
#ifdef NDEBUG
        if (!_PyErr_Occurred(tstate)) {
            _PyErr_SetString(tstate, PyExc_SystemError,
                             "error return without exception set");
        }
#else
        assert(_PyErr_Occurred(tstate));
#endif

        /* Log traceback info. */
        PyFrameObject *f = _PyFrame_GetFrameObject(frame);
        if (f != NULL) {
            PyTraceBack_Here(f);
        }

        if (tstate->c_tracefunc != NULL) {
            /* Make sure state is set to FRAME_UNWINDING for tracing */
            frame->f_state = FRAME_UNWINDING;
            call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
                           tstate, frame);
        }

exception_unwind:
        frame->f_state = FRAME_UNWINDING;
        /* We can't use frame->f_lasti here, as RERAISE may have set it */
        int offset = INSTR_OFFSET()-1;
        int level, handler, lasti;
        if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) {
            // No handlers, so exit.
            assert(_PyErr_Occurred(tstate));

            /* Pop remaining stack entries. */
            PyObject **stackbase = _PyFrame_Stackbase(frame);
            while (stack_pointer > stackbase) {
                PyObject *o = POP();
                Py_XDECREF(o);
            }
            assert(STACK_LEVEL() == 0);
            _PyFrame_SetStackPointer(frame, stack_pointer);
            frame->f_state = FRAME_RAISED;
            TRACE_FUNCTION_UNWIND();
            DTRACE_FUNCTION_EXIT();
            goto exit_unwind;
        }

        assert(STACK_LEVEL() >= level);
        PyObject **new_top = _PyFrame_Stackbase(frame) + level;
        while (stack_pointer > new_top) {
            PyObject *v = POP();
            Py_XDECREF(v);
        }
        PyObject *exc, *val, *tb;
        if (lasti) {
            PyObject *lasti = PyLong_FromLong(frame->f_lasti);
            if (lasti == NULL) {
                goto exception_unwind;
            }
            PUSH(lasti);
        }
        _PyErr_Fetch(tstate, &exc, &val, &tb);
        /* Make the raw exception data
            available to the handler,
            so a program can emulate the
            Python main loop. */
        _PyErr_NormalizeException(tstate, &exc, &val, &tb);
        if (tb != NULL)
            PyException_SetTraceback(val, tb);
        else
            PyException_SetTraceback(val, Py_None);
        Py_XDECREF(tb);
        Py_XDECREF(exc);
        PUSH(val);
        JUMPTO(handler);
        /* Resume normal execution */
        frame->f_state = FRAME_EXECUTING;
        DISPATCH();
    }

exit_unwind:
    assert(_PyErr_Occurred(tstate));
    _Py_LeaveRecursiveCall(tstate);
    if (frame->depth == 0) {
        /* Restore previous cframe and exit */
        tstate->cframe = cframe.previous;
        tstate->cframe->use_tracing = cframe.use_tracing;
        assert(tstate->cframe->current_frame == frame->previous);
        return NULL;
    }
    frame = cframe.current_frame = pop_frame(tstate, frame);

resume_with_error:
    SET_LOCALS_FROM_FRAME();
    goto error;

}

static void
format_missing(PyThreadState *tstate, const char *kind,
               PyCodeObject *co, PyObject *names, PyObject *qualname)
{
    int err;
    Py_ssize_t len = PyList_GET_SIZE(names);
    PyObject *name_str, *comma, *tail, *tmp;

    assert(PyList_CheckExact(names));
    assert(len >= 1);
    /* Deal with the joys of natural language. */
    switch (len) {
    case 1:
        name_str = PyList_GET_ITEM(names, 0);
        Py_INCREF(name_str);
        break;
    case 2:
        name_str = PyUnicode_FromFormat("%U and %U",
                                        PyList_GET_ITEM(names, len - 2),
                                        PyList_GET_ITEM(names, len - 1));
        break;
    default:
        tail = PyUnicode_FromFormat(", %U, and %U",
                                    PyList_GET_ITEM(names, len - 2),
                                    PyList_GET_ITEM(names, len - 1));
        if (tail == NULL)
            return;
        /* Chop off the last two objects in the list. This shouldn't actually
           fail, but we can't be too careful. */
        err = PyList_SetSlice(names, len - 2, len, NULL);
        if (err == -1) {
            Py_DECREF(tail);
            return;
        }
        /* Stitch everything up into a nice comma-separated list. */
        comma = PyUnicode_FromString(", ");
        if (comma == NULL) {
            Py_DECREF(tail);
            return;
        }
        tmp = PyUnicode_Join(comma, names);
        Py_DECREF(comma);
        if (tmp == NULL) {
            Py_DECREF(tail);
            return;
        }
        name_str = PyUnicode_Concat(tmp, tail);
        Py_DECREF(tmp);
        Py_DECREF(tail);
        break;
    }
    if (name_str == NULL)
        return;
    _PyErr_Format(tstate, PyExc_TypeError,
                  "%U() missing %i required %s argument%s: %U",
                  qualname,
                  len,
                  kind,
                  len == 1 ? "" : "s",
                  name_str);
    Py_DECREF(name_str);
}

static void
missing_arguments(PyThreadState *tstate, PyCodeObject *co,
                  Py_ssize_t missing, Py_ssize_t defcount,
                  PyObject **localsplus, PyObject *qualname)
{
    Py_ssize_t i, j = 0;
    Py_ssize_t start, end;
    int positional = (defcount != -1);
    const char *kind = positional ? "positional" : "keyword-only";
    PyObject *missing_names;

    /* Compute the names of the arguments that are missing. */
    missing_names = PyList_New(missing);
    if (missing_names == NULL)
        return;
    if (positional) {
        start = 0;
        end = co->co_argcount - defcount;
    }
    else {
        start = co->co_argcount;
        end = start + co->co_kwonlyargcount;
    }
    for (i = start; i < end; i++) {
        if (localsplus[i] == NULL) {
            PyObject *raw = PyTuple_GET_ITEM(co->co_localsplusnames, i);
            PyObject *name = PyObject_Repr(raw);
            if (name == NULL) {
                Py_DECREF(missing_names);
                return;
            }
            PyList_SET_ITEM(missing_names, j++, name);
        }
    }
    assert(j == missing);
    format_missing(tstate, kind, co, missing_names, qualname);
    Py_DECREF(missing_names);
}

static void
too_many_positional(PyThreadState *tstate, PyCodeObject *co,
                    Py_ssize_t given, PyObject *defaults,
                    PyObject **localsplus, PyObject *qualname)
{
    int plural;
    Py_ssize_t kwonly_given = 0;
    Py_ssize_t i;
    PyObject *sig, *kwonly_sig;
    Py_ssize_t co_argcount = co->co_argcount;

    assert((co->co_flags & CO_VARARGS) == 0);
    /* Count missing keyword-only args. */
    for (i = co_argcount; i < co_argcount + co->co_kwonlyargcount; i++) {
        if (localsplus[i] != NULL) {
            kwonly_given++;
        }
    }
    Py_ssize_t defcount = defaults == NULL ? 0 : PyTuple_GET_SIZE(defaults);
    if (defcount) {
        Py_ssize_t atleast = co_argcount - defcount;
        plural = 1;
        sig = PyUnicode_FromFormat("from %zd to %zd", atleast, co_argcount);
    }
    else {
        plural = (co_argcount != 1);
        sig = PyUnicode_FromFormat("%zd", co_argcount);
    }
    if (sig == NULL)
        return;
    if (kwonly_given) {
        const char *format = " positional argument%s (and %zd keyword-only argument%s)";
        kwonly_sig = PyUnicode_FromFormat(format,
                                          given != 1 ? "s" : "",
                                          kwonly_given,
                                          kwonly_given != 1 ? "s" : "");
        if (kwonly_sig == NULL) {
            Py_DECREF(sig);
            return;
        }
    }
    else {
        /* This will not fail. */
        kwonly_sig = PyUnicode_FromString("");
        assert(kwonly_sig != NULL);
    }
    _PyErr_Format(tstate, PyExc_TypeError,
                  "%U() takes %U positional argument%s but %zd%U %s given",
                  qualname,
                  sig,
                  plural ? "s" : "",
                  given,
                  kwonly_sig,
                  given == 1 && !kwonly_given ? "was" : "were");
    Py_DECREF(sig);
    Py_DECREF(kwonly_sig);
}

static int
positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co,
                                  Py_ssize_t kwcount, PyObject* kwnames,
                                  PyObject *qualname)
{
    int posonly_conflicts = 0;
    PyObject* posonly_names = PyList_New(0);

    for(int k=0; k < co->co_posonlyargcount; k++){
        PyObject* posonly_name = PyTuple_GET_ITEM(co->co_localsplusnames, k);

        for (int k2=0; k2<kwcount; k2++){
            /* Compare the pointers first and fallback to PyObject_RichCompareBool*/
            PyObject* kwname = PyTuple_GET_ITEM(kwnames, k2);
            if (kwname == posonly_name){
                if(PyList_Append(posonly_names, kwname) != 0) {
                    goto fail;
                }
                posonly_conflicts++;
                continue;
            }

            int cmp = PyObject_RichCompareBool(posonly_name, kwname, Py_EQ);

            if ( cmp > 0) {
                if(PyList_Append(posonly_names, kwname) != 0) {
                    goto fail;
                }
                posonly_conflicts++;
            } else if (cmp < 0) {
                goto fail;
            }

        }
    }
    if (posonly_conflicts) {
        PyObject* comma = PyUnicode_FromString(", ");
        if (comma == NULL) {
            goto fail;
        }
        PyObject* error_names = PyUnicode_Join(comma, posonly_names);
        Py_DECREF(comma);
        if (error_names == NULL) {
            goto fail;
        }
        _PyErr_Format(tstate, PyExc_TypeError,
                      "%U() got some positional-only arguments passed"
                      " as keyword arguments: '%U'",
                      qualname, error_names);
        Py_DECREF(error_names);
        goto fail;
    }

    Py_DECREF(posonly_names);
    return 0;

fail:
    Py_XDECREF(posonly_names);
    return 1;

}

/* Exception table parsing code.
 * See Objects/exception_table_notes.txt for details.
 */

static inline unsigned char *
parse_varint(unsigned char *p, int *result) {
    int val = p[0] & 63;
    while (p[0] & 64) {
        p++;
        val = (val << 6) | (p[0] & 63);
    }
    *result = val;
    return p+1;
}

static inline unsigned char *
scan_back_to_entry_start(unsigned char *p) {
    for (; (p[0]&128) == 0; p--);
    return p;
}

static inline unsigned char *
skip_to_next_entry(unsigned char *p, unsigned char *end) {
    while (p < end && ((p[0] & 128) == 0)) {
        p++;
    }
    return p;
}


#define MAX_LINEAR_SEARCH 40

static int
get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti)
{
    unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
    unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
    /* Invariants:
     * start_table == end_table OR
     * start_table points to a legal entry and end_table points
     * beyond the table or to a legal entry that is after index.
     */
    if (end - start > MAX_LINEAR_SEARCH) {
        int offset;
        parse_varint(start, &offset);
        if (offset > index) {
            return 0;
        }
        do {
            unsigned char * mid = start + ((end-start)>>1);
            mid = scan_back_to_entry_start(mid);
            parse_varint(mid, &offset);
            if (offset > index) {
                end = mid;
            }
            else {
                start = mid;
            }

        } while (end - start > MAX_LINEAR_SEARCH);
    }
    unsigned char *scan = start;
    while (scan < end) {
        int start_offset, size;
        scan = parse_varint(scan, &start_offset);
        if (start_offset > index) {
            break;
        }
        scan = parse_varint(scan, &size);
        if (start_offset + size > index) {
            scan = parse_varint(scan, handler);
            int depth_and_lasti;
            parse_varint(scan, &depth_and_lasti);
            *level = depth_and_lasti >> 1;
            *lasti = depth_and_lasti & 1;
            return 1;
        }
        scan = skip_to_next_entry(scan, end);
    }
    return 0;
}

static int
initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
    PyObject **localsplus, PyObject *const *args,
    Py_ssize_t argcount, PyObject *kwnames)
{
    PyCodeObject *co = (PyCodeObject*)func->func_code;
    const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;

    /* Create a dictionary for keyword parameters (**kwags) */
    PyObject *kwdict;
    Py_ssize_t i;
    if (co->co_flags & CO_VARKEYWORDS) {
        kwdict = PyDict_New();
        if (kwdict == NULL) {
            goto fail_pre_positional;
        }
        i = total_args;
        if (co->co_flags & CO_VARARGS) {
            i++;
        }
        assert(localsplus[i] == NULL);
        localsplus[i] = kwdict;
    }
    else {
        kwdict = NULL;
    }

    /* Copy all positional arguments into local variables */
    Py_ssize_t j, n;
    if (argcount > co->co_argcount) {
        n = co->co_argcount;
    }
    else {
        n = argcount;
    }
    for (j = 0; j < n; j++) {
        PyObject *x = args[j];
        assert(localsplus[j] == NULL);
        localsplus[j] = x;
    }

    /* Pack other positional arguments into the *args argument */
    if (co->co_flags & CO_VARARGS) {
        PyObject *u = NULL;
        u = _PyTuple_FromArraySteal(args + n, argcount - n);
        if (u == NULL) {
            goto fail_post_positional;
        }
        assert(localsplus[total_args] == NULL);
        localsplus[total_args] = u;
    }
    else if (argcount > n) {
        /* Too many postional args. Error is reported later */
        for (j = n; j < argcount; j++) {
            Py_DECREF(args[j]);
        }
    }

    /* Handle keyword arguments */
    if (kwnames != NULL) {
        Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
        for (i = 0; i < kwcount; i++) {
            PyObject **co_varnames;
            PyObject *keyword = PyTuple_GET_ITEM(kwnames, i);
            PyObject *value = args[i+argcount];
            Py_ssize_t j;

            if (keyword == NULL || !PyUnicode_Check(keyword)) {
                _PyErr_Format(tstate, PyExc_TypeError,
                            "%U() keywords must be strings",
                          func->func_qualname);
                goto kw_fail;
            }

            /* Speed hack: do raw pointer compares. As names are
            normally interned this should almost always hit. */
            co_varnames = ((PyTupleObject *)(co->co_localsplusnames))->ob_item;
            for (j = co->co_posonlyargcount; j < total_args; j++) {
                PyObject *varname = co_varnames[j];
                if (varname == keyword) {
                    goto kw_found;
                }
            }

            /* Slow fallback, just in case */
            for (j = co->co_posonlyargcount; j < total_args; j++) {
                PyObject *varname = co_varnames[j];
                int cmp = PyObject_RichCompareBool( keyword, varname, Py_EQ);
                if (cmp > 0) {
                    goto kw_found;
                }
                else if (cmp < 0) {
                    goto kw_fail;
                }
            }

            assert(j >= total_args);
            if (kwdict == NULL) {

                if (co->co_posonlyargcount
                    && positional_only_passed_as_keyword(tstate, co,
                                                        kwcount, kwnames,
                                                        func->func_qualname))
                {
                    goto kw_fail;
                }

                _PyErr_Format(tstate, PyExc_TypeError,
                            "%U() got an unexpected keyword argument '%S'",
                          func->func_qualname, keyword);
                goto kw_fail;
            }

            if (PyDict_SetItem(kwdict, keyword, value) == -1) {
                goto kw_fail;
            }
            Py_DECREF(value);
            continue;

        kw_fail:
            for (;i < kwcount; i++) {
                PyObject *value = args[i+argcount];
                Py_DECREF(value);
            }
            goto fail_post_args;

        kw_found:
            if (localsplus[j] != NULL) {
                _PyErr_Format(tstate, PyExc_TypeError,
                            "%U() got multiple values for argument '%S'",
                          func->func_qualname, keyword);
                goto kw_fail;
            }
            localsplus[j] = value;
        }
    }

    /* Check the number of positional arguments */
    if ((argcount > co->co_argcount) && !(co->co_flags & CO_VARARGS)) {
        too_many_positional(tstate, co, argcount, func->func_defaults, localsplus,
                            func->func_qualname);
        goto fail_post_args;
    }

    /* Add missing positional arguments (copy default values from defs) */
    if (argcount < co->co_argcount) {
        Py_ssize_t defcount = func->func_defaults == NULL ? 0 : PyTuple_GET_SIZE(func->func_defaults);
        Py_ssize_t m = co->co_argcount - defcount;
        Py_ssize_t missing = 0;
        for (i = argcount; i < m; i++) {
            if (localsplus[i] == NULL) {
                missing++;
            }
        }
        if (missing) {
            missing_arguments(tstate, co, missing, defcount, localsplus,
                              func->func_qualname);
            goto fail_post_args;
        }
        if (n > m)
            i = n - m;
        else
            i = 0;
        if (defcount) {
            PyObject **defs = &PyTuple_GET_ITEM(func->func_defaults, 0);
            for (; i < defcount; i++) {
                if (localsplus[m+i] == NULL) {
                    PyObject *def = defs[i];
                    Py_INCREF(def);
                    localsplus[m+i] = def;
                }
            }
        }
    }

    /* Add missing keyword arguments (copy default values from kwdefs) */
    if (co->co_kwonlyargcount > 0) {
        Py_ssize_t missing = 0;
        for (i = co->co_argcount; i < total_args; i++) {
            if (localsplus[i] != NULL)
                continue;
            PyObject *varname = PyTuple_GET_ITEM(co->co_localsplusnames, i);
            if (func->func_kwdefaults != NULL) {
                PyObject *def = PyDict_GetItemWithError(func->func_kwdefaults, varname);
                if (def) {
                    Py_INCREF(def);
                    localsplus[i] = def;
                    continue;
                }
                else if (_PyErr_Occurred(tstate)) {
                    goto fail_post_args;
                }
            }
            missing++;
        }
        if (missing) {
            missing_arguments(tstate, co, missing, -1, localsplus,
                              func->func_qualname);
            goto fail_post_args;
        }
    }
    return 0;

fail_pre_positional:
    for (j = 0; j < argcount; j++) {
        Py_DECREF(args[j]);
    }
    /* fall through */
fail_post_positional:
    if (kwnames) {
        Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
        for (j = argcount; j < argcount+kwcount; j++) {
            Py_DECREF(args[j]);
        }
    }
    /* fall through */
fail_post_args:
    return -1;
}

/* Consumes all the references to the args */
static PyObject *
make_coro(PyThreadState *tstate, PyFunctionObject *func,
          PyObject *locals,
          PyObject* const* args, size_t argcount,
          PyObject *kwnames)
{
    assert (((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR));
    PyObject *gen = _Py_MakeCoro(func);
    if (gen == NULL) {
        return NULL;
    }
    InterpreterFrame *frame = (InterpreterFrame *)((PyGenObject *)gen)->gi_iframe;
    PyCodeObject *code = (PyCodeObject *)func->func_code;
    _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus);
    for (int i = 0; i < code->co_nlocalsplus; i++) {
        frame->localsplus[i] = NULL;
    }
    ((PyGenObject *)gen)->gi_frame_valid = 1;
    if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) {
        Py_DECREF(gen);
        return NULL;
    }
    frame->generator = gen;
    return gen;
}

/* Consumes all the references to the args */
static InterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
                        PyObject *locals, PyObject* const* args,
                        size_t argcount, PyObject *kwnames)
{
    PyCodeObject * code = (PyCodeObject *)func->func_code;
    size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
    InterpreterFrame *frame = _PyThreadState_BumpFramePointer(tstate, size);
    if (frame == NULL) {
        goto fail;
    }
    _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus);
    PyObject **localsarray = &frame->localsplus[0];
    for (int i = 0; i < code->co_nlocalsplus; i++) {
        localsarray[i] = NULL;
    }
    if (initialize_locals(tstate, func, localsarray, args, argcount, kwnames)) {
        _PyFrame_Clear(frame);
        return NULL;
    }
    return frame;
fail:
    /* Consume the references */
    for (size_t i = 0; i < argcount; i++) {
        Py_DECREF(args[i]);
    }
    if (kwnames) {
        Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
        for (Py_ssize_t i = 0; i < kwcount; i++) {
            Py_DECREF(args[i+argcount]);
        }
    }
    PyErr_NoMemory();
    return NULL;
}

static void
_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame)
{
    tstate->recursion_remaining--;
    assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0);
    _PyFrame_Clear(frame);
    tstate->recursion_remaining++;
    _PyThreadState_PopFrame(tstate, frame);
}

PyObject *
_PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
               PyObject *locals,
               PyObject* const* args, size_t argcount,
               PyObject *kwnames)
{
    PyCodeObject *code = (PyCodeObject *)func->func_code;
    /* _PyEvalFramePushAndInit and make_coro consume
     * all the references to their arguments
     */
    for (size_t i = 0; i < argcount; i++) {
        Py_INCREF(args[i]);
    }
    if (kwnames) {
        Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
        for (Py_ssize_t i = 0; i < kwcount; i++) {
            Py_INCREF(args[i+argcount]);
        }
    }
    int is_coro = code->co_flags &
        (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR);
    if (is_coro) {
        return make_coro(tstate, func, locals, args, argcount, kwnames);
    }
    InterpreterFrame *frame = _PyEvalFramePushAndInit(
        tstate, func, locals, args, argcount, kwnames);
    if (frame == NULL) {
        return NULL;
    }
    PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
    assert(_PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame));
    _PyEvalFrameClearAndPop(tstate, frame);
    return retval;
}

/* Legacy API */
PyObject *
PyEval_EvalCodeEx(PyObject *_co, PyObject *globals, PyObject *locals,
                  PyObject *const *args, int argcount,
                  PyObject *const *kws, int kwcount,
                  PyObject *const *defs, int defcount,
                  PyObject *kwdefs, PyObject *closure)
{
    PyThreadState *tstate = _PyThreadState_GET();
    PyObject *res;
    PyObject *defaults = _PyTuple_FromArray(defs, defcount);
    if (defaults == NULL) {
        return NULL;
    }
    PyObject *builtins = _PyEval_BuiltinsFromGlobals(tstate, globals); // borrowed ref
    if (builtins == NULL) {
        Py_DECREF(defaults);
        return NULL;
    }
    if (locals == NULL) {
        locals = globals;
    }
    PyObject *kwnames;
    PyObject *const *allargs;
    PyObject **newargs;
    if (kwcount == 0) {
        allargs = args;
        kwnames = NULL;
    }
    else {
        kwnames = PyTuple_New(kwcount);
        if (kwnames == NULL) {
            res = NULL;
            goto fail;
        }
        newargs = PyMem_Malloc(sizeof(PyObject *)*(kwcount+argcount));
        if (newargs == NULL) {
            res = NULL;
            Py_DECREF(kwnames);
            goto fail;
        }
        for (int i = 0; i < argcount; i++) {
            newargs[i] = args[i];
        }
        for (int i = 0; i < kwcount; i++) {
            Py_INCREF(kws[2*i]);
            PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
            newargs[argcount+i] = kws[2*i+1];
        }
        allargs = newargs;
    }
    PyObject **kwargs = PyMem_Malloc(sizeof(PyObject *)*kwcount);
    if (kwargs == NULL) {
        res = NULL;
        Py_DECREF(kwnames);
        goto fail;
    }
    for (int i = 0; i < kwcount; i++) {
        Py_INCREF(kws[2*i]);
        PyTuple_SET_ITEM(kwnames, i, kws[2*i]);
        kwargs[i] = kws[2*i+1];
    }
    PyFrameConstructor constr = {
        .fc_globals = globals,
        .fc_builtins = builtins,
        .fc_name = ((PyCodeObject *)_co)->co_name,
        .fc_qualname = ((PyCodeObject *)_co)->co_name,
        .fc_code = _co,
        .fc_defaults = defaults,
        .fc_kwdefaults = kwdefs,
        .fc_closure = closure
    };
    PyFunctionObject *func = _PyFunction_FromConstructor(&constr);
    if (func == NULL) {
        return NULL;
    }
    res = _PyEval_Vector(tstate, func, locals,
                         allargs, argcount,
                         kwnames);
    Py_DECREF(func);
    if (kwcount) {
        Py_DECREF(kwnames);
        PyMem_Free(newargs);
    }
fail:
    Py_DECREF(defaults);
    return res;
}


/* Logic for the raise statement (too complicated for inlining).
   This *consumes* a reference count to each of its arguments. */
static int
do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
{
    PyObject *type = NULL, *value = NULL;

    if (exc == NULL) {
        /* Reraise */
        _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
        value = exc_info->exc_value;
        if (Py_IsNone(value) || value == NULL) {
            _PyErr_SetString(tstate, PyExc_RuntimeError,
                             "No active exception to reraise");
            return 0;
        }
        assert(PyExceptionInstance_Check(value));
        type = PyExceptionInstance_Class(value);
        Py_XINCREF(type);
        Py_XINCREF(value);
        PyObject *tb = PyException_GetTraceback(value); /* new ref */
        _PyErr_Restore(tstate, type, value, tb);
        return 1;
    }

    /* We support the following forms of raise:
       raise
       raise <instance>
       raise <type> */

    if (PyExceptionClass_Check(exc)) {
        type = exc;
        value = _PyObject_CallNoArgs(exc);
        if (value == NULL)
            goto raise_error;
        if (!PyExceptionInstance_Check(value)) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "calling %R should have returned an instance of "
                          "BaseException, not %R",
                          type, Py_TYPE(value));
             goto raise_error;
        }
    }
    else if (PyExceptionInstance_Check(exc)) {
        value = exc;
        type = PyExceptionInstance_Class(exc);
        Py_INCREF(type);
    }
    else {
        /* Not something you can raise.  You get an exception
           anyway, just not what you specified :-) */
        Py_DECREF(exc);
        _PyErr_SetString(tstate, PyExc_TypeError,
                         "exceptions must derive from BaseException");
        goto raise_error;
    }

    assert(type != NULL);
    assert(value != NULL);

    if (cause) {
        PyObject *fixed_cause;
        if (PyExceptionClass_Check(cause)) {
            fixed_cause = _PyObject_CallNoArgs(cause);
            if (fixed_cause == NULL)
                goto raise_error;
            Py_DECREF(cause);
        }
        else if (PyExceptionInstance_Check(cause)) {
            fixed_cause = cause;
        }
        else if (Py_IsNone(cause)) {
            Py_DECREF(cause);
            fixed_cause = NULL;
        }
        else {
            _PyErr_SetString(tstate, PyExc_TypeError,
                             "exception causes must derive from "
                             "BaseException");
            goto raise_error;
        }
        PyException_SetCause(value, fixed_cause);
    }

    _PyErr_SetObject(tstate, type, value);
    /* _PyErr_SetObject incref's its arguments */
    Py_DECREF(value);
    Py_DECREF(type);
    return 0;

raise_error:
    Py_XDECREF(value);
    Py_XDECREF(type);
    Py_XDECREF(cause);
    return 0;
}

/* Logic for matching an exception in an except* clause (too
   complicated for inlining).
*/

static int
exception_group_match(PyObject* exc_value, PyObject *match_type,
                      PyObject **match, PyObject **rest)
{
    if (Py_IsNone(exc_value)) {
        *match = Py_NewRef(Py_None);
        *rest = Py_NewRef(Py_None);
        return 0;
    }
    assert(PyExceptionInstance_Check(exc_value));

    if (PyErr_GivenExceptionMatches(exc_value, match_type)) {
        /* Full match of exc itself */
        bool is_eg = _PyBaseExceptionGroup_Check(exc_value);
        if (is_eg) {
            *match = Py_NewRef(exc_value);
        }
        else {
            /* naked exception - wrap it */
            PyObject *excs = PyTuple_Pack(1, exc_value);
            if (excs == NULL) {
                return -1;
            }
            PyObject *wrapped = _PyExc_CreateExceptionGroup("", excs);
            Py_DECREF(excs);
            if (wrapped == NULL) {
                return -1;
            }
            *match = wrapped;
        }
        *rest = Py_NewRef(Py_None);
        return 0;
    }

    /* exc_value does not match match_type.
     * Check for partial match if it's an exception group.
     */
    if (_PyBaseExceptionGroup_Check(exc_value)) {
        PyObject *pair = PyObject_CallMethod(exc_value, "split", "(O)",
                                             match_type);
        if (pair == NULL) {
            return -1;
        }
        assert(PyTuple_CheckExact(pair));
        assert(PyTuple_GET_SIZE(pair) == 2);
        *match = Py_NewRef(PyTuple_GET_ITEM(pair, 0));
        *rest = Py_NewRef(PyTuple_GET_ITEM(pair, 1));
        Py_DECREF(pair);
        return 0;
    }
    /* no match */
    *match = Py_NewRef(Py_None);
    *rest = Py_NewRef(Py_None);
    return 0;
}

/* Logic for the final raise/reraise of a try-except* contruct
   (too complicated for inlining).
*/

static bool
is_same_exception_metadata(PyObject *exc1, PyObject *exc2)
{
    assert(PyExceptionInstance_Check(exc1));
    assert(PyExceptionInstance_Check(exc2));

    PyObject *tb1 = PyException_GetTraceback(exc1);
    PyObject *ctx1 = PyException_GetContext(exc1);
    PyObject *cause1 = PyException_GetCause(exc1);
    PyObject *tb2 = PyException_GetTraceback(exc2);
    PyObject *ctx2 = PyException_GetContext(exc2);
    PyObject *cause2 = PyException_GetCause(exc2);

    bool result = (Py_Is(tb1, tb2) &&
                   Py_Is(ctx1, ctx2) &&
                   Py_Is(cause1, cause2));

    Py_XDECREF(tb1);
    Py_XDECREF(ctx1);
    Py_XDECREF(cause1);
    Py_XDECREF(tb2);
    Py_XDECREF(ctx2);
    Py_XDECREF(cause2);
    return result;
}

/*
   excs: a list of exceptions to raise/reraise
   orig: the original except that was caught

   Calculates an exception group to raise. It contains
   all exceptions in excs, where those that were reraised
   have same nesting structure as in orig, and those that
   were raised (if any) are added as siblings in a new EG.

   Returns NULL and sets an exception on failure.
*/
static PyObject *
do_reraise_star(PyObject *excs, PyObject *orig)
{
    assert(PyList_Check(excs));
    assert(PyExceptionInstance_Check(orig));

    Py_ssize_t numexcs = PyList_GET_SIZE(excs);

    if (numexcs == 0) {
        return Py_NewRef(Py_None);
    }

    if (!_PyBaseExceptionGroup_Check(orig)) {
        /* a naked exception was caught and wrapped. Only one except* clause
         * could have executed,so there is at most one exception to raise.
         */

        assert(numexcs == 1 || (numexcs == 2 && PyList_GET_ITEM(excs, 1) == Py_None));

        PyObject *e = PyList_GET_ITEM(excs, 0);
        assert(e != NULL);
        return Py_NewRef(e);
    }


    PyObject *raised_list = PyList_New(0);
    if (raised_list == NULL) {
        return NULL;
    }
    PyObject* reraised_list = PyList_New(0);
    if (reraised_list == NULL) {
        Py_DECREF(raised_list);
        return NULL;
    }

    /* Now we are holding refs to raised_list and reraised_list */

    PyObject *result = NULL;

    /* Split excs into raised and reraised by comparing metadata with orig */
    for (Py_ssize_t i = 0; i < numexcs; i++) {
        PyObject *e = PyList_GET_ITEM(excs, i);
        assert(e != NULL);
        if (Py_IsNone(e)) {
            continue;
        }
        bool is_reraise = is_same_exception_metadata(e, orig);
        PyObject *append_list = is_reraise ? reraised_list : raised_list;
        if (PyList_Append(append_list, e) < 0) {
            goto done;
        }
    }

    PyObject *reraised_eg = _PyExc_ExceptionGroupProjection(orig, reraised_list);
    if (reraised_eg == NULL) {
        goto done;
    }

    if (!Py_IsNone(reraised_eg)) {
        assert(is_same_exception_metadata(reraised_eg, orig));
    }

    Py_ssize_t num_raised = PyList_GET_SIZE(raised_list);
    if (num_raised == 0) {
        result = reraised_eg;
    }
    else if (num_raised > 0) {
        int res = 0;
        if (!Py_IsNone(reraised_eg)) {
            res = PyList_Append(raised_list, reraised_eg);
        }
        Py_DECREF(reraised_eg);
        if (res < 0) {
            goto done;
        }
        result = _PyExc_CreateExceptionGroup("", raised_list);
        if (result == NULL) {
            goto done;
        }
    }

done:
    Py_XDECREF(raised_list);
    Py_XDECREF(reraised_list);
    return result;
}

/* Iterate v argcnt times and store the results on the stack (via decreasing
   sp).  Return 1 for success, 0 if error.

   If argcntafter == -1, do a simple unpack. If it is >= 0, do an unpack
   with a variable target.
*/

static int
unpack_iterable(PyThreadState *tstate, PyObject *v,
                int argcnt, int argcntafter, PyObject **sp)
{
    int i = 0, j = 0;
    Py_ssize_t ll = 0;
    PyObject *it;  /* iter(v) */
    PyObject *w;
    PyObject *l = NULL; /* variable list */

    assert(v != NULL);

    it = PyObject_GetIter(v);
    if (it == NULL) {
        if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) &&
            Py_TYPE(v)->tp_iter == NULL && !PySequence_Check(v))
        {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "cannot unpack non-iterable %.200s object",
                          Py_TYPE(v)->tp_name);
        }
        return 0;
    }

    for (; i < argcnt; i++) {
        w = PyIter_Next(it);
        if (w == NULL) {
            /* Iterator done, via error or exhaustion. */
            if (!_PyErr_Occurred(tstate)) {
                if (argcntafter == -1) {
                    _PyErr_Format(tstate, PyExc_ValueError,
                                  "not enough values to unpack "
                                  "(expected %d, got %d)",
                                  argcnt, i);
                }
                else {
                    _PyErr_Format(tstate, PyExc_ValueError,
                                  "not enough values to unpack "
                                  "(expected at least %d, got %d)",
                                  argcnt + argcntafter, i);
                }
            }
            goto Error;
        }
        *--sp = w;
    }

    if (argcntafter == -1) {
        /* We better have exhausted the iterator now. */
        w = PyIter_Next(it);
        if (w == NULL) {
            if (_PyErr_Occurred(tstate))
                goto Error;
            Py_DECREF(it);
            return 1;
        }
        Py_DECREF(w);
        _PyErr_Format(tstate, PyExc_ValueError,
                      "too many values to unpack (expected %d)",
                      argcnt);
        goto Error;
    }

    l = PySequence_List(it);
    if (l == NULL)
        goto Error;
    *--sp = l;
    i++;

    ll = PyList_GET_SIZE(l);
    if (ll < argcntafter) {
        _PyErr_Format(tstate, PyExc_ValueError,
            "not enough values to unpack (expected at least %d, got %zd)",
            argcnt + argcntafter, argcnt + ll);
        goto Error;
    }

    /* Pop the "after-variable" args off the list. */
    for (j = argcntafter; j > 0; j--, i++) {
        *--sp = PyList_GET_ITEM(l, ll - j);
    }
    /* Resize the list. */
    Py_SET_SIZE(l, ll - argcntafter);
    Py_DECREF(it);
    return 1;

Error:
    for (; i > 0; i--, sp++)
        Py_DECREF(*sp);
    Py_XDECREF(it);
    return 0;
}

#ifdef LLTRACE
static int
prtrace(PyThreadState *tstate, PyObject *v, const char *str)
{
    printf("%s ", str);
    PyObject *type, *value, *traceback;
    PyErr_Fetch(&type, &value, &traceback);
    if (PyObject_Print(v, stdout, 0) != 0) {
        /* Don't know what else to do */
        _PyErr_Clear(tstate);
    }
    printf("\n");
    PyErr_Restore(type, value, traceback);
    return 1;
}
#endif

static void
call_exc_trace(Py_tracefunc func, PyObject *self,
               PyThreadState *tstate,
               InterpreterFrame *f)
{
    PyObject *type, *value, *traceback, *orig_traceback, *arg;
    int err;
    _PyErr_Fetch(tstate, &type, &value, &orig_traceback);
    if (value == NULL) {
        value = Py_None;
        Py_INCREF(value);
    }
    _PyErr_NormalizeException(tstate, &type, &value, &orig_traceback);
    traceback = (orig_traceback != NULL) ? orig_traceback : Py_None;
    arg = PyTuple_Pack(3, type, value, traceback);
    if (arg == NULL) {
        _PyErr_Restore(tstate, type, value, orig_traceback);
        return;
    }
    err = call_trace(func, self, tstate, f, PyTrace_EXCEPTION, arg);
    Py_DECREF(arg);
    if (err == 0) {
        _PyErr_Restore(tstate, type, value, orig_traceback);
    }
    else {
        Py_XDECREF(type);
        Py_XDECREF(value);
        Py_XDECREF(orig_traceback);
    }
}

static int
call_trace_protected(Py_tracefunc func, PyObject *obj,
                     PyThreadState *tstate, InterpreterFrame *frame,
                     int what, PyObject *arg)
{
    PyObject *type, *value, *traceback;
    int err;
    _PyErr_Fetch(tstate, &type, &value, &traceback);
    err = call_trace(func, obj, tstate, frame, what, arg);
    if (err == 0)
    {
        _PyErr_Restore(tstate, type, value, traceback);
        return 0;
    }
    else {
        Py_XDECREF(type);
        Py_XDECREF(value);
        Py_XDECREF(traceback);
        return -1;
    }
}

static void
initialize_trace_info(PyTraceInfo *trace_info, InterpreterFrame *frame)
{
    PyCodeObject *code = frame->f_code;
    if (trace_info->code != code) {
        trace_info->code = code;
        _PyCode_InitAddressRange(code, &trace_info->bounds);
    }
}

static int
call_trace(Py_tracefunc func, PyObject *obj,
           PyThreadState *tstate, InterpreterFrame *frame,
           int what, PyObject *arg)
{
    int result;
    if (tstate->tracing)
        return 0;
    tstate->tracing++;
    _PyThreadState_PauseTracing(tstate);
    PyFrameObject *f = _PyFrame_GetFrameObject(frame);
    if (f == NULL) {
        return -1;
    }
    if (frame->f_lasti < 0) {
        f->f_lineno = frame->f_code->co_firstlineno;
    }
    else {
        initialize_trace_info(&tstate->trace_info, frame);
        f->f_lineno = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &tstate->trace_info.bounds);
    }
    result = func(obj, f, what, arg);
    f->f_lineno = 0;
    _PyThreadState_ResumeTracing(tstate);
    tstate->tracing--;
    return result;
}

PyObject *
_PyEval_CallTracing(PyObject *func, PyObject *args)
{
    PyThreadState *tstate = _PyThreadState_GET();
    int save_tracing = tstate->tracing;
    int save_use_tracing = tstate->cframe->use_tracing;
    PyObject *result;

    tstate->tracing = 0;
    _PyThreadState_ResumeTracing(tstate);
    result = PyObject_Call(func, args, NULL);
    tstate->tracing = save_tracing;
    tstate->cframe->use_tracing = save_use_tracing;
    return result;
}

/* See Objects/lnotab_notes.txt for a description of how tracing works. */
static int
maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
                      PyThreadState *tstate, InterpreterFrame *frame, int instr_prev)
{
    int result = 0;

    /* If the last instruction falls at the start of a line or if it
       represents a jump backwards, update the frame's line number and
       then call the trace function if we're tracing source lines.
    */
    initialize_trace_info(&tstate->trace_info, frame);
    int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &tstate->trace_info.bounds);
    int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &tstate->trace_info.bounds);
    PyFrameObject *f = _PyFrame_GetFrameObject(frame);
    if (f == NULL) {
        return -1;
    }
    if (line != -1 && f->f_trace_lines) {
        /* Trace backward edges (except in 'yield from') or if line number has changed */
        int trace = line != lastline ||
            (frame->f_lasti < instr_prev &&
            _Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND);
        if (trace) {
            result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
        }
    }
    /* Always emit an opcode event if we're tracing all opcodes. */
    if (f->f_trace_opcodes) {
        result = call_trace(func, obj, tstate, frame, PyTrace_OPCODE, Py_None);
    }
    return result;
}

int
_PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
{
    assert(is_tstate_valid(tstate));
    /* The caller must hold the GIL */
    assert(PyGILState_Check());

    /* Call _PySys_Audit() in the context of the current thread state,
       even if tstate is not the current thread state. */
    PyThreadState *current_tstate = _PyThreadState_GET();
    if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) {
        return -1;
    }

    PyObject *profileobj = tstate->c_profileobj;

    tstate->c_profilefunc = NULL;
    tstate->c_profileobj = NULL;
    /* Must make sure that tracing is not ignored if 'profileobj' is freed */
    _PyThreadState_ResumeTracing(tstate);
    Py_XDECREF(profileobj);

    Py_XINCREF(arg);
    tstate->c_profileobj = arg;
    tstate->c_profilefunc = func;

    /* Flag that tracing or profiling is turned on */
    _PyThreadState_ResumeTracing(tstate);
    return 0;
}

void
PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
{
    PyThreadState *tstate = _PyThreadState_GET();
    if (_PyEval_SetProfile(tstate, func, arg) < 0) {
        /* Log _PySys_Audit() error */
        _PyErr_WriteUnraisableMsg("in PyEval_SetProfile", NULL);
    }
}

int
_PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
{
    assert(is_tstate_valid(tstate));
    /* The caller must hold the GIL */
    assert(PyGILState_Check());

    /* Call _PySys_Audit() in the context of the current thread state,
       even if tstate is not the current thread state. */
    PyThreadState *current_tstate = _PyThreadState_GET();
    if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) {
        return -1;
    }

    PyObject *traceobj = tstate->c_traceobj;

    tstate->c_tracefunc = NULL;
    tstate->c_traceobj = NULL;
    /* Must make sure that profiling is not ignored if 'traceobj' is freed */
    _PyThreadState_ResumeTracing(tstate);
    Py_XDECREF(traceobj);

    Py_XINCREF(arg);
    tstate->c_traceobj = arg;
    tstate->c_tracefunc = func;

    /* Flag that tracing or profiling is turned on */
    _PyThreadState_ResumeTracing(tstate);

    return 0;
}

void
PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
{
    PyThreadState *tstate = _PyThreadState_GET();
    if (_PyEval_SetTrace(tstate, func, arg) < 0) {
        /* Log _PySys_Audit() error */
        _PyErr_WriteUnraisableMsg("in PyEval_SetTrace", NULL);
    }
}


void
_PyEval_SetCoroutineOriginTrackingDepth(PyThreadState *tstate, int new_depth)
{
    assert(new_depth >= 0);
    tstate->coroutine_origin_tracking_depth = new_depth;
}

int
_PyEval_GetCoroutineOriginTrackingDepth(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    return tstate->coroutine_origin_tracking_depth;
}

int
_PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
{
    PyThreadState *tstate = _PyThreadState_GET();

    if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_firstiter", NULL) < 0) {
        return -1;
    }

    Py_XINCREF(firstiter);
    Py_XSETREF(tstate->async_gen_firstiter, firstiter);
    return 0;
}

PyObject *
_PyEval_GetAsyncGenFirstiter(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    return tstate->async_gen_firstiter;
}

int
_PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
{
    PyThreadState *tstate = _PyThreadState_GET();

    if (_PySys_Audit(tstate, "sys.set_asyncgen_hook_finalizer", NULL) < 0) {
        return -1;
    }

    Py_XINCREF(finalizer);
    Py_XSETREF(tstate->async_gen_finalizer, finalizer);
    return 0;
}

PyObject *
_PyEval_GetAsyncGenFinalizer(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    return tstate->async_gen_finalizer;
}

InterpreterFrame *
_PyEval_GetFrame(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    return tstate->cframe->current_frame;
}

PyFrameObject *
PyEval_GetFrame(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    if (tstate->cframe->current_frame == NULL) {
        return NULL;
    }
    PyFrameObject *f = _PyFrame_GetFrameObject(tstate->cframe->current_frame);
    if (f == NULL) {
        PyErr_Clear();
    }
    return f;
}

PyObject *
_PyEval_GetBuiltins(PyThreadState *tstate)
{
    InterpreterFrame *frame = tstate->cframe->current_frame;
    if (frame != NULL) {
        return frame->f_builtins;
    }
    return tstate->interp->builtins;
}

PyObject *
PyEval_GetBuiltins(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    return _PyEval_GetBuiltins(tstate);
}

/* Convenience function to get a builtin from its name */
PyObject *
_PyEval_GetBuiltinId(_Py_Identifier *name)
{
    PyThreadState *tstate = _PyThreadState_GET();
    PyObject *attr = _PyDict_GetItemIdWithError(PyEval_GetBuiltins(), name);
    if (attr) {
        Py_INCREF(attr);
    }
    else if (!_PyErr_Occurred(tstate)) {
        _PyErr_SetObject(tstate, PyExc_AttributeError, _PyUnicode_FromId(name));
    }
    return attr;
}

PyObject *
PyEval_GetLocals(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
     InterpreterFrame *current_frame = tstate->cframe->current_frame;
    if (current_frame == NULL) {
        _PyErr_SetString(tstate, PyExc_SystemError, "frame does not exist");
        return NULL;
    }

    if (_PyFrame_FastToLocalsWithError(current_frame) < 0) {
        return NULL;
    }

    PyObject *locals = current_frame->f_locals;
    assert(locals != NULL);
    return locals;
}

PyObject *
PyEval_GetGlobals(void)
{
    PyThreadState *tstate = _PyThreadState_GET();
    InterpreterFrame *current_frame = tstate->cframe->current_frame;
    if (current_frame == NULL) {
        return NULL;
    }
    return current_frame->f_globals;
}

int
PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
{
    PyThreadState *tstate = _PyThreadState_GET();
    InterpreterFrame *current_frame = tstate->cframe->current_frame;
    int result = cf->cf_flags != 0;

    if (current_frame != NULL) {
        const int codeflags = current_frame->f_code->co_flags;
        const int compilerflags = codeflags & PyCF_MASK;
        if (compilerflags) {
            result = 1;
            cf->cf_flags |= compilerflags;
        }
#if 0 /* future keyword */
        if (codeflags & CO_GENERATOR_ALLOWED) {
            result = 1;
            cf->cf_flags |= CO_GENERATOR_ALLOWED;
        }
#endif
    }
    return result;
}


const char *
PyEval_GetFuncName(PyObject *func)
{
    if (PyMethod_Check(func))
        return PyEval_GetFuncName(PyMethod_GET_FUNCTION(func));
    else if (PyFunction_Check(func))
        return PyUnicode_AsUTF8(((PyFunctionObject*)func)->func_name);
    else if (PyCFunction_Check(func))
        return ((PyCFunctionObject*)func)->m_ml->ml_name;
    else
        return Py_TYPE(func)->tp_name;
}

const char *
PyEval_GetFuncDesc(PyObject *func)
{
    if (PyMethod_Check(func))
        return "()";
    else if (PyFunction_Check(func))
        return "()";
    else if (PyCFunction_Check(func))
        return "()";
    else
        return " object";
}

#define C_TRACE(x, call) \
if (use_tracing && tstate->c_profilefunc) { \
    if (call_trace(tstate->c_profilefunc, tstate->c_profileobj, \
        tstate, tstate->cframe->current_frame, \
        PyTrace_C_CALL, func)) { \
        x = NULL; \
    } \
    else { \
        x = call; \
        if (tstate->c_profilefunc != NULL) { \
            if (x == NULL) { \
                call_trace_protected(tstate->c_profilefunc, \
                    tstate->c_profileobj, \
                    tstate, tstate->cframe->current_frame, \
                    PyTrace_C_EXCEPTION, func); \
                /* XXX should pass (type, value, tb) */ \
            } else { \
                if (call_trace(tstate->c_profilefunc, \
                    tstate->c_profileobj, \
                    tstate, tstate->cframe->current_frame, \
                    PyTrace_C_RETURN, func)) { \
                    Py_DECREF(x); \
                    x = NULL; \
                } \
            } \
        } \
    } \
} else { \
    x = call; \
    }


static PyObject *
trace_call_function(PyThreadState *tstate,
                    PyObject *func,
                    PyObject **args, Py_ssize_t nargs,
                    PyObject *kwnames)
{
    int use_tracing = 1;
    PyObject *x;
    if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) {
        C_TRACE(x, PyObject_Vectorcall(func, args, nargs, kwnames));
        return x;
    }
    else if (Py_IS_TYPE(func, &PyMethodDescr_Type) && nargs > 0) {
        /* We need to create a temporary bound method as argument
           for profiling.

           If nargs == 0, then this cannot work because we have no
           "self". In any case, the call itself would raise
           TypeError (foo needs an argument), so we just skip
           profiling. */
        PyObject *self = args[0];
        func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
        if (func == NULL) {
            return NULL;
        }
        C_TRACE(x, PyObject_Vectorcall(func,
                                        args+1, nargs-1,
                                        kwnames));
        Py_DECREF(func);
        return x;
    }
    return PyObject_Vectorcall(func, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
}

static PyObject *
do_call_core(PyThreadState *tstate,
             PyObject *func,
             PyObject *callargs,
             PyObject *kwdict,
             int use_tracing
            )
{
    PyObject *result;

    if (PyCFunction_CheckExact(func) || PyCMethod_CheckExact(func)) {
        C_TRACE(result, PyObject_Call(func, callargs, kwdict));
        return result;
    }
    else if (Py_IS_TYPE(func, &PyMethodDescr_Type)) {
        Py_ssize_t nargs = PyTuple_GET_SIZE(callargs);
        if (nargs > 0 && use_tracing) {
            /* We need to create a temporary bound method as argument
               for profiling.

               If nargs == 0, then this cannot work because we have no
               "self". In any case, the call itself would raise
               TypeError (foo needs an argument), so we just skip
               profiling. */
            PyObject *self = PyTuple_GET_ITEM(callargs, 0);
            func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
            if (func == NULL) {
                return NULL;
            }

            C_TRACE(result, _PyObject_FastCallDictTstate(
                                    tstate, func,
                                    &_PyTuple_ITEMS(callargs)[1],
                                    nargs - 1,
                                    kwdict));
            Py_DECREF(func);
            return result;
        }
    }
    return PyObject_Call(func, callargs, kwdict);
}

/* Extract a slice index from a PyLong or an object with the
   nb_index slot defined, and store in *pi.
   Silently reduce values larger than PY_SSIZE_T_MAX to PY_SSIZE_T_MAX,
   and silently boost values less than PY_SSIZE_T_MIN to PY_SSIZE_T_MIN.
   Return 0 on error, 1 on success.
*/
int
_PyEval_SliceIndex(PyObject *v, Py_ssize_t *pi)
{
    PyThreadState *tstate = _PyThreadState_GET();
    if (!Py_IsNone(v)) {
        Py_ssize_t x;
        if (_PyIndex_Check(v)) {
            x = PyNumber_AsSsize_t(v, NULL);
            if (x == -1 && _PyErr_Occurred(tstate))
                return 0;
        }
        else {
            _PyErr_SetString(tstate, PyExc_TypeError,
                             "slice indices must be integers or "
                             "None or have an __index__ method");
            return 0;
        }
        *pi = x;
    }
    return 1;
}

int
_PyEval_SliceIndexNotNone(PyObject *v, Py_ssize_t *pi)
{
    PyThreadState *tstate = _PyThreadState_GET();
    Py_ssize_t x;
    if (_PyIndex_Check(v)) {
        x = PyNumber_AsSsize_t(v, NULL);
        if (x == -1 && _PyErr_Occurred(tstate))
            return 0;
    }
    else {
        _PyErr_SetString(tstate, PyExc_TypeError,
                         "slice indices must be integers or "
                         "have an __index__ method");
        return 0;
    }
    *pi = x;
    return 1;
}

static PyObject *
import_name(PyThreadState *tstate, InterpreterFrame *frame,
            PyObject *name, PyObject *fromlist, PyObject *level)
{
    _Py_IDENTIFIER(__import__);
    PyObject *import_func, *res;
    PyObject* stack[5];

    import_func = _PyDict_GetItemIdWithError(frame->f_builtins, &PyId___import__);
    if (import_func == NULL) {
        if (!_PyErr_Occurred(tstate)) {
            _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found");
        }
        return NULL;
    }
    PyObject *locals = frame->f_locals;
    /* Fast path for not overloaded __import__. */
    if (import_func == tstate->interp->import_func) {
        int ilevel = _PyLong_AsInt(level);
        if (ilevel == -1 && _PyErr_Occurred(tstate)) {
            return NULL;
        }
        res = PyImport_ImportModuleLevelObject(
                        name,
                        frame->f_globals,
                        locals == NULL ? Py_None :locals,
                        fromlist,
                        ilevel);
        return res;
    }

    Py_INCREF(import_func);

    stack[0] = name;
    stack[1] = frame->f_globals;
    stack[2] = locals == NULL ? Py_None : locals;
    stack[3] = fromlist;
    stack[4] = level;
    res = _PyObject_FastCall(import_func, stack, 5);
    Py_DECREF(import_func);
    return res;
}

static PyObject *
import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
{
    PyObject *x;
    PyObject *fullmodname, *pkgname, *pkgpath, *pkgname_or_unknown, *errmsg;

    if (_PyObject_LookupAttr(v, name, &x) != 0) {
        return x;
    }
    /* Issue #17636: in case this failed because of a circular relative
       import, try to fallback on reading the module directly from
       sys.modules. */
    pkgname = _PyObject_GetAttrId(v, &PyId___name__);
    if (pkgname == NULL) {
        goto error;
    }
    if (!PyUnicode_Check(pkgname)) {
        Py_CLEAR(pkgname);
        goto error;
    }
    fullmodname = PyUnicode_FromFormat("%U.%U", pkgname, name);
    if (fullmodname == NULL) {
        Py_DECREF(pkgname);
        return NULL;
    }
    x = PyImport_GetModule(fullmodname);
    Py_DECREF(fullmodname);
    if (x == NULL && !_PyErr_Occurred(tstate)) {
        goto error;
    }
    Py_DECREF(pkgname);
    return x;
 error:
    pkgpath = PyModule_GetFilenameObject(v);
    if (pkgname == NULL) {
        pkgname_or_unknown = PyUnicode_FromString("<unknown module name>");
        if (pkgname_or_unknown == NULL) {
            Py_XDECREF(pkgpath);
            return NULL;
        }
    } else {
        pkgname_or_unknown = pkgname;
    }

    if (pkgpath == NULL || !PyUnicode_Check(pkgpath)) {
        _PyErr_Clear(tstate);
        errmsg = PyUnicode_FromFormat(
            "cannot import name %R from %R (unknown location)",
            name, pkgname_or_unknown
        );
        /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
        PyErr_SetImportError(errmsg, pkgname, NULL);
    }
    else {
        _Py_IDENTIFIER(__spec__);
        PyObject *spec = _PyObject_GetAttrId(v, &PyId___spec__);
        const char *fmt =
            _PyModuleSpec_IsInitializing(spec) ?
            "cannot import name %R from partially initialized module %R "
            "(most likely due to a circular import) (%S)" :
            "cannot import name %R from %R (%S)";
        Py_XDECREF(spec);

        errmsg = PyUnicode_FromFormat(fmt, name, pkgname_or_unknown, pkgpath);
        /* NULL checks for errmsg and pkgname done by PyErr_SetImportError. */
        PyErr_SetImportError(errmsg, pkgname, pkgpath);
    }

    Py_XDECREF(errmsg);
    Py_XDECREF(pkgname_or_unknown);
    Py_XDECREF(pkgpath);
    return NULL;
}

static int
import_all_from(PyThreadState *tstate, PyObject *locals, PyObject *v)
{
    _Py_IDENTIFIER(__all__);
    _Py_IDENTIFIER(__dict__);
    PyObject *all, *dict, *name, *value;
    int skip_leading_underscores = 0;
    int pos, err;

    if (_PyObject_LookupAttrId(v, &PyId___all__, &all) < 0) {
        return -1; /* Unexpected error */
    }
    if (all == NULL) {
        if (_PyObject_LookupAttrId(v, &PyId___dict__, &dict) < 0) {
            return -1;
        }
        if (dict == NULL) {
            _PyErr_SetString(tstate, PyExc_ImportError,
                    "from-import-* object has no __dict__ and no __all__");
            return -1;
        }
        all = PyMapping_Keys(dict);
        Py_DECREF(dict);
        if (all == NULL)
            return -1;
        skip_leading_underscores = 1;
    }

    for (pos = 0, err = 0; ; pos++) {
        name = PySequence_GetItem(all, pos);
        if (name == NULL) {
            if (!_PyErr_ExceptionMatches(tstate, PyExc_IndexError)) {
                err = -1;
            }
            else {
                _PyErr_Clear(tstate);
            }
            break;
        }
        if (!PyUnicode_Check(name)) {
            PyObject *modname = _PyObject_GetAttrId(v, &PyId___name__);
            if (modname == NULL) {
                Py_DECREF(name);
                err = -1;
                break;
            }
            if (!PyUnicode_Check(modname)) {
                _PyErr_Format(tstate, PyExc_TypeError,
                              "module __name__ must be a string, not %.100s",
                              Py_TYPE(modname)->tp_name);
            }
            else {
                _PyErr_Format(tstate, PyExc_TypeError,
                              "%s in %U.%s must be str, not %.100s",
                              skip_leading_underscores ? "Key" : "Item",
                              modname,
                              skip_leading_underscores ? "__dict__" : "__all__",
                              Py_TYPE(name)->tp_name);
            }
            Py_DECREF(modname);
            Py_DECREF(name);
            err = -1;
            break;
        }
        if (skip_leading_underscores) {
            if (PyUnicode_READY(name) == -1) {
                Py_DECREF(name);
                err = -1;
                break;
            }
            if (PyUnicode_READ_CHAR(name, 0) == '_') {
                Py_DECREF(name);
                continue;
            }
        }
        value = PyObject_GetAttr(v, name);
        if (value == NULL)
            err = -1;
        else if (PyDict_CheckExact(locals))
            err = PyDict_SetItem(locals, name, value);
        else
            err = PyObject_SetItem(locals, name, value);
        Py_DECREF(name);
        Py_XDECREF(value);
        if (err != 0)
            break;
    }
    Py_DECREF(all);
    return err;
}

#define CANNOT_CATCH_MSG "catching classes that do not inherit from "\
                         "BaseException is not allowed"

#define CANNOT_EXCEPT_STAR_EG "catching ExceptionGroup with except* "\
                              "is not allowed. Use except instead."

static int
check_except_type_valid(PyThreadState *tstate, PyObject* right)
{
    if (PyTuple_Check(right)) {
        Py_ssize_t i, length;
        length = PyTuple_GET_SIZE(right);
        for (i = 0; i < length; i++) {
            PyObject *exc = PyTuple_GET_ITEM(right, i);
            if (!PyExceptionClass_Check(exc)) {
                _PyErr_SetString(tstate, PyExc_TypeError,
                    CANNOT_CATCH_MSG);
                return -1;
            }
        }
    }
    else {
        if (!PyExceptionClass_Check(right)) {
            _PyErr_SetString(tstate, PyExc_TypeError,
                CANNOT_CATCH_MSG);
            return -1;
        }
    }
    return 0;
}

static int
check_except_star_type_valid(PyThreadState *tstate, PyObject* right)
{
    if (check_except_type_valid(tstate, right) < 0) {
        return -1;
    }

    /* reject except *ExceptionGroup */

    int is_subclass = 0;
    if (PyTuple_Check(right)) {
        Py_ssize_t length = PyTuple_GET_SIZE(right);
        for (Py_ssize_t i = 0; i < length; i++) {
            PyObject *exc = PyTuple_GET_ITEM(right, i);
            is_subclass = PyObject_IsSubclass(exc, PyExc_BaseExceptionGroup);
            if (is_subclass < 0) {
                return -1;
            }
            if (is_subclass) {
                break;
            }
        }
    }
    else {
        is_subclass = PyObject_IsSubclass(right, PyExc_BaseExceptionGroup);
        if (is_subclass < 0) {
            return -1;
        }
    }
    if (is_subclass) {
        _PyErr_SetString(tstate, PyExc_TypeError,
            CANNOT_EXCEPT_STAR_EG);
            return -1;
    }
    return 0;
}

static int
check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args)
{
    if (Py_TYPE(args)->tp_iter == NULL && !PySequence_Check(args)) {
        /* check_args_iterable() may be called with a live exception:
         * clear it to prevent calling _PyObject_FunctionStr() with an
         * exception set. */
        _PyErr_Clear(tstate);
        PyObject *funcstr = _PyObject_FunctionStr(func);
        if (funcstr != NULL) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "%U argument after * must be an iterable, not %.200s",
                          funcstr, Py_TYPE(args)->tp_name);
            Py_DECREF(funcstr);
        }
        return -1;
    }
    return 0;
}

static void
format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
{
    /* _PyDict_MergeEx raises attribute
     * error (percolated from an attempt
     * to get 'keys' attribute) instead of
     * a type error if its second argument
     * is not a mapping.
     */
    if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) {
        _PyErr_Clear(tstate);
        PyObject *funcstr = _PyObject_FunctionStr(func);
        if (funcstr != NULL) {
            _PyErr_Format(
                tstate, PyExc_TypeError,
                "%U argument after ** must be a mapping, not %.200s",
                funcstr, Py_TYPE(kwargs)->tp_name);
            Py_DECREF(funcstr);
        }
    }
    else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
        PyObject *exc, *val, *tb;
        _PyErr_Fetch(tstate, &exc, &val, &tb);
        if (val && PyTuple_Check(val) && PyTuple_GET_SIZE(val) == 1) {
            _PyErr_Clear(tstate);
            PyObject *funcstr = _PyObject_FunctionStr(func);
            if (funcstr != NULL) {
                PyObject *key = PyTuple_GET_ITEM(val, 0);
                _PyErr_Format(
                    tstate, PyExc_TypeError,
                    "%U got multiple values for keyword argument '%S'",
                    funcstr, key);
                Py_DECREF(funcstr);
            }
            Py_XDECREF(exc);
            Py_XDECREF(val);
            Py_XDECREF(tb);
        }
        else {
            _PyErr_Restore(tstate, exc, val, tb);
        }
    }
}

static void
format_exc_check_arg(PyThreadState *tstate, PyObject *exc,
                     const char *format_str, PyObject *obj)
{
    const char *obj_str;

    if (!obj)
        return;

    obj_str = PyUnicode_AsUTF8(obj);
    if (!obj_str)
        return;

    _PyErr_Format(tstate, exc, format_str, obj_str);

    if (exc == PyExc_NameError) {
        // Include the name in the NameError exceptions to offer suggestions later.
        _Py_IDENTIFIER(name);
        PyObject *type, *value, *traceback;
        PyErr_Fetch(&type, &value, &traceback);
        PyErr_NormalizeException(&type, &value, &traceback);
        if (PyErr_GivenExceptionMatches(value, PyExc_NameError)) {
            // We do not care if this fails because we are going to restore the
            // NameError anyway.
            (void)_PyObject_SetAttrId(value, &PyId_name, obj);
        }
        PyErr_Restore(type, value, traceback);
    }
}

static void
format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
{
    PyObject *name;
    /* Don't stomp existing exception */
    if (_PyErr_Occurred(tstate))
        return;
    name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
    if (oparg < co->co_nplaincellvars + co->co_nlocals) {
        format_exc_check_arg(tstate, PyExc_UnboundLocalError,
                             UNBOUNDLOCAL_ERROR_MSG, name);
    } else {
        format_exc_check_arg(tstate, PyExc_NameError,
                             UNBOUNDFREE_ERROR_MSG, name);
    }
}

static void
format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevopcode, int prevopcode)
{
    if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) {
        if (prevopcode == BEFORE_ASYNC_WITH) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "'async with' received an object from __aenter__ "
                          "that does not implement __await__: %.100s",
                          type->tp_name);
        }
        else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_NO_KW && prevprevopcode == DUP_TOP)) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "'async with' received an object from __aexit__ "
                          "that does not implement __await__: %.100s",
                          type->tp_name);
        }
    }
}

#ifdef DYNAMIC_EXECUTION_PROFILE

static PyObject *
getarray(long a[256])
{
    int i;
    PyObject *l = PyList_New(256);
    if (l == NULL) return NULL;
    for (i = 0; i < 256; i++) {
        PyObject *x = PyLong_FromLong(a[i]);
        if (x == NULL) {
            Py_DECREF(l);
            return NULL;
        }
        PyList_SET_ITEM(l, i, x);
    }
    for (i = 0; i < 256; i++)
        a[i] = 0;
    return l;
}

PyObject *
_Py_GetDXProfile(PyObject *self, PyObject *args)
{
#ifndef DXPAIRS
    return getarray(dxp);
#else
    int i;
    PyObject *l = PyList_New(257);
    if (l == NULL) return NULL;
    for (i = 0; i < 257; i++) {
        PyObject *x = getarray(dxpairs[i]);
        if (x == NULL) {
            Py_DECREF(l);
            return NULL;
        }
        PyList_SET_ITEM(l, i, x);
    }
    return l;
#endif
}

#endif

Py_ssize_t
_PyEval_RequestCodeExtraIndex(freefunc free)
{
    PyInterpreterState *interp = _PyInterpreterState_GET();
    Py_ssize_t new_index;

    if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) {
        return -1;
    }
    new_index = interp->co_extra_user_count++;
    interp->co_extra_freefuncs[new_index] = free;
    return new_index;
}

static void
dtrace_function_entry(InterpreterFrame *frame)
{
    const char *filename;
    const char *funcname;
    int lineno;

    PyCodeObject *code = frame->f_code;
    filename = PyUnicode_AsUTF8(code->co_filename);
    funcname = PyUnicode_AsUTF8(code->co_name);
    lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT));

    PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno);
}

static void
dtrace_function_return(InterpreterFrame *frame)
{
    const char *filename;
    const char *funcname;
    int lineno;

    PyCodeObject *code = frame->f_code;
    filename = PyUnicode_AsUTF8(code->co_filename);
    funcname = PyUnicode_AsUTF8(code->co_name);
    lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti*sizeof(_Py_CODEUNIT));

    PyDTrace_FUNCTION_RETURN(filename, funcname, lineno);
}

/* DTrace equivalent of maybe_call_line_trace. */
static void
maybe_dtrace_line(InterpreterFrame *frame,
                  PyTraceInfo *trace_info,
                  int instr_prev)
{
    const char *co_filename, *co_name;

    /* If the last instruction executed isn't in the current
       instruction window, reset the window.
    */
    initialize_trace_info(trace_info, frame);
    int lastline = _PyCode_CheckLineNumber(instr_prev*sizeof(_Py_CODEUNIT), &trace_info->bounds);
    int line = _PyCode_CheckLineNumber(frame->f_lasti*sizeof(_Py_CODEUNIT), &trace_info->bounds);
    if (line != -1) {
        /* Trace backward edges or first instruction of a new line */
        if (frame->f_lasti < instr_prev ||
            (line != lastline && frame->f_lasti*sizeof(_Py_CODEUNIT) == (unsigned int)trace_info->bounds.ar_start))
        {
            co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename);
            if (!co_filename) {
                co_filename = "?";
            }
            co_name = PyUnicode_AsUTF8(frame->f_code->co_name);
            if (!co_name) {
                co_name = "?";
            }
            PyDTrace_LINE(co_filename, co_name, line);
        }
    }
}

/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions
   for the limited API. */

#undef Py_EnterRecursiveCall

int Py_EnterRecursiveCall(const char *where)
{
    return _Py_EnterRecursiveCall_inline(where);
}

#undef Py_LeaveRecursiveCall

void Py_LeaveRecursiveCall(void)
{
    _Py_LeaveRecursiveCall_inline();
}