diff options
| -rw-r--r-- | Include/internal/pycore_magic_number.h | 280 | ||||
| -rw-r--r-- | InternalDocs/compiler.md | 2 | ||||
| -rw-r--r-- | Lib/importlib/_bootstrap_external.py | 279 | ||||
| -rw-r--r-- | Lib/importlib/util.py | 3 | ||||
| -rw-r--r-- | Lib/test/test_import/__init__.py | 9 | ||||
| -rw-r--r-- | Lib/zipimport.py | 2 | ||||
| -rw-r--r-- | Python/import.c | 27 |
7 files changed, 307 insertions, 295 deletions
diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h new file mode 100644 index 0000000..3af6817 --- /dev/null +++ b/Include/internal/pycore_magic_number.h @@ -0,0 +1,280 @@ +#ifndef Py_INTERNAL_MAGIC_NUMBER_H +#define Py_INTERNAL_MAGIC_NUMBER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* + +Magic number to reject .pyc files generated by other Python versions. +It should change for each incompatible change to the bytecode. + +PYC_MAGIC_NUMBER must change whenever the bytecode emitted by the compiler may +no longer be understood by older implementations of the eval loop (usually due +to the addition of new opcodes). + +The value of CR and LF is incorporated so if you ever read or write +a .pyc file in text mode the magic number will be wrong; also, the +Apple MPW compiler swaps their values, botching string constants. + +There were a variety of old schemes for setting the magic number. Starting with +Python 3.11, Python 3.n starts with magic number 2900+50n. Within each minor +version, the magic number is incremented by 1 each time the file format changes. + +Known values: + Python 1.5: 20121 + Python 1.5.1: 20121 + Python 1.5.2: 20121 + Python 1.6: 50428 + Python 2.0: 50823 + Python 2.0.1: 50823 + Python 2.1: 60202 + Python 2.1.1: 60202 + Python 2.1.2: 60202 + Python 2.2: 60717 + Python 2.3a0: 62011 + Python 2.3a0: 62021 + Python 2.3a0: 62011 (!) + Python 2.4a0: 62041 + Python 2.4a3: 62051 + Python 2.4b1: 62061 + Python 2.5a0: 62071 + Python 2.5a0: 62081 (ast-branch) + Python 2.5a0: 62091 (with) + Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) + Python 2.5b3: 62101 (fix wrong code: for x, in ...) + Python 2.5b3: 62111 (fix wrong code: x += yield) + Python 2.5c1: 62121 (fix wrong lnotab with for loops and + storing constants that should have been removed) + Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) + Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) + Python 2.6a1: 62161 (WITH_CLEANUP optimization) + Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) + Python 2.7a0: 62181 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) + Python 2.7a0 62191 (introduce SETUP_WITH) + Python 2.7a0 62201 (introduce BUILD_SET) + Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) + Python 3000: 3000 + 3010 (removed UNARY_CONVERT) + 3020 (added BUILD_SET) + 3030 (added keyword-only parameters) + 3040 (added signature annotations) + 3050 (print becomes a function) + 3060 (PEP 3115 metaclass syntax) + 3061 (string literals become unicode) + 3071 (PEP 3109 raise changes) + 3081 (PEP 3137 make __file__ and __name__ unicode) + 3091 (kill str8 interning) + 3101 (merge from 2.6a0, see 62151) + 3103 (__file__ points to source file) + Python 3.0a4: 3111 (WITH_CLEANUP optimization). + Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT + #3021) + Python 3.1a1: 3141 (optimize list, set and dict comprehensions: + change LIST_APPEND and SET_ADD, add MAP_ADD #2183) + Python 3.1a1: 3151 (optimize conditional branches: + introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE + #4715) + Python 3.2a1: 3160 (add SETUP_WITH #6101) + Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) + Python 3.2a3 3180 (add DELETE_DEREF #4617) + Python 3.3a1 3190 (__class__ super closure changed) + Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) + Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) + Python 3.3a2 3220 (changed PEP 380 implementation #14230) + Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) + Python 3.4a1 3250 (evaluate positional default arguments before + keyword-only defaults #16967) + Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override + free vars #17853) + Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) + Python 3.4a1 3280 (remove implicit class argument) + Python 3.4a4 3290 (changes to __qualname__ computation #19301) + Python 3.4a4 3300 (more changes to __qualname__ computation #19301) + Python 3.4rc2 3310 (alter __qualname__ computation #20625) + Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) + Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) + Python 3.5b2 3340 (fix dictionary display evaluation order #11205) + Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) + Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) + Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) + Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) + Python 3.6a2 3370 (16 bit wordcode #26647) + Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) + Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE + #27095) + Python 3.6b1 3373 (add BUILD_STRING opcode #27078) + Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes + #27985) + Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL + #27213) + Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) + Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) + Python 3.6rc1 3379 (more thorough __class__ validation #23722) + Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) + Python 3.7a2 3391 (update GET_AITER #31709) + Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) + Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) + Python 3.7b5 3394 (restored docstring as the first stmt in the body; + this might affected the first line number #32911) + Python 3.8a1 3400 (move frame block handling to compiler #17611) + Python 3.8a1 3401 (add END_ASYNC_FOR #33041) + Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) + Python 3.8b2 3411 (Reverse evaluation order of key: value in dict + comprehensions #35224) + Python 3.8b2 3412 (Swap the position of positional args and positional + only args in ast.arguments #37593) + Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) + Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) + Python 3.9a0 3421 (simplified bytecode for with blocks #32949) + Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) + Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) + Python 3.9a2 3424 (simplify bytecodes for *value unpacking) + Python 3.9a2 3425 (simplify bytecodes for **value unpacking) + Python 3.10a1 3430 (Make 'annotations' future by default) + Python 3.10a1 3431 (New line number table format -- PEP 626) + Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) + Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) + Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) + Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). + Python 3.10b1 3436 (Add GEN_START bytecode #43683) + Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) + Python 3.10b1 3438 Safer line number table handling. + Python 3.10b1 3439 (Add ROT_N) + Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) + Python 3.11a1 3451 (Add CALL_METHOD_KW) + Python 3.11a1 3452 (drop nlocals from marshaled code objects) + Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) + Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) + Python 3.11a1 3455 (add MAKE_CELL bpo-43693) + Python 3.11a1 3456 (interleave cell args bpo-43693) + Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) + Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) + Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) + Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) + Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) + Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change + MATCH_CLASS and MATCH_KEYS, and add COPY) + Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the + active exception) + Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*INPLACE_* into + BINARY_OP) + Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) + Python 3.11a4 3466 (bpo-45292: PEP-654 except*) + Python 3.11a4 3467 (Change CALL_xxx opcodes) + Python 3.11a4 3468 (Add SEND opcode) + Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) + Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) + Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) + Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) + Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) + Python 3.11a4 3474 (Add RESUME opcode) + Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) + Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) + Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and + ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) + Python 3.11a5 3478 (New CALL opcodes) + Python 3.11a5 3479 (Add PUSH_NULL opcode) + Python 3.11a5 3480 (New CALL opcodes, second iteration) + Python 3.11a5 3481 (Use inline cache for BINARY_OP) + Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) + Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) + Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and + STORE_ATTR) + Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) + Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) + Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) + Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) + Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) + Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) + Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, + add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) + Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) + Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) + Python 3.11a7 3494 (New location info table) + Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) + Python 3.12a1 3500 (Remove PRECALL opcode) + Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) + Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) + Python 3.12a1 3503 (Shrink LOAD_METHOD cache) + Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) + Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) + Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) + Python 3.12a1 3507 (Set lineno of module's RESUME to 0) + Python 3.12a1 3508 (Add CLEANUP_THROW) + Python 3.12a1 3509 (Conditional jumps only jump forward) + Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) + Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) + Python 3.12a2 3512 (Remove all unused consts from code objects) + Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) + Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) + Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) + Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) + Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) + Python 3.12a6 3518 (Add RETURN_CONST instruction) + Python 3.12a6 3519 (Modify SEND instruction) + Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) + Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) + Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) + Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) + Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) + Python 3.12b1 3525 (Shrink the CALL caches) + Python 3.12b1 3526 (Add instrumentation support) + Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) + Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) + Python 3.12b1 3529 (Inline list/dict/set comprehensions) + Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) + Python 3.12b1 3531 (Add PEP 695 changes) + Python 3.13a1 3550 (Plugin optimizer support) + Python 3.13a1 3551 (Compact superinstructions) + Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) + Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) + Python 3.13a1 3554 (more efficient bytecodes for f-strings) + Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) + Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) + Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) + Python 3.13a1 3558 (Reorder the stack items for CALL) + Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) + Python 3.13a1 3560 (Add RESUME_CHECK instruction) + Python 3.13a1 3561 (Add cache entry to branch instructions) + Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) + Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) + Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) + Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) + Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) + Python 3.13a1 3567 (Reimplement line number propagation by the compiler) + Python 3.13a1 3568 (Change semantics of END_FOR) + Python 3.13a5 3569 (Specialize CONTAINS_OP) + Python 3.13a6 3570 (Add __firstlineno__ class attribute) + Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) + Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) + Python 3.14a1 3602 (Add LOAD_SPECIAL. Remove BEFORE_WITH and BEFORE_ASYNC_WITH) + Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) + + Python 3.15 will start with 3650 + + Please don't copy-paste the same pre-release tag for new entries above!!! + You should always use the *upcoming* tag. For example, if 3.12a6 came out + a week ago, I should put "Python 3.12a7" next to my new magic number. + +Whenever PYC_MAGIC_NUMBER is changed, the ranges in the magic_values array in +PC/launcher.c must also be updated. + +*/ + +#define PYC_MAGIC_NUMBER 3603 +/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes + (little-endian) and then appending b'\r\n'. */ +#define PYC_MAGIC_NUMBER_TOKEN \ + ((uint32_t)PYC_MAGIC_NUMBER | ((uint32_t)'\r' << 16) | ((uint32_t)'\n' << 24)) + + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MAGIC_NUMBER_H diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index b3dc0a4..52a3ab2 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -616,7 +616,7 @@ Important files * [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) : opcode utilities exposed to Python. - * [Lib/importlib/_bootstrap_external.py](https://github.com/python/cpython/blob/main/Lib/importlib/_bootstrap_external.py) + * [Include/core/pycore_magic_number.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_magic_number.h) : Home of the magic number (named ``MAGIC_NUMBER``) for bytecode versioning. diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 2bb44b2..4d154dc 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -221,280 +221,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) -# Python 3.13a1 3550 (Plugin optimizer support) -# Python 3.13a1 3551 (Compact superinstructions) -# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) -# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) -# Python 3.13a1 3554 (more efficient bytecodes for f-strings) -# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) -# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) -# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) -# Python 3.13a1 3558 (Reorder the stack items for CALL) -# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) -# Python 3.13a1 3560 (Add RESUME_CHECK instruction) -# Python 3.13a1 3561 (Add cache entry to branch instructions) -# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) -# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) -# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) -# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) -# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) -# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) -# Python 3.13a1 3568 (Change semantics of END_FOR) -# Python 3.13a5 3569 (Specialize CONTAINS_OP) -# Python 3.13a6 3570 (Add __firstlineno__ class attribute) -# Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT) -# Python 3.14a1 3601 (Fix miscompilation of private names in generic classes) -# Python 3.14a1 3602 (Add LOAD_SPECIAL. Remove BEFORE_WITH and BEFORE_ASYNC_WITH) -# Python 3.14a1 3603 (Remove BUILD_CONST_KEY_MAP) - -# Python 3.15 will start with 3650 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (3603).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = (_imp.pyc_magic_number).to_bytes(2, 'little') + b'\r\n' _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -1133,7 +860,7 @@ class SourceLoader(_LoaderBasics): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1162,7 +889,7 @@ class SourceLoader(_LoaderBasics): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 8403ef9..2b564e9 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -18,7 +17,7 @@ import types def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index e29097b..56c6ffe 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3113,6 +3113,15 @@ class CAPITests(unittest.TestCase): self.assertIs(mod, sys.modules[name]) +@cpython_only +class TestMagicNumber(unittest.TestCase): + def test_magic_number_endianness(self): + magic_number = (_imp.pyc_magic_number).to_bytes(2, 'little') + b'\r\n' + raw_magic_number = int.from_bytes(magic_number, 'little') + + self.assertEqual(raw_magic_number, _imp.pyc_magic_number_token) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 68f031f..f2724dd 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -705,7 +705,7 @@ def _unmarshal_code(self, pathname, fullpath, fullname, data): source_bytes = _get_pyc_source(self, fullpath) if source_bytes is not None: source_hash = _imp.source_hash( - _bootstrap_external._RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) diff --git a/Python/import.c b/Python/import.c index 40b7fea..540874a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -6,6 +6,7 @@ #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // struct _import_runtime_state +#include "pycore_magic_number.h" // PYC_MAGIC_NUMBER #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // _Py_SetImmortal() #include "pycore_pyerrors.h" // _PyErr_SetString() @@ -2475,23 +2476,9 @@ _PyImport_GetBuiltinModuleNames(void) long PyImport_GetMagicNumber(void) { - long res; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *external, *pyc_magic; - - external = PyObject_GetAttrString(IMPORTLIB(interp), "_bootstrap_external"); - if (external == NULL) - return -1; - pyc_magic = PyObject_GetAttrString(external, "_RAW_MAGIC_NUMBER"); - Py_DECREF(external); - if (pyc_magic == NULL) - return -1; - res = PyLong_AsLong(pyc_magic); - Py_DECREF(pyc_magic); - return res; + return PYC_MAGIC_NUMBER_TOKEN; } - extern const char * _PySys_ImplCacheTag; const char * @@ -4823,6 +4810,16 @@ imp_module_exec(PyObject *module) return -1; } + if (PyModule_AddIntConstant(module, "pyc_magic_number", PYC_MAGIC_NUMBER) < 0) { + return -1; + } + + if (PyModule_AddIntConstant( + module, "pyc_magic_number_token", PYC_MAGIC_NUMBER_TOKEN) < 0) + { + return -1; + } + return 0; } |