From 72119d16a5f658939809febef29dadeca02cf34d Mon Sep 17 00:00:00 2001
From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com>
Date: Wed, 23 Aug 2023 18:39:00 +0100
Subject: gh-105481: remove regen-opcode. Generated _PyOpcode_Caches in
regen-cases. (#108367)
---
Include/internal/pycore_opcode.h | 40 -------------
Include/internal/pycore_opcode_metadata.h | 20 +++++++
Include/internal/pycore_opcode_utils.h | 3 +-
Lib/opcode.py | 54 ++++++++---------
Makefile.pre.in | 14 +----
.../2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst | 2 +
Objects/codeobject.c | 3 +-
Objects/frameobject.c | 3 +-
PCbuild/regen.targets | 14 +----
Python/assemble.c | 3 +-
Python/ceval.c | 3 +-
Python/compile.c | 1 +
Python/executor.c | 2 +
Python/instrumentation.c | 6 +-
Python/optimizer.c | 1 -
Python/optimizer_analysis.c | 1 -
Python/specialize.c | 5 +-
Tools/build/generate_opcode_h.py | 67 ----------------------
Tools/cases_generator/generate_cases.py | 12 ++++
19 files changed, 78 insertions(+), 176 deletions(-)
delete mode 100644 Include/internal/pycore_opcode.h
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst
delete mode 100644 Tools/build/generate_opcode_h.py
diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h
deleted file mode 100644
index b47e796..0000000
--- a/Include/internal/pycore_opcode.h
+++ /dev/null
@@ -1,40 +0,0 @@
-// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py
-
-#ifndef Py_INTERNAL_OPCODE_H
-#define Py_INTERNAL_OPCODE_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef Py_BUILD_CORE
-# error "this header requires Py_BUILD_CORE define"
-#endif
-
-#include "opcode.h"
-
-extern const uint8_t _PyOpcode_Caches[256];
-
-#ifdef NEED_OPCODE_TABLES
-
-const uint8_t _PyOpcode_Caches[256] = {
- [LOAD_GLOBAL] = 4,
- [BINARY_OP] = 1,
- [UNPACK_SEQUENCE] = 1,
- [COMPARE_OP] = 1,
- [BINARY_SUBSCR] = 1,
- [FOR_ITER] = 1,
- [LOAD_SUPER_ATTR] = 1,
- [LOAD_ATTR] = 9,
- [STORE_ATTR] = 4,
- [CALL] = 3,
- [STORE_SUBSCR] = 1,
- [SEND] = 1,
- [JUMP_BACKWARD] = 1,
- [TO_BOOL] = 3,
-};
-#endif // NEED_OPCODE_TABLES
-
-#ifdef __cplusplus
-}
-#endif
-#endif // !Py_INTERNAL_OPCODE_H
diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h
index e35db0c..cc8894a 100644
--- a/Include/internal/pycore_opcode_metadata.h
+++ b/Include/internal/pycore_opcode_metadata.h
@@ -1814,6 +1814,26 @@ const char *const _PyOpcode_OpName[268] = {
};
#endif // NEED_OPCODE_METADATA
+extern const uint8_t _PyOpcode_Caches[256];
+#ifdef NEED_OPCODE_METADATA
+const uint8_t _PyOpcode_Caches[256] = {
+ [TO_BOOL] = 3,
+ [BINARY_OP] = 1,
+ [BINARY_SUBSCR] = 1,
+ [STORE_SUBSCR] = 1,
+ [SEND] = 1,
+ [UNPACK_SEQUENCE] = 1,
+ [STORE_ATTR] = 4,
+ [LOAD_GLOBAL] = 4,
+ [LOAD_SUPER_ATTR] = 1,
+ [LOAD_ATTR] = 9,
+ [COMPARE_OP] = 1,
+ [FOR_ITER] = 1,
+ [CALL] = 3,
+ [JUMP_BACKWARD] = 1,
+};
+#endif // NEED_OPCODE_METADATA
+
extern const uint8_t _PyOpcode_Deopt[256];
#ifdef NEED_OPCODE_METADATA
const uint8_t _PyOpcode_Deopt[256] = {
diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h
index f176129..c4acb00 100644
--- a/Include/internal/pycore_opcode_utils.h
+++ b/Include/internal/pycore_opcode_utils.h
@@ -8,8 +8,7 @@ extern "C" {
# error "this header requires Py_BUILD_CORE define"
#endif
-#include "pycore_opcode.h" // JUMP_FORWARD
-
+#include "opcode_ids.h"
#define MAX_REAL_OPCODE 254
diff --git a/Lib/opcode.py b/Lib/opcode.py
index f848752..386a2fb 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -5,48 +5,40 @@ operate on bytecodes (e.g. peephole optimizers).
"""
-# Note that __all__ is further extended below
-__all__ = ["cmp_op", "stack_effect", "hascompare"]
+__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap",
+ "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname",
+ "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"]
import _opcode
from _opcode import stack_effect
-import sys
-# The build uses older versions of Python which do not have _opcode_metadata
-if sys.version_info[:2] >= (3, 13):
- from _opcode_metadata import _specializations, _specialized_opmap
- from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE
- EXTENDED_ARG = opmap['EXTENDED_ARG']
+from _opcode_metadata import (_specializations, _specialized_opmap, opmap,
+ HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE)
+EXTENDED_ARG = opmap['EXTENDED_ARG']
- opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)]
- for op, i in opmap.items():
- opname[i] = op
-
- __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"])
+opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)]
+for op, i in opmap.items():
+ opname[i] = op
cmp_op = ('<', '<=', '==', '!=', '>', '>=')
-# The build uses older versions of Python which do not have _opcode.has_* functions
-if sys.version_info[:2] >= (3, 13):
- # These lists are documented as part of the dis module's API
- hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
- hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
- hasname = [op for op in opmap.values() if _opcode.has_name(op)]
- hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
- hasjrel = hasjump # for backward compatibility
- hasjabs = []
- hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
- haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
- hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
+# These lists are documented as part of the dis module's API
+hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
+hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
+hasname = [op for op in opmap.values() if _opcode.has_name(op)]
+hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
+hasjrel = hasjump # for backward compatibility
+hasjabs = []
+hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
+haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
+hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
- __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel",
- "hasjabs", "hasfree", "haslocal", "hasexc"])
- _intrinsic_1_descs = _opcode.get_intrinsic1_descs()
- _intrinsic_2_descs = _opcode.get_intrinsic2_descs()
- _nb_ops = _opcode.get_nb_ops()
+_intrinsic_1_descs = _opcode.get_intrinsic1_descs()
+_intrinsic_2_descs = _opcode.get_intrinsic2_descs()
+_nb_ops = _opcode.get_nb_ops()
- hascompare = [opmap["COMPARE_OP"]]
+hascompare = [opmap["COMPARE_OP"]]
_cache_format = {
"LOAD_GLOBAL": {
diff --git a/Makefile.pre.in b/Makefile.pre.in
index d66764e..54b64e1 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1321,7 +1321,7 @@ regen-limited-abi: all
# Regenerate all generated files
.PHONY: regen-all
-regen-all: regen-cases regen-opcode regen-typeslots \
+regen-all: regen-cases regen-typeslots \
regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \
regen-pegen-metaparser regen-pegen regen-test-frozenmain \
regen-test-levenshtein regen-global-objects
@@ -1425,15 +1425,6 @@ regen-ast:
$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast_state.h $(srcdir)/Include/internal/pycore_ast_state.h.new
$(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
-.PHONY: regen-opcode
-regen-opcode:
- # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py
- # using Tools/build/generate_opcode_h.py
- $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \
- $(srcdir)/Lib/opcode.py \
- $(srcdir)/Include/internal/pycore_opcode.h.new
- $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new
-
.PHONY: regen-token
regen-token:
# Regenerate Doc/library/token-list.inc from Grammar/Tokens
@@ -1651,6 +1642,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/object.h \
$(srcdir)/Include/objimpl.h \
$(srcdir)/Include/opcode.h \
+ $(srcdir)/Include/opcode_ids.h \
$(srcdir)/Include/osdefs.h \
$(srcdir)/Include/osmodule.h \
$(srcdir)/Include/patchlevel.h \
@@ -1790,7 +1782,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_object_state.h \
$(srcdir)/Include/internal/pycore_obmalloc.h \
$(srcdir)/Include/internal/pycore_obmalloc_init.h \
- $(srcdir)/Include/internal/pycore_opcode.h \
+ $(srcdir)/Include/internal/pycore_opcode_metadata.h \
$(srcdir)/Include/internal/pycore_opcode_utils.h \
$(srcdir)/Include/internal/pycore_optimizer.h \
$(srcdir)/Include/internal/pycore_pathconfig.h \
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst
new file mode 100644
index 0000000..19746eb
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst
@@ -0,0 +1,2 @@
+The regen-opcode build stage was removed and its work is now done in
+regen-cases.
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index dca5804..70a0c2e 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -6,8 +6,7 @@
#include "pycore_code.h" // _PyCodeConstructor
#include "pycore_frame.h" // FRAME_SPECIALS_SIZE
#include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs
-#include "pycore_opcode.h" // _PyOpcode_Caches
-#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt
+#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_setobject.h" // _PySet_NextEntry()
#include "pycore_tuple.h" // _PyTuple_ITEMS()
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 80e118e..28f5a5a 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -6,8 +6,7 @@
#include "pycore_function.h" // _PyFunction_FromConstructor()
#include "pycore_moduleobject.h" // _PyModule_GetDict()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
-#include "pycore_opcode.h" // _PyOpcode_Caches
-#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt
+#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches
#include "frameobject.h" // PyFrameObject
diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets
index 2ff18a8..cc9469c 100644
--- a/PCbuild/regen.targets
+++ b/PCbuild/regen.targets
@@ -13,8 +13,6 @@
<_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c">
-C
- <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
- <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" />
<_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
rst
@@ -34,7 +32,7 @@
-
@@ -55,14 +53,6 @@
WorkingDirectory="$(PySourcePath)" />
-
-
-
-
-
@@ -89,7 +79,7 @@
+ DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects">
diff --git a/Python/assemble.c b/Python/assemble.c
index 4f66cf2..c770fd1 100644
--- a/Python/assemble.c
+++ b/Python/assemble.c
@@ -3,9 +3,8 @@
#include "Python.h"
#include "pycore_code.h" // write_location_entry_start()
#include "pycore_compile.h"
-#include "pycore_opcode.h" // _PyOpcode_Caches[] and opcode category macros
#include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE
-#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR
+#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches
#define DEFAULT_CODE_SIZE 128
diff --git a/Python/ceval.c b/Python/ceval.c
index f7dfaeb..55dfe6b 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -14,8 +14,7 @@
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_object.h" // _PyObject_GC_TRACK()
-#include "pycore_opcode.h" // EXTRA_CASES
-#include "pycore_opcode_metadata.h"
+#include "pycore_opcode_metadata.h" // EXTRA_CASES
#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
diff --git a/Python/compile.c b/Python/compile.c
index b67a188..6b816b4 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -24,6 +24,7 @@
#include
#include "Python.h"
+#include "opcode.h"
#include "pycore_ast.h" // _PyAST_GetDocString()
#define NEED_OPCODE_TABLES
#include "pycore_opcode_utils.h"
diff --git a/Python/executor.c b/Python/executor.c
index 88c039d..0ff5106 100644
--- a/Python/executor.c
+++ b/Python/executor.c
@@ -1,5 +1,7 @@
#include "Python.h"
+#include "opcode.h"
+
#include "pycore_call.h"
#include "pycore_ceval.h"
#include "pycore_dict.h"
diff --git a/Python/instrumentation.c b/Python/instrumentation.c
index f77c2e6..8c7a3a0 100644
--- a/Python/instrumentation.c
+++ b/Python/instrumentation.c
@@ -1,4 +1,7 @@
#include "Python.h"
+
+#include "opcode_ids.h"
+
#include "pycore_call.h"
#include "pycore_frame.h"
#include "pycore_interp.h"
@@ -6,8 +9,7 @@
#include "pycore_modsupport.h" // _PyModule_CreateInitialized()
#include "pycore_namespace.h"
#include "pycore_object.h"
-#include "pycore_opcode.h"
-#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE
+#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches
#include "pycore_pyerrors.h"
#include "pycore_pystate.h" // _PyInterpreterState_GET()
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 5751840..bbc1259 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -1,7 +1,6 @@
#include "Python.h"
#include "opcode.h"
#include "pycore_interp.h"
-#include "pycore_opcode.h"
#include "pycore_opcode_metadata.h"
#include "pycore_opcode_utils.h"
#include "pycore_optimizer.h"
diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index e48e018..2d177f1 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -1,7 +1,6 @@
#include "Python.h"
#include "opcode.h"
#include "pycore_interp.h"
-#include "pycore_opcode.h"
#include "pycore_opcode_metadata.h"
#include "pycore_opcode_utils.h"
#include "pycore_pystate.h" // _PyInterpreterState_GET()
diff --git a/Python/specialize.c b/Python/specialize.c
index 2d514c0..a467f16 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -1,4 +1,7 @@
#include "Python.h"
+
+#include "opcode.h"
+
#include "pycore_code.h"
#include "pycore_descrobject.h" // _PyMethodWrapper_Type
#include "pycore_dict.h"
@@ -7,7 +10,7 @@
#include "pycore_long.h"
#include "pycore_moduleobject.h"
#include "pycore_object.h"
-#include "pycore_opcode.h" // _PyOpcode_Caches
+#include "pycore_opcode_metadata.h" // _PyOpcode_Caches
#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock()
diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py
deleted file mode 100644
index 643918c..0000000
--- a/Tools/build/generate_opcode_h.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# This script generates the pycore_opcode.h header file.
-
-import sys
-import tokenize
-
-SCRIPT_NAME = "Tools/build/generate_opcode_h.py"
-PYTHON_OPCODE = "Lib/opcode.py"
-
-internal_header = f"""
-// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}
-
-#ifndef Py_INTERNAL_OPCODE_H
-#define Py_INTERNAL_OPCODE_H
-#ifdef __cplusplus
-extern "C" {{
-#endif
-
-#ifndef Py_BUILD_CORE
-# error "this header requires Py_BUILD_CORE define"
-#endif
-
-#include "opcode.h"
-""".lstrip()
-
-internal_footer = """
-#ifdef __cplusplus
-}
-#endif
-#endif // !Py_INTERNAL_OPCODE_H
-"""
-
-DEFINE = "#define {:<38} {:>3}\n"
-
-UINT32_MASK = (1<<32)-1
-
-def get_python_module_dict(filename):
- mod = {}
- with tokenize.open(filename) as fp:
- code = fp.read()
- exec(code, mod)
- return mod
-
-def main(opcode_py,
- internal_opcode_h='Include/internal/pycore_opcode.h'):
-
- opcode = get_python_module_dict(opcode_py)
-
- with open(internal_opcode_h, 'w') as iobj:
- iobj.write(internal_header)
-
- iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
- iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
-
- iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
- for name, entries in opcode["_inline_cache_entries"].items():
- iobj.write(f" [{name}] = {entries},\n")
- iobj.write("};\n")
-
- iobj.write("#endif // NEED_OPCODE_TABLES\n")
-
- iobj.write(internal_footer)
-
- print(f"{internal_opcode_h} regenerated from {opcode_py}")
-
-
-if __name__ == '__main__':
- main(sys.argv[1], sys.argv[2])
diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py
index f605dcc..8f9a650 100644
--- a/Tools/cases_generator/generate_cases.py
+++ b/Tools/cases_generator/generate_cases.py
@@ -540,6 +540,18 @@ class Generator(Analyzer):
for name in self.opmap:
self.out.emit(f'[{name}] = "{name}",')
+ with self.metadata_item(
+ f"const uint8_t _PyOpcode_Caches[256]",
+ "=",
+ ";",
+ ):
+ for name, _ in self.families.items():
+ instr = self.instrs[name]
+ if instr.cache_offset > 0:
+ self.out.emit(f'[{name}] = {instr.cache_offset},')
+ # Irregular case:
+ self.out.emit('[JUMP_BACKWARD] = 1,')
+
deoptcodes = {}
for name, op in self.opmap.items():
if op < 256:
--
cgit v0.12