diff options
Diffstat (limited to 'Tools')
| -rw-r--r-- | Tools/c-analyzer/TODO | 2 | ||||
| -rw-r--r-- | Tools/scripts/generate_global_objects.py | 495 |
2 files changed, 486 insertions, 11 deletions
diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO index 4b9b285..55338eb 100644 --- a/Tools/c-analyzer/TODO +++ b/Tools/c-analyzer/TODO @@ -156,8 +156,6 @@ Objects/codeobject.c:PyCode_NewEmpty():emptystring static PyObject Objects/exceptions.c:_check_for_legacy_statements():exec_prefix static PyObject *exec_prefix Objects/exceptions.c:_check_for_legacy_statements():print_prefix static PyObject *print_prefix Objects/funcobject.c:PyFunction_NewWithQualName():__name__ static PyObject *__name__ -Objects/typeobject.c:object___reduce_ex___impl():objreduce static PyObject *objreduce -Objects/typeobject.c:resolve_slotdups():pname static PyObject *pname Objects/unicodeobject.c:unicode_empty static PyObject *unicode_empty Objects/unicodeobject.c:unicode_latin1 static PyObject *unicode_latin1[256] Python/_warnings.c:is_internal_frame():bootstrap_string static PyObject *bootstrap_string diff --git a/Tools/scripts/generate_global_objects.py b/Tools/scripts/generate_global_objects.py index a06d201..b184e74 100644 --- a/Tools/scripts/generate_global_objects.py +++ b/Tools/scripts/generate_global_objects.py @@ -1,9 +1,7 @@ -import argparse -import ast -import builtins -import collections import contextlib +import glob import os.path +import re import sys @@ -12,6 +10,298 @@ ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) INTERNAL = os.path.join(ROOT, 'Include', 'internal') +STRING_LITERALS = { + 'empty': '', + 'dot': '.', + 'comma_sep': ', ', + 'percent': '%', + 'dbl_percent': '%%', + + '"anonymous" labels': None, + 'anon_dictcomp': '<dictcomp>', + 'anon_genexpr': '<genexpr>', + 'anon_lambda': '<lambda>', + 'anon_listcomp': '<listcomp>', + 'anon_module': '<module>', + 'anon_setcomp': '<setcomp>', + 'anon_string': '<string>', + 'dot_locals': '.<locals>', +} +IDENTIFIERS = [ + 'Py_Repr', + 'TextIOWrapper', + 'WarningMessage', + '_', + '__IOBase_closed', + '__abc_tpflags__', + '__abs__', + '__abstractmethods__', + '__add__', + '__aenter__', + '__aexit__', + '__aiter__', + '__all__', + '__and__', + '__anext__', + '__annotations__', + '__args__', + '__await__', + '__bases__', + '__bool__', + '__build_class__', + '__builtins__', + '__bytes__', + '__call__', + '__cantrace__', + '__class__', + '__class_getitem__', + '__classcell__', + '__complex__', + '__contains__', + '__copy__', + '__del__', + '__delattr__', + '__delete__', + '__delitem__', + '__dict__', + '__dir__', + '__divmod__', + '__doc__', + '__enter__', + '__eq__', + '__exit__', + '__file__', + '__float__', + '__floordiv__', + '__format__', + '__fspath__', + '__ge__', + '__get__', + '__getattr__', + '__getattribute__', + '__getinitargs__', + '__getitem__', + '__getnewargs__', + '__getnewargs_ex__', + '__getstate__', + '__gt__', + '__hash__', + '__iadd__', + '__iand__', + '__ifloordiv__', + '__ilshift__', + '__imatmul__', + '__imod__', + '__import__', + '__imul__', + '__index__', + '__init__', + '__init_subclass__', + '__instancecheck__', + '__int__', + '__invert__', + '__ior__', + '__ipow__', + '__irshift__', + '__isabstractmethod__', + '__isub__', + '__iter__', + '__itruediv__', + '__ixor__', + '__le__', + '__len__', + '__length_hint__', + '__loader__', + '__lshift__', + '__lt__', + '__ltrace__', + '__main__', + '__matmul__', + '__missing__', + '__mod__', + '__module__', + '__mro_entries__', + '__mul__', + '__name__', + '__ne__', + '__neg__', + '__new__', + '__newobj__', + '__newobj_ex__', + '__next__', + '__note__', + '__or__', + '__origin__', + '__package__', + '__parameters__', + '__path__', + '__pos__', + '__pow__', + '__prepare__', + '__qualname__', + '__radd__', + '__rand__', + '__rdivmod__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rfloordiv__', + '__rlshift__', + '__rmatmul__', + '__rmod__', + '__rmul__', + '__ror__', + '__round__', + '__rpow__', + '__rrshift__', + '__rshift__', + '__rsub__', + '__rtruediv__', + '__rxor__', + '__set__', + '__set_name__', + '__setattr__', + '__setitem__', + '__setstate__', + '__sizeof__', + '__slotnames__', + '__slots__', + '__spec__', + '__str__', + '__sub__', + '__subclasscheck__', + '__subclasshook__', + '__truediv__', + '__trunc__', + '__warningregistry__', + '__weakref__', + '__xor__', + '_abc_impl', + '_blksize', + '_dealloc_warn', + '_finalizing', + '_find_and_load', + '_fix_up_module', + '_get_sourcefile', + '_handle_fromlist', + '_initializing', + '_is_text_encoding', + '_lock_unlock_module', + '_showwarnmsg', + '_shutdown', + '_slotnames', + '_strptime_time', + '_uninitialized_submodules', + '_warn_unawaited_coroutine', + '_xoptions', + 'add', + 'append', + 'big', + 'buffer', + 'builtins', + 'clear', + 'close', + 'code', + 'copy', + 'copyreg', + 'decode', + 'default', + 'defaultaction', + 'difference_update', + 'dispatch_table', + 'displayhook', + 'enable', + 'encoding', + 'end_lineno', + 'end_offset', + 'errors', + 'excepthook', + 'extend', + 'filename', + 'fileno', + 'fillvalue', + 'filters', + 'find_class', + 'flush', + 'get', + 'get_source', + 'getattr', + 'ignore', + 'importlib', + 'intersection', + 'isatty', + 'items', + 'iter', + 'keys', + 'last_traceback', + 'last_type', + 'last_value', + 'latin1', + 'lineno', + 'little', + 'match', + 'metaclass', + 'mode', + 'modules', + 'mro', + 'msg', + 'n_fields', + 'n_sequence_fields', + 'n_unnamed_fields', + 'name', + 'obj', + 'offset', + 'onceregistry', + 'open', + 'parent', + 'partial', + 'path', + 'peek', + 'persistent_id', + 'persistent_load', + 'print_file_and_line', + 'ps1', + 'ps2', + 'raw', + 'read', + 'read1', + 'readable', + 'readall', + 'readinto', + 'readinto1', + 'readline', + 'reducer_override', + 'reload', + 'replace', + 'reset', + 'return', + 'reversed', + 'seek', + 'seekable', + 'send', + 'setstate', + 'sort', + 'stderr', + 'stdin', + 'stdout', + 'strict', + 'symmetric_difference_update', + 'tell', + 'text', + 'threading', + 'throw', + 'unraisablehook', + 'values', + 'version', + 'warnings', + 'warnoptions', + 'writable', + 'write', + 'zipimporter', +] + + ####################################### # helpers @@ -64,6 +354,38 @@ START = '/* The following is auto-generated by Tools/scripts/generate_global_obj END = '/* End auto-generated code */' +def generate_global_strings(): + filename = os.path.join(INTERNAL, 'pycore_global_strings.h') + + # Read the non-generated part of the file. + with open(filename) as infile: + before = ''.join(iter_to_marker(infile, START))[:-1] + for _ in iter_to_marker(infile, END): + pass + after = infile.read()[:-1] + + # Generate the file. + with open(filename, 'w', encoding='utf-8') as outfile: + printer = Printer(outfile) + printer.write(before) + printer.write(START) + with printer.block('struct _Py_global_strings', ';'): + with printer.block('struct', ' literals;'): + for name, literal in STRING_LITERALS.items(): + if literal is None: + outfile.write('\n') + printer.write(f'// {name}') + else: + printer.write(f'STRUCT_FOR_STR({name}, "{literal}")') + outfile.write('\n') + with printer.block('struct', ' identifiers;'): + for name in sorted(IDENTIFIERS): + assert name.isidentifier(), name + printer.write(f'STRUCT_FOR_ID({name})') + printer.write(END) + printer.write(after) + + def generate_runtime_init(): # First get some info from the declarations. nsmallposints = None @@ -106,19 +428,174 @@ def generate_runtime_init(): with printer.block('.bytes_characters =', ','): for i in range(256): printer.write(f'_PyBytes_CHAR_INIT({i}),') + printer.write('') + # Global strings. + with printer.block('.strings =', ','): + with printer.block('.literals =', ','): + for name, literal in STRING_LITERALS.items(): + if literal is None: + printer.write('') + else: + printer.write(f'INIT_STR({name}, "{literal}"),') + with printer.block('.identifiers =', ','): + for name in sorted(IDENTIFIERS): + assert name.isidentifier(), name + printer.write(f'INIT_ID({name}),') printer.write(END) printer.write(after) ####################################### +# checks + +def err(msg): + print(msg, file=sys.stderr) + + +GETTER_RE = re.compile(r''' + ^ + .*? + (?: + (?: + _Py_ID + [(] + ( \w+ ) # <identifier> + [)] + ) + | + (?: + _Py_STR + [(] + ( \w+ ) # <literal> + [)] + ) + ) +''', re.VERBOSE) +TYPESLOTS_RE = re.compile(r''' + ^ + .*? + (?: + (?: + SLOT0 [(] .*?, \s* + ( \w+ ) # <slot0> + [)] + ) + | + (?: + SLOT1 [(] .*?, \s* + ( \w+ ) # <slot1> + , .* [)] + ) + | + (?: + SLOT1BIN [(] .*?, .*?, \s* + ( \w+ ) # <slot1bin> + , \s* + ( \w+ ) # <reverse> + [)] + ) + | + (?: + SLOT1BINFULL [(] .*?, .*?, .*?, \s* + ( \w+ ) # <slot1binfull> + , \s* + ( \w+ ) # <fullreverse> + [)] + ) + | + ( SLOT \d .* [^)] $ ) # <wrapped> + ) +''', re.VERBOSE) + +def check_orphan_strings(): + literals = set(n for n, s in STRING_LITERALS.items() if s) + identifiers = set(IDENTIFIERS) + files = glob.iglob(os.path.join(ROOT, '**', '*.[ch]'), recursive=True) + for i, filename in enumerate(files, start=1): + print('.', end='') + if i % 5 == 0: + print(' ', end='') + if i % 20 == 0: + print() + if i % 100 == 0: + print() + with open(filename) as infile: + wrapped = None + for line in infile: + identifier = literal = reverse = None + + line = line.splitlines()[0] + if wrapped: + line = f'{wrapped.rstrip()} {line}' + wrapped = None + + if os.path.basename(filename) == '_warnings.c': + m = re.match(r'^.* = GET_WARNINGS_ATTR[(][^,]*, (\w+),', line) + if m: + identifier, = m.groups() + elif os.path.basename(filename) == 'typeobject.c': + m = TYPESLOTS_RE.match(line) + if m: + (slot0, + slot1, + slot1bin, reverse, + slot1binfull, fullreverse, + wrapped, + ) = m.groups() + identifier = slot0 or slot1 or slot1bin or slot1binfull + reverse = reverse or fullreverse + + if not identifier and not literal: + m = GETTER_RE.match(line) + if not m: + continue + identifier, literal = m.groups() + + if literal: + if literals and literal in literals: + literals.remove(literal) + if identifier: + if identifiers and identifier in identifiers: + identifiers.remove(identifier) + if reverse: + if identifiers and reverse in identifiers: + identifiers.remove(reverse) + if not literals and not identifiers: + break + else: + continue + break + if i % 20: + print() + if not literals and not identifiers: + return + print('ERROR:', file=sys.stderr) + if literals: + err(' unused global string literals:') + for name in sorted(literals): + err(f' {name}') + if identifiers: + if literals: + print() + err(' unused global identifiers:') + for name in sorted(identifiers): + err(f' {name}') + + +####################################### # the script -def main() -> None: +def main(*, check=False) -> None: + generate_global_strings() generate_runtime_init() + if check: + check_orphan_strings() + if __name__ == '__main__': - argv = sys.argv[1:] - if argv: - sys.exit(f'ERROR: got unexpected args {argv}') - main() + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + main(**vars(args)) |