path: root/Tools/c-analyzer/c_globals/supported.py

import os.path
import re

from c_analyzer_common import DATA_DIR
from c_analyzer_common.info import ID
from c_analyzer_common.util import read_tsv, write_tsv


IGNORED_FILE = os.path.join(DATA_DIR, 'ignored.tsv')

IGNORED_COLUMNS = ('filename', 'funcname', 'name', 'kind', 'reason')
IGNORED_HEADER = '\t'.join(IGNORED_COLUMNS)

# XXX Move these to ignored.tsv.
IGNORED = {
        # global
        'PyImport_FrozenModules': 'process-global',
        'M___hello__': 'process-global',
        'inittab_copy': 'process-global',
        'PyHash_Func': 'process-global',
        '_Py_HashSecret_Initialized': 'process-global',
        '_TARGET_LOCALES': 'process-global',

        # startup (only changed before/during)
        '_PyRuntime': 'runtime startup',
        'runtime_initialized': 'runtime startup',
        'static_arg_parsers': 'runtime startup',
        'orig_argv': 'runtime startup',
        'opt_ptr': 'runtime startup',
        '_preinit_warnoptions': 'runtime startup',
        '_Py_StandardStreamEncoding': 'runtime startup',
        'Py_FileSystemDefaultEncoding': 'runtime startup',
        '_Py_StandardStreamErrors': 'runtime startup',
        'Py_FileSystemDefaultEncodeErrors': 'runtime startup',
        'Py_BytesWarningFlag': 'runtime startup',
        'Py_DebugFlag': 'runtime startup',
        'Py_DontWriteBytecodeFlag': 'runtime startup',
        'Py_FrozenFlag': 'runtime startup',
        'Py_HashRandomizationFlag': 'runtime startup',
        'Py_IgnoreEnvironmentFlag': 'runtime startup',
        'Py_InspectFlag': 'runtime startup',
        'Py_InteractiveFlag': 'runtime startup',
        'Py_IsolatedFlag': 'runtime startup',
        'Py_NoSiteFlag': 'runtime startup',
        'Py_NoUserSiteDirectory': 'runtime startup',
        'Py_OptimizeFlag': 'runtime startup',
        'Py_QuietFlag': 'runtime startup',
        'Py_UTF8Mode': 'runtime startup',
        'Py_UnbufferedStdioFlag': 'runtime startup',
        'Py_VerboseFlag': 'runtime startup',
        '_Py_path_config': 'runtime startup',
        '_PyOS_optarg': 'runtime startup',
        '_PyOS_opterr': 'runtime startup',
        '_PyOS_optind': 'runtime startup',
        '_Py_HashSecret': 'runtime startup',

        # REPL
        '_PyOS_ReadlineLock': 'repl',
        '_PyOS_ReadlineTState': 'repl',

        # effectively const
        'tracemalloc_empty_traceback': 'const',
        '_empty_bitmap_node': 'const',
        'posix_constants_pathconf': 'const',
        'posix_constants_confstr': 'const',
        'posix_constants_sysconf': 'const',
        '_PySys_ImplCacheTag': 'const',
        '_PySys_ImplName': 'const',
        'PyImport_Inittab': 'const',
        '_PyImport_DynLoadFiletab': 'const',
        '_PyParser_Grammar': 'const',
        'Py_hexdigits': 'const',
        '_PyImport_Inittab': 'const',
        '_PyByteArray_empty_string': 'const',
        '_PyLong_DigitValue': 'const',
        '_Py_SwappedOp': 'const',
        'PyStructSequence_UnnamedField': 'const',

        # signals are main-thread only
        'faulthandler_handlers': 'signals are main-thread only',
        'user_signals': 'signals are main-thread only',
        'wakeup': 'signals are main-thread only',

        # hacks
        '_PySet_Dummy': 'only used as a placeholder',
        }

BENIGN = 'races here are benign and unlikely'


def is_supported(variable, ignored=None, known=None, *,
                 _ignored=(lambda *a, **k: _is_ignored(*a, **k)),
                 _vartype_okay=(lambda *a, **k: _is_vartype_okay(*a, **k)),
                 ):
    """Return True if the given global variable is okay in CPython."""
    if _ignored(variable,
                ignored and ignored.get('variables')):
        return True
    elif _vartype_okay(variable.vartype,
                       ignored.get('types')):
        return True
    else:
        return False


def _is_ignored(variable, ignoredvars=None, *,
                _IGNORED=IGNORED,
                ):
    """Return the reason if the variable is a supported global.

    Return None if the variable is not a supported global.
    """
    if ignoredvars and (reason := ignoredvars.get(variable.id)):
        return reason

    if variable.funcname is None:
        if reason := _IGNORED.get(variable.name):
            return reason

    # compiler
    if variable.filename == 'Python/graminit.c':
        if variable.vartype.startswith('static state '):
            return 'compiler'
    if variable.filename == 'Python/symtable.c':
        if variable.vartype.startswith('static identifier '):
            return 'compiler'
    if variable.filename == 'Python/Python-ast.c':
        # These should be const.
        if variable.name.endswith('_field'):
            return 'compiler'
        if variable.name.endswith('_attribute'):
            return 'compiler'

    # other
    if variable.filename == 'Python/dtoa.c':
        # guarded by lock?
        if variable.name in ('p5s', 'freelist'):
            return 'dtoa is thread-safe?'
        if variable.name in ('private_mem', 'pmem_next'):
            return 'dtoa is thread-safe?'
    if variable.filename == 'Python/thread.c':
        # Threads do not become an issue until after these have been set
        # and these never get changed after that.
        if variable.name in ('initialized', 'thread_debug'):
            return 'thread-safe'
    if variable.filename == 'Python/getversion.c':
        if variable.name == 'version':
            # Races are benign here, as well as unlikely.
            return BENIGN
    if variable.filename == 'Python/fileutils.c':
        if variable.name == 'force_ascii':
            return BENIGN
        if variable.name == 'ioctl_works':
            return BENIGN
        if variable.name == '_Py_open_cloexec_works':
            return BENIGN
    if variable.filename == 'Python/codecs.c':
        if variable.name == 'ucnhash_CAPI':
            return BENIGN
    if variable.filename == 'Python/bootstrap_hash.c':
        if variable.name == 'getrandom_works':
            return BENIGN
    if variable.filename == 'Objects/unicodeobject.c':
        if variable.name == 'ucnhash_CAPI':
            return BENIGN
        if variable.name == 'bloom_linebreak':
            # *mostly* benign
            return BENIGN
    if variable.filename == 'Modules/getbuildinfo.c':
        if variable.name == 'buildinfo':
            # The static is used for pre-allocation.
            return BENIGN
    if variable.filename == 'Modules/posixmodule.c':
        if variable.name == 'ticks_per_second':
            return BENIGN
        if variable.name == 'dup3_works':
            return BENIGN
    if variable.filename == 'Modules/timemodule.c':
        if variable.name == 'ticks_per_second':
            return BENIGN
    if variable.filename == 'Objects/longobject.c':
        if variable.name == 'log_base_BASE':
            return BENIGN
        if variable.name == 'convwidth_base':
            return BENIGN
        if variable.name == 'convmultmax_base':
            return BENIGN

    return None


def _is_vartype_okay(vartype, ignoredtypes=None):
    if _is_object(vartype):
        return None

    if vartype.startswith('static const '):
        return 'const'
    if vartype.startswith('const '):
        return 'const'

    # components for TypeObject definitions
    for name in ('PyMethodDef', 'PyGetSetDef', 'PyMemberDef'):
        if name in vartype:
            return 'const'
    for name in ('PyNumberMethods', 'PySequenceMethods', 'PyMappingMethods',
                 'PyBufferProcs', 'PyAsyncMethods'):
        if name in vartype:
            return 'const'
    for name in ('slotdef', 'newfunc'):
        if name in vartype:
            return 'const'

    # structseq
    for name in ('PyStructSequence_Desc', 'PyStructSequence_Field'):
        if name in vartype:
            return 'const'

    # other definiitions
    if 'PyModuleDef' in vartype:
        return 'const'

    # thread-safe
    if '_Py_atomic_int' in vartype:
        return 'thread-safe'
    if 'pthread_condattr_t' in vartype:
        return 'thread-safe'

    # startup
    if '_Py_PreInitEntry' in vartype:
        return 'startup'

    # global
#    if 'PyMemAllocatorEx' in vartype:
#        return True

    # others
#    if 'PyThread_type_lock' in vartype:
#        return True

    # XXX ???
    # _Py_tss_t
    # _Py_hashtable_t
    # stack_t
    # _PyUnicode_Name_CAPI

    # functions
    if '(' in vartype and '[' not in vartype:
        return 'function pointer'

    # XXX finish!
    # * allow const values?
    #raise NotImplementedError
    return None


PYOBJECT_RE = re.compile(r'''
        ^
        (
            # must start with "static "
            static \s+
            (
                identifier
            )
            \b
        ) |
        (
            # may start with "static "
            ( static \s+ )?
            (
                .*
                (
                    PyObject |
                    PyTypeObject |
                    _? Py \w+ Object |
                    _PyArg_Parser |
                    _Py_Identifier |
                    traceback_t |
                    PyAsyncGenASend |
                    _PyAsyncGenWrappedValue |
                    PyContext |
                    method_cache_entry
                )
                \b
            ) |
            (
                (
                    _Py_IDENTIFIER |
                    _Py_static_string
                )
                [(]
            )
        )
        ''', re.VERBOSE)


def _is_object(vartype):
    if 'PyDictKeysObject' in vartype:
        return False
    if PYOBJECT_RE.match(vartype):
        return True
    if vartype.endswith((' _Py_FalseStruct', ' _Py_TrueStruct')):
        return True

    # XXX Add more?

    #for part in vartype.split():
    #    # XXX const is automatic True?
    #    if part == 'PyObject' or part.startswith('PyObject['):
    #        return True
    return False


def ignored_from_file(infile, *,
                      _read_tsv=read_tsv,
                      ):
    """Yield a Variable for each ignored var in the file."""
    ignored = {
        'variables': {},
        #'types': {},
        #'constants': {},
        #'macros': {},
        }
    for row in _read_tsv(infile, IGNORED_HEADER):
        filename, funcname, name, kind, reason = row
        if not funcname or funcname == '-':
            funcname = None
        id = ID(filename, funcname, name)
        if kind == 'variable':
            values = ignored['variables']
        else:
            raise ValueError(f'unsupported kind in row {row}')
        values[id] = reason
    return ignored


##################################
# generate

def _get_row(varid, reason):
    return (
            varid.filename,
            varid.funcname or '-',
            varid.name,
            'variable',
            str(reason),
            )


def _get_rows(variables, ignored=None, *,
              _as_row=_get_row,
              _is_ignored=_is_ignored,
              _vartype_okay=_is_vartype_okay,
              ):
    count = 0
    for variable in variables:
        reason = _is_ignored(variable,
                             ignored and ignored.get('variables'),
                             )
        if not reason:
            reason = _vartype_okay(variable.vartype,
                                   ignored and ignored.get('types'))
        if not reason:
            continue

        print(' ', variable, repr(reason))
        yield _as_row(variable.id, reason)
        count += 1
    print(f'total: {count}')


def _generate_ignored_file(variables, filename=None, *,
                           _generate_rows=_get_rows,
                           _write_tsv=write_tsv,
                           ):
    if not filename:
        filename = IGNORED_FILE + '.new'
    rows = _generate_rows(variables)
    _write_tsv(filename, IGNORED_HEADER, rows)


if __name__ == '__main__':
    from c_analyzer_common import SOURCE_DIRS
    from c_analyzer_common.known import (
        from_file as known_from_file,
        DATA_FILE as KNOWN_FILE,
        )
    from . import find
    known = known_from_file(KNOWN_FILE)
    knownvars = (known or {}).get('variables')
    variables = find.globals_from_binary(knownvars=knownvars,
                                         dirnames=SOURCE_DIRS)

    _generate_ignored_file(variables)