path: root/Tools/c-analyzer/cpython/__main__.py

import logging
import sys
import textwrap

from c_common.scriptutil import (
    VERBOSITY,
    add_verbosity_cli,
    add_traceback_cli,
    add_commands_cli,
    add_kind_filtering_cli,
    add_files_cli,
    add_progress_cli,
    process_args_by_key,
    configure_logger,
    get_prog,
)
from c_parser.info import KIND
import c_parser.__main__ as c_parser
import c_analyzer.__main__ as c_analyzer
import c_analyzer as _c_analyzer
from c_analyzer.info import UNKNOWN
from . import _analyzer, _builtin_types, _capi, _files, _parser, REPO_ROOT


logger = logging.getLogger(__name__)


CHECK_EXPLANATION = textwrap.dedent('''
    -------------------------

    Non-constant global variables are generally not supported
    in the CPython repo.  We use a tool to analyze the C code
    and report if any unsupported globals are found.  The tool
    may be run manually with:

      ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE]

    Occasionally the tool is unable to parse updated code.
    If this happens then add the file to the "EXCLUDED" list
    in Tools/c-analyzer/cpython/_parser.py and create a new
    issue for fixing the tool (and CC ericsnowcurrently
    on the issue).

    If the tool reports an unsupported global variable and
    it is actually const (and thus supported) then first try
    fixing the declaration appropriately in the code.  If that
    doesn't work then add the variable to the "should be const"
    section of Tools/c-analyzer/cpython/ignored.tsv.

    If the tool otherwise reports an unsupported global variable
    then first try to make it non-global, possibly adding to
    PyInterpreterState (for core code) or module state (for
    extension modules).  In an emergency, you can add the
    variable to Tools/c-analyzer/cpython/globals-to-fix.tsv
    to get CI passing, but doing so should be avoided.  If
    this course it taken, be sure to create an issue for
    eliminating the global (and CC ericsnowcurrently).
''')


def _resolve_filenames(filenames):
    if filenames:
        resolved = (_files.resolve_filename(f) for f in filenames)
    else:
        resolved = _files.iter_filenames()
    return resolved


#######################################
# the formats

def fmt_summary(analysis):
    # XXX Support sorting and grouping.
    supported = []
    unsupported = []
    for item in analysis:
        if item.supported:
            supported.append(item)
        else:
            unsupported.append(item)
    total = 0

    def section(name, groupitems):
        nonlocal total
        items, render = c_analyzer.build_section(name, groupitems,
                                                 relroot=REPO_ROOT)
        yield from render()
        total += len(items)

    yield ''
    yield '===================='
    yield 'supported'
    yield '===================='

    yield from section('types', supported)
    yield from section('variables', supported)

    yield ''
    yield '===================='
    yield 'unsupported'
    yield '===================='

    yield from section('types', unsupported)
    yield from section('variables', unsupported)

    yield ''
    yield f'grand total: {total}'


#######################################
# the checks

CHECKS = dict(c_analyzer.CHECKS, **{
    'globals': _analyzer.check_globals,
})

#######################################
# the commands

FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')


def _cli_parse(parser):
    process_output = c_parser.add_output_cli(parser)
    process_kind = add_kind_filtering_cli(parser)
    process_preprocessor = c_parser.add_preprocessor_cli(
        parser,
        get_preprocessor=_parser.get_preprocessor,
    )
    process_files = add_files_cli(parser, **FILES_KWARGS)
    return [
        process_output,
        process_kind,
        process_preprocessor,
        process_files,
    ]


def cmd_parse(filenames=None, **kwargs):
    filenames = _resolve_filenames(filenames)
    if 'get_file_preprocessor' not in kwargs:
        kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
    c_parser.cmd_parse(
        filenames,
        relroot=REPO_ROOT,
        file_maxsizes=_parser.MAX_SIZES,
        **kwargs
    )


def _cli_check(parser, **kwargs):
    return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)


def cmd_check(filenames=None, **kwargs):
    filenames = _resolve_filenames(filenames)
    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
    try:
        c_analyzer.cmd_check(
            filenames,
            relroot=REPO_ROOT,
            _analyze=_analyzer.analyze,
            _CHECKS=CHECKS,
            file_maxsizes=_parser.MAX_SIZES,
            **kwargs
        )
    except SystemExit as exc:
        num_failed = exc.args[0] if getattr(exc, 'args', None) else None
        if isinstance(num_failed, int):
            if num_failed > 0:
                sys.stderr.flush()
                print(CHECK_EXPLANATION, flush=True)
        raise  # re-raise
    except Exception:
        sys.stderr.flush()
        print(CHECK_EXPLANATION, flush=True)
        raise  # re-raise


def cmd_analyze(filenames=None, **kwargs):
    formats = dict(c_analyzer.FORMATS)
    formats['summary'] = fmt_summary
    filenames = _resolve_filenames(filenames)
    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
    c_analyzer.cmd_analyze(
        filenames,
        relroot=REPO_ROOT,
        _analyze=_analyzer.analyze,
        formats=formats,
        file_maxsizes=_parser.MAX_SIZES,
        **kwargs
    )


def _cli_data(parser):
    filenames = False
    known = True
    return c_analyzer._cli_data(parser, filenames, known)


def cmd_data(datacmd, **kwargs):
    formats = dict(c_analyzer.FORMATS)
    formats['summary'] = fmt_summary
    filenames = (file
                 for file in _resolve_filenames(None)
                 if file not in _parser.EXCLUDED)
    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
    if datacmd == 'show':
        types = _analyzer.read_known()
        results = []
        for decl, info in types.items():
            if info is UNKNOWN:
                if decl.kind in (KIND.STRUCT, KIND.UNION):
                    extra = {'unsupported': ['type unknown'] * len(decl.members)}
                else:
                    extra = {'unsupported': ['type unknown']}
                info = (info, extra)
            results.append((decl, info))
            if decl.shortkey == 'struct _object':
                tempinfo = info
        known = _analyzer.Analysis.from_results(results)
        analyze = None
    elif datacmd == 'dump':
        known = _analyzer.KNOWN_FILE
        def analyze(files, **kwargs):
            decls = []
            for decl in _analyzer.iter_decls(files, **kwargs):
                if not KIND.is_type_decl(decl.kind):
                    continue
                if not decl.filename.endswith('.h'):
                    if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
                        continue
                decls.append(decl)
            results = _c_analyzer.analyze_decls(
                decls,
                known={},
                analyze_resolved=_analyzer.analyze_resolved,
            )
            return _analyzer.Analysis.from_results(results)
    else:  # check
        known = _analyzer.read_known()
        def analyze(files, **kwargs):
            return _analyzer.iter_decls(files, **kwargs)
    extracolumns = None
    c_analyzer.cmd_data(
        datacmd,
        filenames,
        known,
        _analyze=analyze,
        formats=formats,
        extracolumns=extracolumns,
        relroot=REPO_ROOT,
        **kwargs
    )


def _cli_capi(parser):
    parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
    parser.add_argument(f'--public', dest='levels',
                        action='append_const', const='public')
    parser.add_argument(f'--no-public', dest='levels',
                        action='append_const', const='no-public')
    for level in _capi.LEVELS:
        parser.add_argument(f'--{level}', dest='levels',
                            action='append_const', const=level)
    def process_levels(args, *, argv=None):
        levels = []
        for raw in args.levels or ():
            for level in raw.replace(',', ' ').strip().split():
                if level == 'public':
                    levels.append('stable')
                    levels.append('cpython')
                elif level == 'no-public':
                    levels.append('private')
                    levels.append('internal')
                elif level in _capi.LEVELS:
                    levels.append(level)
                else:
                    parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
        args.levels = set(levels)

    parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
    for kind in _capi.KINDS:
        parser.add_argument(f'--{kind}', dest='kinds',
                            action='append_const', const=kind)
    def process_kinds(args, *, argv=None):
        kinds = []
        for raw in args.kinds or ():
            for kind in raw.replace(',', ' ').strip().split():
                if kind in _capi.KINDS:
                    kinds.append(kind)
                else:
                    parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
        args.kinds = set(kinds)

    parser.add_argument('--group-by', dest='groupby',
                        choices=['level', 'kind'])

    parser.add_argument('--format', default='table')
    parser.add_argument('--summary', dest='format',
                        action='store_const', const='summary')
    def process_format(args, *, argv=None):
        orig = args.format
        args.format = _capi.resolve_format(args.format)
        if isinstance(args.format, str):
            if args.format not in _capi._FORMATS:
                parser.error(f'unsupported format {orig!r}')

    parser.add_argument('--show-empty', dest='showempty', action='store_true')
    parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
    parser.set_defaults(showempty=None)

    # XXX Add --sort-by, --sort and --no-sort.

    parser.add_argument('--ignore', dest='ignored', action='append')
    def process_ignored(args, *, argv=None):
        ignored = []
        for raw in args.ignored or ():
            ignored.extend(raw.replace(',', ' ').strip().split())
        args.ignored = ignored or None

    parser.add_argument('filenames', nargs='*', metavar='FILENAME')
    process_progress = add_progress_cli(parser)

    return [
        process_levels,
        process_kinds,
        process_format,
        process_ignored,
        process_progress,
    ]


def cmd_capi(filenames=None, *,
             levels=None,
             kinds=None,
             groupby='kind',
             format='table',
             showempty=None,
             ignored=None,
             track_progress=None,
             verbosity=VERBOSITY,
             **kwargs
             ):
    render = _capi.get_renderer(format)

    filenames = _files.iter_header_files(filenames, levels=levels)
    #filenames = (file for file, _ in main_for_filenames(filenames))
    if track_progress:
        filenames = track_progress(filenames)
    items = _capi.iter_capi(filenames)
    if levels:
        items = (item for item in items if item.level in levels)
    if kinds:
        items = (item for item in items if item.kind in kinds)

    filter = _capi.resolve_filter(ignored)
    if filter:
        items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))

    lines = render(
        items,
        groupby=groupby,
        showempty=showempty,
        verbose=verbosity > VERBOSITY,
    )
    print()
    for line in lines:
        print(line)


def _cli_builtin_types(parser):
    parser.add_argument('--format', dest='fmt', default='table')
#    parser.add_argument('--summary', dest='format',
#                        action='store_const', const='summary')
    def process_format(args, *, argv=None):
        orig = args.fmt
        args.fmt = _builtin_types.resolve_format(args.fmt)
        if isinstance(args.fmt, str):
            if args.fmt not in _builtin_types._FORMATS:
                parser.error(f'unsupported format {orig!r}')

    parser.add_argument('--include-modules', dest='showmodules',
                        action='store_true')
    def process_modules(args, *, argv=None):
        pass

    return [
        process_format,
        process_modules,
    ]


def cmd_builtin_types(fmt, *,
                      showmodules=False,
                      verbosity=VERBOSITY,
                      ):
    render = _builtin_types.get_renderer(fmt)
    types = _builtin_types.iter_builtin_types()
    match = _builtin_types.resolve_matcher(showmodules)
    if match:
        types = (t for t in types if match(t, log=lambda msg: logger.log(1, msg)))

    lines = render(
        types,
#        verbose=verbosity > VERBOSITY,
    )
    print()
    for line in lines:
        print(line)


# We do not define any other cmd_*() handlers here,
# favoring those defined elsewhere.

COMMANDS = {
    'check': (
        'analyze and fail if the CPython source code has any problems',
        [_cli_check],
        cmd_check,
    ),
    'analyze': (
        'report on the state of the CPython source code',
        [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
        cmd_analyze,
    ),
    'parse': (
        'parse the CPython source files',
        [_cli_parse],
        cmd_parse,
    ),
    'data': (
        'check/manage local data (e.g. known types, ignored vars, caches)',
        [_cli_data],
        cmd_data,
    ),
    'capi': (
        'inspect the C-API',
        [_cli_capi],
        cmd_capi,
    ),
    'builtin-types': (
        'show the builtin types',
        [_cli_builtin_types],
        cmd_builtin_types,
    ),
}


#######################################
# the script

def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
    import argparse
    parser = argparse.ArgumentParser(
        prog=prog or get_prog(),
    )

#    if subset == 'check' or subset == ['check']:
#        if checks is not None:
#            commands = dict(COMMANDS)
#            commands['check'] = list(commands['check'])
#            cli = commands['check'][1][0]
#            commands['check'][1][0] = (lambda p: cli(p, checks=checks))
    processors = add_commands_cli(
        parser,
        commands=COMMANDS,
        commonspecs=[
            add_verbosity_cli,
            add_traceback_cli,
        ],
        subset=subset,
    )

    args = parser.parse_args(argv)
    ns = vars(args)

    cmd = ns.pop('cmd')

    verbosity, traceback_cm = process_args_by_key(
        args,
        argv,
        processors[cmd],
        ['verbosity', 'traceback_cm'],
    )
    if cmd != 'parse':
        # "verbosity" is sent to the commands, so we put it back.
        args.verbosity = verbosity

    return cmd, ns, verbosity, traceback_cm


def main(cmd, cmd_kwargs):
    try:
        run_cmd = COMMANDS[cmd][-1]
    except KeyError:
        raise ValueError(f'unsupported cmd {cmd!r}')
    run_cmd(**cmd_kwargs)


if __name__ == '__main__':
    cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
    configure_logger(verbosity)
    with traceback_cm:
        main(cmd, cmd_kwargs)