bpo-36876: Fix the C analyzer tool. (GH-22841)

The original tool wasn't working right and it was simpler to create a new one, partially re-using some of the old code. At this point the tool runs properly on the master. (Try: ./python Tools/c-analyzer/c-analyzer.py analyze.)  It take ~40 seconds on my machine to analyze the full CPython code base.

Note that we'll need to iron out some OS-specific stuff (e.g. preprocessor). We're okay though since this tool isn't used yet in our workflow. We will also need to verify the analysis results in detail before activating the check in CI, though I'm pretty sure it's close.

https://bugs.python.org/issue36876

1 files changed, 103 insertions, 0 deletions

diff --git a/Tools/c-analyzer/c_analyzer/__init__.py b/Tools/c-analyzer/c_analyzer/__init__.py
index e69de29..4a01cd3 100644
--- a/Tools/c-analyzer/c_analyzer/__init__.py
+++ b/Tools/c-analyzer/c_analyzer/__init__.py
@@ -0,0 +1,103 @@
+from c_parser import (
+    parse_files as _parse_files,
+)
+from c_parser.info import (
+    KIND,
+    TypeDeclaration,
+    filter_by_kind,
+    collate_by_kind_group,
+    resolve_parsed,
+)
+from . import (
+    analyze as _analyze,
+    datafiles as _datafiles,
+)
+from .info import Analysis
+
+
+def analyze(filenmes, **kwargs):
+    results = iter_analyis_results(filenames, **kwargs)
+    return Analysis.from_results(results)
+
+
+def iter_analysis_results(filenmes, *,
+                          known=None,
+                          **kwargs
+                          ):
+    decls = iter_decls(filenames, **kwargs)
+    yield from analyze_decls(decls, known)
+
+
+def iter_decls(filenames, *,
+               kinds=None,
+               parse_files=_parse_files,
+               **kwargs
+               ):
+    kinds = KIND.DECLS if kinds is None else (KIND.DECLS & set(kinds))
+    parse_files = parse_files or _parse_files
+
+    parsed = parse_files(filenames, **kwargs)
+    parsed = filter_by_kind(parsed, kinds)
+    for item in parsed:
+        yield resolve_parsed(item)
+
+
+def analyze_decls(decls, known, *,
+                  analyze_resolved=None,
+                  handle_unresolved=True,
+                  relroot=None,
+                  ):
+    knowntypes, knowntypespecs = _datafiles.get_known(
+        known,
+        handle_unresolved=handle_unresolved,
+        analyze_resolved=analyze_resolved,
+        relroot=relroot,
+    )
+
+    decls = list(decls)
+    collated = collate_by_kind_group(decls)
+
+    types = {decl: None for decl in collated['type']}
+    typespecs = _analyze.get_typespecs(types)
+
+    def analyze_decl(decl):
+        return _analyze.analyze_decl(
+            decl,
+            typespecs,
+            knowntypespecs,
+            types,
+            knowntypes,
+            analyze_resolved=analyze_resolved,
+        )
+    _analyze.analyze_type_decls(types, analyze_decl, handle_unresolved)
+    for decl in decls:
+        if decl in types:
+            resolved = types[decl]
+        else:
+            resolved = analyze_decl(decl)
+            if resolved and handle_unresolved:
+                typedeps, _ = resolved
+                if not isinstance(typedeps, TypeDeclaration):
+                    if not typedeps or None in typedeps:
+                        raise NotImplementedError((decl, resolved))
+
+        yield decl, resolved
+
+
+#######################################
+# checks
+
+def check_all(analysis, checks, *, failfast=False):
+    for check in checks or ():
+        for data, failure in check(analysis):
+            if failure is None:
+                continue
+
+            yield data, failure
+            if failfast:
+                yield None, None
+                break
+        else:
+            continue
+        # We failed fast.
+        break