bpo-36876: [c-analyzer tool] Tighten up the results and output. (GH-23431)

We also update the "ignored" file with a temporary list of all known globals.

3 files changed, 66 insertions, 23 deletions

diff --git a/Tools/c-analyzer/c_parser/__main__.py b/Tools/c-analyzer/c_parser/__main__.py
index 1752a70..539cec5 100644
--- a/Tools/c-analyzer/c_parser/__main__.py
+++ b/Tools/c-analyzer/c_parser/__main__.py
@@ -2,6 +2,7 @@ import logging
 import os.path
 import sys
 
+from c_common import fsutil
 from c_common.scriptutil import (
     CLIArgSpec as Arg,
     add_verbosity_cli,
@@ -64,8 +65,9 @@ def fmt_raw(filename, item, *, showfwd=None):
 
 
 def fmt_summary(filename, item, *, showfwd=None):
-    if item.filename and item.filename != os.path.join('.', filename):
+    if item.filename != filename:
         yield f'> {item.filename}'
+
     if showfwd is None:
         LINE = ' {lno:>5} {kind:10} {funcname:40} {fwd:1} {name:40} {data}'
     else:
@@ -172,6 +174,7 @@ def cmd_parse(filenames, *,
               fmt='summary',
               showfwd=None,
               iter_filenames=None,
+              relroot=None,
               **kwargs
               ):
     if 'get_file_preprocessor' not in kwargs:
@@ -180,9 +183,10 @@ def cmd_parse(filenames, *,
         do_fmt = FORMATS[fmt]
     except KeyError:
         raise ValueError(f'unsupported fmt {fmt!r}')
-    for filename in main_for_filenames(filenames, iter_filenames):
+    for filename, relfile in main_for_filenames(filenames, iter_filenames, relroot):
         for item in _iter_parsed(filename, **kwargs):
-            for line in do_fmt(filename, item, showfwd=showfwd):
+            item = item.fix_filename(relroot, fixroot=False, normalize=False)
+            for line in do_fmt(relfile, item, showfwd=showfwd):
                 print(line)
 
 
diff --git a/Tools/c-analyzer/c_parser/datafiles.py b/Tools/c-analyzer/c_parser/datafiles.py
index cdd69b1..f053056 100644
--- a/Tools/c-analyzer/c_parser/datafiles.py
+++ b/Tools/c-analyzer/c_parser/datafiles.py
@@ -1,5 +1,6 @@
 import os.path
 
+from c_common import fsutil
 import c_common.tables as _tables
 import c_parser.info as _info
 
@@ -81,21 +82,27 @@ def _get_format_handlers(group, fmt):
 
 # tsv
 
-def iter_decls_tsv(infile, extracolumns=None, relroot=None):
-    for info, extra in _iter_decls_tsv(infile, extracolumns, relroot):
+def iter_decls_tsv(infile, extracolumns=None, relroot=fsutil.USE_CWD):
+    if relroot and relroot is not fsutil.USE_CWD:
+        relroot = os.path.abspath(relroot)
+    for info, extra in _iter_decls_tsv(infile, extracolumns):
         decl = _info.Declaration.from_row(info)
+        decl = decl.fix_filename(relroot, formatted=False, fixroot=False)
         yield decl, extra
 
 
 def write_decls_tsv(decls, outfile, extracolumns=None, *,
-                    relroot=None,
+                    relroot=fsutil.USE_CWD,
                     **kwargs
                     ):
+    if relroot and relroot is not fsutil.USE_CWD:
+        relroot = os.path.abspath(relroot)
+    decls = (d.fix_filename(relroot, fixroot=False) for d in decls)
     # XXX Move the row rendering here.
-    _write_decls_tsv(decls, outfile, extracolumns, relroot, kwargs)
+    _write_decls_tsv(decls, outfile, extracolumns, kwargs)
 
 
-def _iter_decls_tsv(infile, extracolumns=None, relroot=None):
+def _iter_decls_tsv(infile, extracolumns=None):
     columns = _get_columns('decls', extracolumns)
     for row in _tables.read_table(infile, columns, sep='\t'):
         if extracolumns:
@@ -104,15 +111,13 @@ def _iter_decls_tsv(infile, extracolumns=None, relroot=None):
         else:
             declinfo = row
             extra = None
-        if relroot:
-            # XXX Use something like tables.fix_row() here.
-            declinfo = [None if v == '-' else v
-                        for v in declinfo]
-            declinfo[0] = os.path.join(relroot, declinfo[0])
+        # XXX Use something like tables.fix_row() here.
+        declinfo = [None if v == '-' else v
+                    for v in declinfo]
         yield declinfo, extra
 
 
-def _write_decls_tsv(decls, outfile, extracolumns, relroot,kwargs):
+def _write_decls_tsv(decls, outfile, extracolumns, kwargs):
     columns = _get_columns('decls', extracolumns)
     if extracolumns:
         def render_decl(decl):
@@ -121,7 +126,7 @@ def _write_decls_tsv(decls, outfile, extracolumns, relroot,kwargs):
             else:
                 extra = ()
             extra += ('???',) * (len(extraColumns) - len(extra))
-            *row, declaration = _render_known_row(decl, relroot)
+            *row, declaration = _render_known_row(decl)
             row += extra + (declaration,)
             return row
     else:
@@ -129,13 +134,13 @@ def _write_decls_tsv(decls, outfile, extracolumns, relroot,kwargs):
     _tables.write_table(
         outfile,
         header='\t'.join(columns),
-        rows=(render_decl(d, relroot) for d in decls),
+        rows=(render_decl(d) for d in decls),
         sep='\t',
         **kwargs
     )
 
 
-def _render_known_decl(decl, relroot, *,
+def _render_known_decl(decl, *,
                        # These match BASE_COLUMNS + END_COLUMNS[group].
                        _columns = 'filename parent name kind data'.split(),
                        ):
@@ -143,8 +148,6 @@ def _render_known_decl(decl, relroot, *,
         # e.g. Analyzed
         decl = decl.decl
     rowdata = decl.render_rowdata(_columns)
-    if relroot:
-        rowdata['filename'] = os.path.relpath(rowdata['filename'], relroot)
     return [rowdata[c] or '-' for c in _columns]
     # XXX
     #return _tables.fix_row(rowdata[c] for c in columns)
diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py
index 798a45d..98ff511 100644
--- a/Tools/c-analyzer/c_parser/info.py
+++ b/Tools/c-analyzer/c_parser/info.py
@@ -3,6 +3,7 @@ import enum
 import os.path
 import re
 
+from c_common import fsutil
 from c_common.clsutil import classonly
 import c_common.misc as _misc
 import c_common.strutil as _strutil
@@ -148,6 +149,16 @@ def get_kind_group(item):
 #############################
 # low-level
 
+def _fix_filename(filename, relroot, *,
+                  formatted=True,
+                  **kwargs):
+    if formatted:
+        fix = fsutil.format_filename
+    else:
+        fix = fsutil.fix_filename
+    return fix(filename, relroot=relroot, **kwargs)
+
+
 class FileInfo(namedtuple('FileInfo', 'filename lno')):
     @classmethod
     def from_raw(cls, raw):
@@ -165,8 +176,10 @@ class FileInfo(namedtuple('FileInfo', 'filename lno')):
     def __str__(self):
         return self.filename
 
-    def fix_filename(self, relroot):
-        filename = os.path.relpath(self.filename, relroot)
+    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
+        filename = _fix_filename(self.filename, relroot, **kwargs)
+        if filename == self.filename:
+            return self
         return self._replace(filename=filename)
 
 
@@ -194,6 +207,16 @@ class DeclID(namedtuple('DeclID', 'filename funcname name')):
         row = _tables.fix_row(row, **markers)
         return cls(*row)
 
+    # We have to provde _make() becaose we implemented __new__().
+
+    @classmethod
+    def _make(cls, iterable):
+        try:
+            return cls(*iterable)
+        except Exception:
+            super()._make(iterable)
+            raise  # re-raise
+
     def __new__(cls, filename, funcname, name):
         self = super().__new__(
             cls,
@@ -221,6 +244,12 @@ class DeclID(namedtuple('DeclID', 'filename funcname name')):
             return NotImplemented
         return self._compare > other
 
+    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
+        filename = _fix_filename(self.filename, relroot, **kwargs)
+        if filename == self.filename:
+            return self
+        return self._replace(filename=filename)
+
 
 class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')):
 
@@ -290,6 +319,12 @@ class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')):
         else:
             return self.parent.name
 
+    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
+        fixed = self.file.fix_filename(relroot, **kwargs)
+        if fixed == self.file:
+            return self
+        return self._replace(file=fixed)
+
     def as_row(self, columns=None):
         if not columns:
             columns = self._fields
@@ -591,9 +626,10 @@ class HighlevelParsedItem:
             )
             return self._parsed
 
-    def fix_filename(self, relroot):
+    def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
         if self.file:
-            self.file = self.file.fix_filename(relroot)
+            self.file = self.file.fix_filename(relroot, **kwargs)
+        return self
 
     def as_rowdata(self, columns=None):
         columns, datacolumns, colnames = self._parse_columns(columns)