bpo-36876: Make sure the c-analyzer is checking all the source files.' (gh-31264)

https://bugs.python.org/issue36876

5 files changed, 159 insertions, 22 deletions

diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py
index 49de9d8..741fbad 100644
--- a/Tools/c-analyzer/cpython/_analyzer.py
+++ b/Tools/c-analyzer/cpython/_analyzer.py
@@ -307,7 +307,7 @@ class Analyzed(_info.Analyzed):
         # XXX ???
         #if itemonly:
         #    yield from rendered
-        supported = self._supported
+        supported = self.supported
         if fmt in ('line', 'brief'):
             rendered, = rendered
             parts = [
diff --git a/Tools/c-analyzer/cpython/_files.py b/Tools/c-analyzer/cpython/_files.py
index 3e39788..ee9e46f 100644
--- a/Tools/c-analyzer/cpython/_files.py
+++ b/Tools/c-analyzer/cpython/_files.py
@@ -6,15 +6,17 @@ from . import REPO_ROOT, INCLUDE_DIRS, SOURCE_DIRS
 
 GLOBS = [
     'Include/*.h',
+    # Technically, this is covered by "Include/*.h":
+    #'Include/cpython/*.h',
     'Include/internal/*.h',
     'Modules/**/*.h',
     'Modules/**/*.c',
     'Objects/**/*.h',
     'Objects/**/*.c',
-    'Python/**/*.h',
+    'Parser/**/*.h',
     'Parser/**/*.c',
     'Python/**/*.h',
-    'Parser/**/*.c',
+    'Python/**/*.c',
 ]
 LEVEL_GLOBS = {
     'stable': 'Include/*.h',
diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py
index 6b55c41..e58871c 100644
--- a/Tools/c-analyzer/cpython/_parser.py
+++ b/Tools/c-analyzer/cpython/_parser.py
@@ -81,6 +81,7 @@ Modules/unicodename_db.h
 Objects/unicodetype_db.h
 
 # generated
+Python/deepfreeze/*.c
 Python/frozen_modules/*.h
 Python/opcode_targets.h
 Python/stdlib_module_names.h
@@ -157,6 +158,7 @@ Modules/cmathmodule.c	Py_BUILD_CORE	1
 Modules/faulthandler.c	Py_BUILD_CORE	1
 Modules/gcmodule.c	Py_BUILD_CORE	1
 Modules/getpath.c	Py_BUILD_CORE	1
+Modules/getpath_noop.c	Py_BUILD_CORE	1
 Modules/itertoolsmodule.c	Py_BUILD_CORE	1
 Modules/main.c	Py_BUILD_CORE	1
 Modules/mathmodule.c	Py_BUILD_CORE	1
@@ -169,9 +171,10 @@ Modules/timemodule.c	Py_BUILD_CORE	1
 Modules/unicodedata.c	Py_BUILD_CORE	1
 Objects/stringlib/codecs.h	Py_BUILD_CORE	1
 Objects/stringlib/unicode_format.h	Py_BUILD_CORE	1
+Parser/string_parser.h	Py_BUILD_CORE	1
+Parser/pegen.h	Py_BUILD_CORE	1
 Python/ceval_gil.h	Py_BUILD_CORE	1
 Python/condvar.h	Py_BUILD_CORE	1
-Modules/getpath_noop.c	Py_BUILD_CORE	1
 
 Modules/_json.c	Py_BUILD_CORE_BUILTIN	1
 Modules/_pickle.c	Py_BUILD_CORE_BUILTIN	1
@@ -293,6 +296,9 @@ MAX_SIZES = {
     _abs('Modules/expat/expat.h'): (10_000, 400),
     _abs('Objects/stringlib/unicode_format.h'): (10_000, 400),
     _abs('Objects/typeobject.c'): (20_000, 200),
+    _abs('Python/compile.c'): (20_000, 500),
+    _abs('Python/pylifecycle.c'): (200_000, 5000),
+    _abs('Python/pystate.c'): (200_000, 5000),
 }
 
 
diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv
index 5690481..4ef0a6c 100644
--- a/Tools/c-analyzer/cpython/globals-to-fix.tsv
+++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv
@@ -106,6 +106,28 @@ Objects/unionobject.c	-	_Py_UnionType	-
 Objects/weakrefobject.c	-	_PyWeakref_CallableProxyType	-
 Objects/weakrefobject.c	-	_PyWeakref_ProxyType	-
 Objects/weakrefobject.c	-	_PyWeakref_RefType	-
+Python/bltinmodule.c	-	PyFilter_Type	-
+Python/bltinmodule.c	-	PyMap_Type	-
+Python/bltinmodule.c	-	PyZip_Type	-
+Python/context.c	-	PyContextToken_Type	-
+Python/context.c	-	PyContextVar_Type	-
+Python/context.c	-	PyContext_Type	-
+Python/context.c	-	_PyContextTokenMissing_Type	-
+Python/errors.c	-	UnraisableHookArgsType	-
+Python/hamt.c	-	_PyHamtItems_Type	-
+Python/hamt.c	-	_PyHamtKeys_Type	-
+Python/hamt.c	-	_PyHamtValues_Type	-
+Python/hamt.c	-	_PyHamt_ArrayNode_Type	-
+Python/hamt.c	-	_PyHamt_BitmapNode_Type	-
+Python/hamt.c	-	_PyHamt_CollisionNode_Type	-
+Python/hamt.c	-	_PyHamt_Type	-
+Python/symtable.c	-	PySTEntry_Type	-
+Python/sysmodule.c	-	AsyncGenHooksType	-
+Python/sysmodule.c	-	FlagsType	-
+Python/sysmodule.c	-	Hash_InfoType	-
+Python/sysmodule.c	-	VersionInfoType	-
+Python/thread.c	-	ThreadInfoType	-
+Python/traceback.c	-	PyTraceBack_Type	-
 
 #-----------------------
 # builtin exception types
@@ -306,12 +328,20 @@ Python/compile.c	compiler_visit_annotations	return_str	-
 Python/import.c	PyImport_Import	builtins_str	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c	PyImport_Import	import_str	-
+Python/symtable.c	-	__class__	-
+Python/symtable.c	-	_annotation	-
+Python/symtable.c	-	dictcomp	-
+Python/symtable.c	-	genexpr	-
+Python/symtable.c	-	lambda	-
+Python/symtable.c	-	listcomp	-
+Python/symtable.c	-	setcomp	-
+Python/symtable.c	-	top	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/sysmodule.c	-	whatstrings	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/sysmodule.c	sys_displayhook	newline	-
 
-# _PyArg_Parser
+# _PyArg_Parser (holds tuple of strings)
 Objects/clinic/bytearrayobject.c.h	bytearray___init__	_parser	-
 Objects/clinic/bytearrayobject.c.h	bytearray_decode	_parser	-
 Objects/clinic/bytearrayobject.c.h	bytearray_hex	_parser	-
@@ -367,6 +397,9 @@ Python/clinic/sysmodule.c.h	sys_addaudithook	_parser	-
 Python/clinic/sysmodule.c.h	sys_set_coroutine_origin_tracking_depth	_parser	-
 Python/clinic/traceback.c.h	tb_new	_parser	-
 
+# holds strings
+Objects/typeobject.c	-	slotdefs	-
+
 # other
 Objects/typeobject.c	-	method_cache	-
 Objects/unicodeobject.c	-	_string_module	-
@@ -400,29 +433,73 @@ Python/import.c	-	extensions	-
 # global non-objects to fix in core code
 
 #-----------------------
-# initialized once
+# initialized/set once
 
 # pre-allocated buffer
 Modules/getbuildinfo.c	Py_GetBuildInfo	buildinfo	-
 
 # during init
+Objects/typeobject.c	-	slotdefs_initialized	-
+Objects/unicodeobject.c	-	bloom_linebreak	-
 Parser/parser.c	-	Py_DebugFlag	-
-
-# other
-Objects/codeobject.c	PyCode_NewEmpty	nulltuple	-
+Python/bootstrap_hash.c	-	_Py_HashSecret_Initialized	-
+Python/bootstrap_hash.c	py_getrandom	getrandom_works	-
+Python/pyhash.c	-	_Py_HashSecret	-
+Python/pylifecycle.c	-	runtime_initialized	-
+Python/sysmodule.c	-	_PySys_ImplCacheTag	-
+Python/sysmodule.c	-	_PySys_ImplName	-
+Python/sysmodule.c	-	_preinit_warnoptions	-
+Python/sysmodule.c	-	_preinit_xoptions	-
+Python/thread.c	-	thread_debug	-
+Python/thread.c	-	initialized	-
+
+# lazy
+Objects/floatobject.c	-	double_format	-
+Objects/floatobject.c	-	float_format	-
+Objects/floatobject.c	-	detected_double_format	-
+Objects/floatobject.c	-	detected_float_format	-
 Objects/longobject.c	PyLong_FromString	log_base_BASE	-
 Objects/longobject.c	PyLong_FromString	convwidth_base	-
 Objects/longobject.c	PyLong_FromString	convmultmax_base	-
-Objects/typeobject.c	-	slotdefs	-
-Objects/typeobject.c	-	slotdefs_initialized	-
-Objects/unicodeobject.c	-	bloom_linebreak	-
 Objects/unicodeobject.c	-	ucnhash_capi	-
 Parser/action_helpers.c	_PyPegen_dummy_name	cache	-
-Parser/pegen.c	_PyPegen_dummy_name	cache	-
+Python/dtoa.c	-	p5s	-
+Python/fileutils.c	-	force_ascii	-
+Python/fileutils.c	set_inheritable	ioctl_works	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c	-	import_lock	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c	import_find_and_load	header	-
+Python/specialize.c	-	_list_append	-
+
+#-----------------------
+# unlikely to change after init (or main thread)
+
+# through C-API
+Python/frozen.c	-	PyImport_FrozenModules	-
+Python/frozen.c	-	_PyImport_FrozenAliases	-
+Python/frozen.c	-	_PyImport_FrozenBootstrap	-
+Python/frozen.c	-	_PyImport_FrozenStdlib	-
+Python/frozen.c	-	_PyImport_FrozenTest	-
+Python/import.c	-	inittab_copy	-
+Python/import.c	-	PyImport_Inittab	-
+Python/preconfig.c	-	Py_FileSystemDefaultEncoding	-
+Python/preconfig.c	-	Py_HasFileSystemDefaultEncoding	-
+Python/preconfig.c	-	Py_FileSystemDefaultEncodeErrors	-
+Python/preconfig.c	-	_Py_HasFileSystemDefaultEncodeErrors	-
+
+# REPL
+Parser/myreadline.c	-	_PyOS_ReadlineLock	-
+Parser/myreadline.c	-	_PyOS_ReadlineTState	-
+Parser/myreadline.c	-	PyOS_InputHook	-
+Parser/myreadline.c	-	PyOS_ReadlineFunctionPointer	-
+
+# handling C argv
+Python/getopt.c	-	_PyOS_optarg	-
+Python/getopt.c	-	_PyOS_opterr	-
+Python/getopt.c	-	_PyOS_optind	-
+Python/getopt.c	-	opt_ptr	-
+Python/pathconfig.c	-	_Py_path_config	-
 
 #-----------------------
 # state
@@ -444,20 +521,22 @@ Objects/obmalloc.c	-	unused_arena_objects	-
 Objects/obmalloc.c	-	usable_arenas	-
 Objects/obmalloc.c	new_arena	debug_stats	-
 
-# REPL
-Parser/myreadline.c	-	_PyOS_ReadlineLock	-
-Parser/myreadline.c	-	_PyOS_ReadlineTState	-
-Parser/myreadline.c	-	PyOS_InputHook	-
-Parser/myreadline.c	-	PyOS_ReadlineFunctionPointer	-
+# pre-allocated memory
+Python/dtoa.c	-	freelist	-
+Python/dtoa.c	-	private_mem	-
+
+# local buffer
+Python/getversion.c	Py_GetVersion	version	-
+Python/suggestions.c	levenshtein_distance	buffer	-
+
+# linked list
+Python/dtoa.c	-	pmem_next	-
+Python/getargs.c	-	static_arg_parsers	-
 
 # other
 Objects/dictobject.c	-	_pydict_global_version	-
 Objects/dictobject.c	-	next_dict_keys_version	-
 Objects/dictobject.c	-	pydict_global_version	-
-Objects/floatobject.c	-	double_format	-
-Objects/floatobject.c	-	float_format	-
-Objects/floatobject.c	-	detected_double_format	-
-Objects/floatobject.c	-	detected_float_format	-
 Objects/funcobject.c	-	next_func_version	-
 Objects/moduleobject.c	-	max_module_number	-
 Objects/object.c	-	_Py_RefTotal	-
@@ -480,10 +559,14 @@ Python/import.c	-	import_lock_thread	-
 Python/import.c	import_find_and_load	accumulated	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/import.c	import_find_and_load	import_level	-
+Python/modsupport.c	-	_Py_PackageContext	-
+Python/pyfpe.c	-	PyFPE_counter	-
+Python/pylifecycle.c	_Py_FatalErrorFormat	reentrant	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/pylifecycle.c	-	_Py_UnhandledKeyboardInterrupt	-
 # XXX This should have been found by the analyzer but wasn't:
 Python/pylifecycle.c	fatal_error	reentrant	-
+Python/specialize.c	-	_Py_QuickenedCount	-
 
 
 ##################################
diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv
index 9b8eff3..c945542 100644
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -2,6 +2,12 @@ filename	funcname	name	reason
 #???	-	somevar	???
 
 ##################################
+# ignored by design
+
+Python/pylifecycle.c	-	_PyRuntime	-
+
+
+##################################
 # forward/extern references
 # XXX The analyzer should have ignored these.
 
@@ -384,6 +390,13 @@ Objects/interpreteridobject.c	interpid_new	kwlist	-
 Objects/weakrefobject.c	weakref_call	kwlist	-
 Objects/exceptions.c	NameError_init	kwlist	-
 Objects/exceptions.c	AttributeError_init	kwlist	-
+Python/_warnings.c	warnings_warn_explicit	kwd_list	-
+Python/bltinmodule.c	builtin___import__	kwlist	-
+Python/bltinmodule.c	min_max	kwlist	-
+Python/bltinmodule.c	zip_new	kwlist	-
+Python/context.c	contextvar_tp_new	kwlist	-
+Python/sysmodule.c	sys_getsizeof	kwlist	-
+Python/sysmodule.c	sys_set_asyncgen_hooks	keywords	-
 
 #-----------------------
 # PyModuleDef
@@ -397,6 +410,13 @@ Modules/_threadmodule.c	-	thread_module	-
 Modules/_typingmodule.c	-	typingmodule	-
 Modules/signalmodule.c	-	signal_module	-
 Modules/xxlimited_35.c	-	xxmodule	-
+Python/Python-ast.c	-	_astmodule	-
+Python/Python-tokenize.c	-	_tokenizemodule	-
+Python/_warnings.c	-	warnings_module	-
+Python/bltinmodule.c	-	builtinsmodule	-
+Python/import.c	-	imp_module	-
+Python/marshal.c	-	marshalmodule	-
+Python/sysmodule.c	-	sysmodule	-
 
 #-----------------------
 # PyModuleDef_Slot
@@ -478,6 +498,8 @@ Modules/xxmodule.c	-	xx_slots	-
 Modules/xxsubtype.c	-	xxsubtype_slots	-
 Modules/zlibmodule.c	-	zlib_slots	-
 Python/Python-ast.c	-	astmodule_slots	-
+Python/Python-tokenize.c	-	tokenizemodule_slots	-
+Python/_warnings.c	-	warnings_slots	-
 Python/marshal.c	-	marshalmodule_slots	-
 
 #-----------------------
@@ -798,6 +820,7 @@ Objects/unionobject.c	-	union_methods	-
 Objects/weakrefobject.c	-	proxy_methods	-
 Objects/weakrefobject.c	-	weakref_methods	-
 Python/Python-ast.c	-	ast_type_methods	-
+Python/Python-tokenize.c	-	tokenize_methods	-
 Python/_warnings.c	-	warnings_functions	-
 Python/bltinmodule.c	-	builtin_methods	-
 Python/bltinmodule.c	-	filter_methods	-
@@ -807,6 +830,7 @@ Python/context.c	-	PyContextTokenType_methods	-
 Python/context.c	-	PyContextVar_methods	-
 Python/context.c	-	PyContext_methods	-
 Python/hamt.c	-	PyHamt_methods	-
+Python/import.c	-	imp_slots	-
 Python/import.c	-	imp_methods	-
 Python/marshal.c	-	marshal_methods	-
 Python/sysmodule.c	-	sys_methods	-
@@ -1218,6 +1242,7 @@ Modules/xxlimited_35.c	-	Xxo_Type_slots	-
 Modules/zlibmodule.c	-	Comptype_slots	-
 Modules/zlibmodule.c	-	Decomptype_slots	-
 Python/Python-ast.c	-	AST_type_slots	-
+Python/Python-tokenize.c	-	tokenizeriter_slots	-
 
 # PyType_Spec
 Modules/_abc.c	-	_abc_data_type_spec	-
@@ -1317,6 +1342,7 @@ Modules/xxlimited_35.c	-	Xxo_Type_spec	-
 Modules/zlibmodule.c	-	Comptype_spec	-
 Modules/zlibmodule.c	-	Decomptype_spec	-
 Python/Python-ast.c	-	AST_type_spec	-
+Python/Python-tokenize.c	-	tokenizeriter_spec	-
 
 #-----------------------
 # for structseq
@@ -1470,4 +1496,24 @@ Objects/unicodeobject.c	unicode_translate_call_errorhandler	argparse	-
 Parser/parser.c	-	reserved_keywords	-
 Parser/parser.c	-	soft_keywords	-
 Parser/tokenizer.c	-	type_comment_prefix	-
+Python/ast_opt.c	fold_unaryop	ops	-
+Python/codecs.c	-	Py_hexdigits	-
+Python/codecs.c	-	ucnhash_capi	-
+Python/codecs.c	_PyCodecRegistry_Init	methods	-
+Python/dynload_shlib.c	-	_PyImport_DynLoadFiletab	-
+Python/dynload_stub.c	-	_PyImport_DynLoadFiletab	-
+Python/frozen.c	-	aliases	-
+Python/frozen.c	-	bootstrap_modules	-
+Python/frozen.c	-	stdlib_modules	-
+Python/frozen.c	-	test_modules	-
+Python/getopt.c	-	longopts	-
+Python/import.c	-	_PyImport_Inittab	-
+Python/import.c	-	_PySys_ImplCacheTag	-
 Python/opcode_targets.h	-	opcode_targets	-
+Python/pyhash.c	-	PyHash_Func	-
+Python/pylifecycle.c	-	_C_LOCALE_WARNING	-
+Python/pylifecycle.c	-	_PyOS_mystrnicmp_hack	-
+Python/pylifecycle.c	-	_TARGET_LOCALES	-
+Python/specialize.c	-	adaptive_opcodes	-
+Python/specialize.c	-	cache_requirements	-
+Python/specialize.c	-	compare_masks	-