diff options
-rw-r--r-- | .github/workflows/reusable-macos.yml | 19 | ||||
-rw-r--r-- | .github/workflows/reusable-ubuntu.yml | 19 | ||||
-rw-r--r-- | Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst | 1 | ||||
-rw-r--r-- | Tools/build/.warningignore_macos | 227 | ||||
-rw-r--r-- | Tools/build/.warningignore_ubuntu | 260 | ||||
-rw-r--r-- | Tools/build/check_warnings.py | 168 | ||||
-rwxr-xr-x | configure | 215 | ||||
-rw-r--r-- | configure.ac | 15 |
8 files changed, 809 insertions, 115 deletions
diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d77723e..b422754 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -35,7 +35,7 @@ jobs: path: config.cache key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk + run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk make - name: Configure CPython run: | GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ @@ -44,14 +44,27 @@ jobs: --config-cache \ --with-pydebug \ --enable-slower-safety \ + --enable-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt + if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + run: gmake -j8 + - name: Build CPython for compiler warning check + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: >- + python3 Tools/build/check_warnings.py + --compiler-output-file-path=compiler_output_macos.txt + --warning-ignore-file-path=Tools/build/.warningignore_macos + --compiler-output-type=clang + --fail-on-regression + --fail-on-improvement + --path-prefix="./" - name: Tests run: make test diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index b197db8..01bd914 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -67,20 +67,33 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: >- ../cpython-ro-srcdir/configure - CFLAGS="-fdiagnostics-format=json" --config-cache --with-pydebug --enable-slower-safety + --enable-safety --with-openssl=$OPENSSL_DIR ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - name: Build CPython out-of-tree + if: ${{ inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt + run: make -j4 + - name: Build CPython out-of-tree (for compiler warning check) + if: ${{ !inputs.free-threading}} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo - name: Check compiler warnings - run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json + if: ${{ !inputs.free-threading }} + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu + --compiler-output-type=gcc + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw diff --git a/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst new file mode 100644 index 0000000..9750cf2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst @@ -0,0 +1 @@ +Enable compiler options that warn of potential security vulnerabilities. diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index 67f5011..3034638 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -3,3 +3,230 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +Include/internal/mimalloc/mimalloc/internal.h 4 +Include/internal/pycore_backoff.h 1 +Include/internal/pycore_dict.h 2 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_long.h 2 +Include/internal/pycore_object.h 4 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 2 +Modules/_bz2module.c 5 +Modules/_collectionsmodule.c 2 +Modules/_csv.c 3 +Modules/_ctypes/_ctypes.c 37 +Modules/_ctypes/_ctypes_test_generated.c.h 141 +Modules/_ctypes/callbacks.c 6 +Modules/_ctypes/callproc.c 15 +Modules/_ctypes/cfield.c 56 +Modules/_ctypes/malloc_closure.c 3 +Modules/_ctypes/stgdict.c 17 +Modules/_cursesmodule.c 24 +Modules/_datetimemodule.c 28 +Modules/_dbmmodule.c 8 +Modules/_decimal/_decimal.c 15 +Modules/_elementtree.c 42 +Modules/_functoolsmodule.c 6 +Modules/_gdbmmodule.c 5 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24 +Modules/_hashopenssl.c 16 +Modules/_interpchannelsmodule.c 1 +Modules/_interpqueuesmodule.c 1 +Modules/_io/_iomodule.c 1 +Modules/_io/bufferedio.c 4 +Modules/_io/bytesio.c 11 +Modules/_io/fileio.c 9 +Modules/_io/stringio.c 8 +Modules/_io/textio.c 11 +Modules/_json.c 19 +Modules/_localemodule.c 3 +Modules/_lzmamodule.c 10 +Modules/_multiprocessing/semaphore.c 2 +Modules/_operator.c 5 +Modules/_pickle.c 71 +Modules/_posixsubprocess.c 8 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_scproxy.c 3 +Modules/_sqlite/connection.c 4 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sre/sre.c 18 +Modules/_sre/sre_lib.h 62 +Modules/_ssl.c 29 +Modules/_struct.c 1 +Modules/_testbuffer.c 22 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/long.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/monitoring.c 3 +Modules/_testcapi/pyatomic.c 1 +Modules/_testcapi/unicode.c 2 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/watchers.c 3 +Modules/_testcapimodule.c 3 +Modules/_testclinic.c 14 +Modules/_testexternalinspection.c 8 +Modules/_testinternalcapi.c 8 +Modules/_testinternalcapi/pytime.c 8 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testinternalcapi/test_lock.c 2 +Modules/_testlimitedcapi/heaptype_relative.c 4 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 2 +Modules/_threadmodule.c 2 +Modules/_tkinter.c 6 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 11 +Modules/_zoneinfo.c 14 +Modules/arraymodule.c 32 +Modules/atexitmodule.c 1 +Modules/binascii.c 206 +Modules/blake2module.c 6 +Modules/cjkcodecs/_codecs_cn.c 1 +Modules/cjkcodecs/_codecs_iso2022.c 2 +Modules/cjkcodecs/_codecs_jp.c 14 +Modules/cjkcodecs/_codecs_kr.c 3 +Modules/cjkcodecs/cjkcodecs.h 1 +Modules/cjkcodecs/multibytecodec.c 2 +Modules/clinic/_testclinic.c.h 1 +Modules/clinic/arraymodule.c.h 1 +Modules/clinic/unicodedata.c.h 10 +Modules/cmathmodule.c 1 +Modules/expat/siphash.h 8 +Modules/expat/xmlparse.c 45 +Modules/expat/xmltok.c 17 +Modules/expat/xmltok_impl.c 34 +Modules/faulthandler.c 3 +Modules/fcntlmodule.c 1 +Modules/getpath.c 7 +Modules/grpmodule.c 4 +Modules/itertoolsmodule.c 7 +Modules/main.c 2 +Modules/mathmodule.c 15 +Modules/mmapmodule.c 20 +Modules/posixmodule.c 67 +Modules/pwdmodule.c 4 +Modules/pyexpat.c 20 +Modules/readline.c 1 +Modules/resource.c 3 +Modules/rotatingtree.c 1 +Modules/selectmodule.c 6 +Modules/sha3module.c 4 +Modules/signalmodule.c 1 +Modules/socketmodule.c 44 +Modules/syslogmodule.c 3 +Modules/timemodule.c 4 +Modules/unicodedata.c 28 +Modules/unicodedata_db.h 1 +Modules/xxsubtype.c 2 +Modules/zlibmodule.c 16 +Objects/abstract.c 2 +Objects/bytearrayobject.c 34 +Objects/bytes_methods.c 9 +Objects/bytesobject.c 35 +Objects/call.c 13 +Objects/classobject.c 4 +Objects/codeobject.c 15 +Objects/descrobject.c 2 +Objects/dictobject.c 28 +Objects/fileobject.c 3 +Objects/floatobject.c 30 +Objects/frameobject.c 19 +Objects/funcobject.c 1 +Objects/genobject.c 5 +Objects/listobject.c 43 +Objects/longobject.c 46 +Objects/memoryobject.c 6 +Objects/methodobject.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 1 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 1 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 1 +Objects/mimalloc/prim/osx/../unix/prim.c 2 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 1 +Objects/moduleobject.c 2 +Objects/object.c 1 +Objects/obmalloc.c 6 +Objects/odictobject.c 3 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 4 +Objects/stringlib/codecs.h 26 +Objects/stringlib/eq.h 1 +Objects/stringlib/fastsearch.h 14 +Objects/stringlib/join.h 1 +Objects/stringlib/replace.h 4 +Objects/stringlib/repr.h 21 +Objects/stringlib/transmogrify.h 5 +Objects/structseq.c 14 +Objects/tupleobject.c 10 +Objects/typeobject.c 17 +Objects/unicodectype.c 7 +Objects/unicodeobject.c 113 +Parser/action_helpers.c 4 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 12 +Parser/parser.c 116 +Parser/pegen.c 7 +Parser/string_parser.c 7 +Parser/tokenizer/file_tokenizer.c 8 +Parser/tokenizer/helpers.c 7 +Parser/tokenizer/readline_tokenizer.c 3 +Programs/_freeze_module.c 1 +Python/Python-ast.c 15 +Python/asdl.c 3 +Python/assemble.c 7 +Python/ast_opt.c 7 +Python/bltinmodule.c 9 +Python/bootstrap_hash.c 4 +Python/ceval.c 8 +Python/ceval_gil.c 2 +Python/codecs.c 32 +Python/codegen.c 6 +Python/compile.c 2 +Python/context.c 1 +Python/crossinterp.c 2 +Python/crossinterp_data_lookup.h 1 +Python/dtoa.c 34 +Python/errors.c 1 +Python/fileutils.c 7 +Python/flowgraph.c 8 +Python/formatter_unicode.c 7 +Python/frame.c 4 +Python/gc.c 8 +Python/generated_cases.c.h 35 +Python/getargs.c 11 +Python/import.c 5 +Python/initconfig.c 11 +Python/instrumentation.c 31 +Python/intrinsics.c 1 +Python/legacy_tracing.c 3 +Python/lock.c 4 +Python/marshal.c 11 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/preconfig.c 2 +Python/pyarena.c 1 +Python/pyhash.c 2 +Python/pylifecycle.c 7 +Python/pystate.c 6 +Python/pystrhex.c 19 +Python/pystrtod.c 3 +Python/qsbr.c 2 +Python/specialize.c 10 +Python/suggestions.c 12 +Python/symtable.c 18 +Python/sysmodule.c 2 +Python/thread_pthread.h 1 +Python/traceback.c 6 +Python/tracemalloc.c 6 diff --git a/Tools/build/.warningignore_ubuntu b/Tools/build/.warningignore_ubuntu index 469c727..e98305e 100644 --- a/Tools/build/.warningignore_ubuntu +++ b/Tools/build/.warningignore_ubuntu @@ -3,3 +3,263 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) +/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/evp.h 2 +/home/runner/work/cpython/cpython/multissl/openssl/3.0.15/include/openssl/ssl.h 4 +/usr/include/tcl8.6/tclTomMathDecls.h 1 +Include/cpython/bytearrayobject.h 1 +Include/cpython/bytesobject.h 3 +Include/cpython/dictobject.h 2 +Include/cpython/listobject.h 1 +Include/cpython/pyctype.h 2 +Include/cpython/tupleobject.h 1 +Include/cpython/unicodeobject.h 7 +Include/internal/mimalloc/mimalloc/internal.h 4 +Include/internal/mimalloc/mimalloc/types.h 2 +Include/internal/pycore_asdl.h 1 +Include/internal/pycore_backoff.h 3 +Include/internal/pycore_blocks_output_buffer.h 1 +Include/internal/pycore_dict.h 2 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_gc.h 1 +Include/internal/pycore_interp.h 1 +Include/internal/pycore_list.h 1 +Include/internal/pycore_long.h 3 +Include/internal/pycore_object.h 4 +Include/internal/pycore_obmalloc.h 1 +Include/internal/pycore_pymath.h 1 +Include/internal/pycore_runtime_init.h 1 +Include/longobject.h 1 +Include/object.h 4 +Include/opcode_ids.h 1 +Include/pymacro.h 4 +Include/pymath.h 1 +Include/pymem.h 2 +Include/pyport.h 2 +Modules/_asynciomodule.c 3 +Modules/_bisectmodule.c 4 +Modules/_bz2module.c 5 +Modules/_collectionsmodule.c 2 +Modules/_csv.c 2 +Modules/_ctypes/_ctypes.c 53 +Modules/_ctypes/_ctypes_test.c 7 +Modules/_ctypes/_ctypes_test_generated.c.h 2 +Modules/_ctypes/callbacks.c 3 +Modules/_ctypes/callproc.c 13 +Modules/_ctypes/cfield.c 33 +Modules/_ctypes/stgdict.c 17 +Modules/_cursesmodule.c 27 +Modules/_datetimemodule.c 38 +Modules/_datetimemodule.c 38 +Modules/_dbmmodule.c 7 +Modules/_decimal/_decimal.c 19 +Modules/_elementtree.c 37 +Modules/_functoolsmodule.c 6 +Modules/_gdbmmodule.c 4 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84 +Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 4 +Modules/_hashopenssl.c 13 +Modules/_io/_iomodule.c 1 +Modules/_io/bufferedio.c 15 +Modules/_io/bytesio.c 14 +Modules/_io/fileio.c 9 +Modules/_io/stringio.c 8 +Modules/_io/textio.c 17 +Modules/_json.c 19 +Modules/_localemodule.c 2 +Modules/_lsprof.c 5 +Modules/_lzmamodule.c 6 +Modules/_multiprocessing/posixshmem.c 1 +Modules/_multiprocessing/semaphore.c 1 +Modules/_operator.c 5 +Modules/_pickle.c 73 +Modules/_posixsubprocess.c 11 +Modules/_queuemodule.c 4 +Modules/_randommodule.c 3 +Modules/_sqlite/connection.c 5 +Modules/_sqlite/cursor.c 3 +Modules/_sqlite/module.c 2 +Modules/_sre/sre.c 14 +Modules/_sre/sre_lib.h 25 +Modules/_ssl.c 26 +Modules/_struct.c 3 +Modules/_testbuffer.c 27 +Modules/_testcapi/bytes.c 1 +Modules/_testcapi/heaptype.c 1 +Modules/_testcapi/long.c 2 +Modules/_testcapi/mem.c 2 +Modules/_testcapi/monitoring.c 3 +Modules/_testcapi/pyatomic.c 4 +Modules/_testcapi/pyatomic.c 4 +Modules/_testcapi/unicode.c 1 +Modules/_testcapi/vectorcall.c 3 +Modules/_testcapi/watchers.c 3 +Modules/_testcapimodule.c 1 +Modules/_testclinic.c 14 +Modules/_testclinic.c 14 +Modules/_testexternalinspection.c 7 +Modules/_testinternalcapi.c 10 +Modules/_testinternalcapi/test_critical_sections.c 1 +Modules/_testinternalcapi/test_lock.c 4 +Modules/_testlimitedcapi/heaptype_relative.c 3 +Modules/_testlimitedcapi/object.c 2 +Modules/_testlimitedcapi/unicode.c 1 +Modules/_testmultiphase.c 1 +Modules/_tkinter.c 8 +Modules/_xxtestfuzz/_xxtestfuzz.c 1 +Modules/_xxtestfuzz/fuzzer.c 13 +Modules/_zoneinfo.c 17 +Modules/arraymodule.c 48 +Modules/binascii.c 208 +Modules/blake2module.c 8 +Modules/cjkcodecs/_codecs_iso2022.c 1 +Modules/cjkcodecs/_codecs_jp.c 17 +Modules/cjkcodecs/_codecs_kr.c 7 +Modules/cjkcodecs/alg_jisx0201.h 2 +Modules/cjkcodecs/cjkcodecs.h 1 +Modules/cjkcodecs/multibytecodec.c 12 +Modules/expat/pyexpatns.h 3 +Modules/expat/siphash.h 1 +Modules/expat/xmlparse.c 43 +Modules/expat/xmltok.c 15 +Modules/expat/xmltok.c 15 +Modules/expat/xmltok_impl.c 8 +Modules/faulthandler.c 5 +Modules/fcntlmodule.c 6 +Modules/getpath.c 7 +Modules/grpmodule.c 4 +Modules/itertoolsmodule.c 4 +Modules/main.c 2 +Modules/mathmodule.c 14 +Modules/mmapmodule.c 22 +Modules/mmapmodule.c 22 +Modules/posixmodule.c 79 +Modules/pwdmodule.c 4 +Modules/pyexpat.c 10 +Modules/readline.c 1 +Modules/resource.c 4 +Modules/rotatingtree.c 2 +Modules/selectmodule.c 1 +Modules/sha3module.c 4 +Modules/signalmodule.c 3 +Modules/socketmodule.c 75 +Modules/syslogmodule.c 3 +Modules/termios.c 1 +Modules/timemodule.c 10 +Modules/unicodedata.c 24 +Modules/unicodedata_db.h 1 +Modules/zlibmodule.c 24 +Objects/abstract.c 6 +Objects/bytearrayobject.c 42 +Objects/bytes_methods.c 4 +Objects/bytesobject.c 45 +Objects/call.c 12 +Objects/classobject.c 4 +Objects/codeobject.c 19 +Objects/descrobject.c 2 +Objects/dictobject.c 31 +Objects/fileobject.c 3 +Objects/floatobject.c 10 +Objects/frameobject.c 16 +Objects/funcobject.c 1 +Objects/genobject.c 3 +Objects/listobject.c 38 +Objects/longobject.c 47 +Objects/memoryobject.c 12 +Objects/methodobject.c 1 +Objects/mimalloc/alloc.c 6 +Objects/mimalloc/arena.c 6 +Objects/mimalloc/heap.c 2 +Objects/mimalloc/init.c 2 +Objects/mimalloc/options.c 4 +Objects/mimalloc/os.c 4 +Objects/mimalloc/page-queue.c 2 +Objects/mimalloc/page.c 2 +Objects/mimalloc/prim/unix/prim.c 6 +Objects/mimalloc/random.c 1 +Objects/mimalloc/segment.c 11 +Objects/mimalloc/stats.c 5 +Objects/moduleobject.c 4 +Objects/object.c 1 +Objects/obmalloc.c 6 +Objects/odictobject.c 6 +Objects/rangeobject.c 10 +Objects/setobject.c 13 +Objects/sliceobject.c 2 +Objects/stringlib/codecs.h 12 +Objects/stringlib/eq.h 1 +Objects/stringlib/fastsearch.h 8 +Objects/stringlib/join.h 3 +Objects/stringlib/replace.h 4 +Objects/stringlib/repr.h 21 +Objects/stringlib/transmogrify.h 26 +Objects/structseq.c 10 +Objects/tupleobject.c 8 +Objects/typeobject.c 38 +Objects/unicodectype.c 7 +Objects/unicodeobject.c 135 +Parser/action_helpers.c 3 +Parser/lexer/buffer.c 1 +Parser/lexer/lexer.c 14 +Parser/parser.c 116 +Parser/pegen.c 8 +Parser/string_parser.c 7 +Parser/tokenizer/file_tokenizer.c 9 +Parser/tokenizer/helpers.c 7 +Parser/tokenizer/readline_tokenizer.c 4 +Python/assemble.c 11 +Python/ast_opt.c 5 +Python/bltinmodule.c 8 +Python/bootstrap_hash.c 7 +Python/ceval.c 8 +Python/ceval_gil.c 2 +Python/codecs.c 28 +Python/codegen.c 6 +Python/compile.c 2 +Python/context.c 1 +Python/crossinterp.c 2 +Python/crossinterp_data_lookup.h 1 +Python/dtoa.c 30 +Python/errors.c 1 +Python/fileutils.c 11 +Python/flowgraph.c 7 +Python/formatter_unicode.c 6 +Python/frame.c 3 +Python/gc.c 9 +Python/gc.c 9 +Python/generated_cases.c.h 27 +Python/generated_cases.c.h 27 +Python/getargs.c 7 +Python/hashtable.c 1 +Python/import.c 6 +Python/import.c 7 +Python/initconfig.c 11 +Python/instrumentation.c 43 +Python/intrinsics.c 1 +Python/legacy_tracing.c 3 +Python/lock.c 4 +Python/marshal.c 16 +Python/modsupport.c 3 +Python/mystrtoul.c 4 +Python/pathconfig.c 1 +Python/perf_jit_trampoline.c 32 +Python/perf_trampoline.c 12 +Python/preconfig.c 2 +Python/pyarena.c 1 +Python/pyhash.c 4 +Python/pylifecycle.c 3 +Python/pystate.c 4 +Python/pystrhex.c 15 +Python/pystrtod.c 12 +Python/pytime.c 2 +Python/qsbr.c 2 +Python/specialize.c 9 +Python/suggestions.c 12 +Python/symtable.c 15 +Python/sysmodule.c 2 +Python/thread.c 1 +Python/thread_pthread.h 6 +Python/traceback.c 6 +Python/tracemalloc.c 6 diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 1ed8344..e58ee2a 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -1,105 +1,68 @@ """ -Parses compiler output with -fdiagnostics-format=json and checks that warnings +Parses compiler output from Clang or GCC and checks that warnings exist only in files that are expected to have warnings. """ import argparse from collections import defaultdict -import json import re import sys from pathlib import Path from typing import NamedTuple + class FileWarnings(NamedTuple): name: str count: int -def extract_warnings_from_compiler_output_clang( +def extract_warnings_from_compiler_output( compiler_output: str, + compiler_output_type: str, + path_prefix: str = "", ) -> list[dict]: """ - Extracts warnings from the compiler output when using clang - """ - # Regex to find warnings in the compiler output - clang_warning_regex = re.compile( - r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: " - r"(?P<message>.*) (?P<option>\[-[^\]]+\])$" - ) - compiler_warnings = [] - for line in compiler_output.splitlines(): - if match := clang_warning_regex.match(line): - compiler_warnings.append( - { - "file": match.group("file"), - "line": match.group("line"), - "column": match.group("column"), - "message": match.group("message"), - "option": match.group("option").lstrip("[").rstrip("]"), - } - ) - - return compiler_warnings - - -def extract_warnings_from_compiler_output_json( - compiler_output: str, -) -> list[dict]: - """ - Extracts warnings from the compiler output when using - -fdiagnostics-format=json. - - Compiler output as a whole is not a valid json document, - but includes many json objects and may include other output - that is not json. + Extracts warnings from the compiler output based on compiler + output type. Removes path prefix from file paths if provided. + Compatible with GCC and Clang compiler output. """ - # Regex to find json arrays at the top level of the file - # in the compiler output - json_arrays = re.findall(r"\[(?:[^[\]]|\[[^]]*])*]", compiler_output) + # Choose pattern and compile regex for particular compiler output + if compiler_output_type == "gcc": + regex_pattern = ( + r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: " + r"(?P<message>.*?)(?: (?P<option>\[-[^\]]+\]))?$" + ) + elif compiler_output_type == "clang": + regex_pattern = ( + r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: " + r"(?P<message>.*) (?P<option>\[-[^\]]+\])$" + ) + compiled_regex = re.compile(regex_pattern) compiler_warnings = [] - for array in json_arrays: - try: - json_data = json.loads(array) - json_objects_in_array = [entry for entry in json_data] - warning_list = [ - entry - for entry in json_objects_in_array - if entry.get("kind") == "warning" - ] - for warning in warning_list: - locations = warning["locations"] - for location in locations: - for key in ["caret", "start", "end"]: - if key in location: - compiler_warnings.append( - { - # Remove leading current directory if present - "file": location[key]["file"].lstrip("./"), - "line": location[key]["line"], - "column": location[key]["column"], - "message": warning["message"], - "option": warning["option"], - } - ) - # Found a caret, start, or end in location so - # break out completely to address next warning - break - else: - continue - break - - except json.JSONDecodeError: - continue # Skip malformed JSON + for i, line in enumerate(compiler_output.splitlines(), start=1): + if match := compiled_regex.match(line): + try: + compiler_warnings.append( + { + "file": match.group("file").removeprefix(path_prefix), + "line": match.group("line"), + "column": match.group("column"), + "message": match.group("message"), + "option": match.group("option").lstrip("[").rstrip("]"), + } + ) + except: + print(f"Error parsing compiler output. Unable to extract warning on line {i}:\n{line}") + sys.exit(1) return compiler_warnings def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: """ - Returns a dictionary where the key is the file and the data is the warnings - in that file. Does not include duplicate warnings for a file from list of - provided warnings. + Returns a dictionary where the key is the file and the data is the + warnings in that file. Does not include duplicate warnings for a + file from list of provided warnings. """ warnings_by_file = defaultdict(list) warnings_added = set() @@ -124,22 +87,28 @@ def get_unexpected_warnings( are associated with a file that is not found in the list of files with expected warnings """ - unexpected_warnings = [] + unexpected_warnings = {} for file in files_with_warnings.keys(): found_file_in_ignore_list = False for ignore_file in files_with_expected_warnings: if file == ignore_file.name: if len(files_with_warnings[file]) > ignore_file.count: - unexpected_warnings.extend(files_with_warnings[file]) + unexpected_warnings[file] = (files_with_warnings[file], ignore_file.count) found_file_in_ignore_list = True break if not found_file_in_ignore_list: - unexpected_warnings.extend(files_with_warnings[file]) + unexpected_warnings[file] = (files_with_warnings[file], 0) if unexpected_warnings: print("Unexpected warnings:") - for warning in unexpected_warnings: - print(warning) + for file in unexpected_warnings: + print( + f"{file} expected {unexpected_warnings[file][1]} warnings," + f" found {len(unexpected_warnings[file][0])}" + ) + for warning in unexpected_warnings[file][0]: + print(warning) + return 1 return 0 @@ -156,14 +125,14 @@ def get_unexpected_improvements( unexpected_improvements = [] for file in files_with_expected_warnings: if file.name not in files_with_warnings.keys(): - unexpected_improvements.append(file) + unexpected_improvements.append((file.name, file.count, 0)) elif len(files_with_warnings[file.name]) < file.count: - unexpected_improvements.append(file) + unexpected_improvements.append((file.name, file.count, len(files_with_warnings[file.name]))) if unexpected_improvements: print("Unexpected improvements:") for file in unexpected_improvements: - print(file.name) + print(f"{file[0]} expected {file[1]} warnings, found {file[2]}") return 1 return 0 @@ -204,8 +173,15 @@ def main(argv: list[str] | None = None) -> int: "--compiler-output-type", type=str, required=True, - choices=["json", "clang"], - help="Type of compiler output file (json or clang)", + choices=["gcc", "clang"], + help="Type of compiler output file (GCC or Clang)", + ) + parser.add_argument( + "-p", + "--path-prefix", + type=str, + help="Path prefix to remove from the start of file paths" + " in compiler output", ) args = parser.parse_args(argv) @@ -241,7 +217,9 @@ def main(argv: list[str] | None = None) -> int: # where the first element is the file name and the second element # is the number of warnings expected in that file files_with_expected_warnings = { - FileWarnings(file.strip().split()[0], int(file.strip().split()[1])) + FileWarnings( + file.strip().split()[0], int(file.strip().split()[1]) + ) for file in clean_files if file.strip() and not file.startswith("#") } @@ -249,14 +227,11 @@ def main(argv: list[str] | None = None) -> int: with Path(args.compiler_output_file_path).open(encoding="UTF-8") as f: compiler_output_file_contents = f.read() - if args.compiler_output_type == "json": - warnings = extract_warnings_from_compiler_output_json( - compiler_output_file_contents - ) - elif args.compiler_output_type == "clang": - warnings = extract_warnings_from_compiler_output_clang( - compiler_output_file_contents - ) + warnings = extract_warnings_from_compiler_output( + compiler_output_file_contents, + args.compiler_output_type, + args.path_prefix + ) files_with_warnings = get_warnings_by_file(warnings) @@ -272,6 +247,11 @@ def main(argv: list[str] | None = None) -> int: if args.fail_on_improvement: exit_code |= status + print( + "For information about this tool and its configuration" + " visit https://devguide.python.org/development-tools/warnings/" + ) + return exit_code @@ -1829,7 +1829,7 @@ Optional Features: (default is no) --enable-bolt enable usage of the llvm-bolt post-link optimizer (default is no) - --disable-safety disable usage of the security compiler options with + --enable-safety enable usage of the security compiler options with no performance overhead --enable-slower-safety enable usage of the security compiler options with performance overhead @@ -9671,25 +9671,25 @@ fi # Enable flags that warn and protect for potential security vulnerabilities. # These flags should be enabled by default for all builds. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-safety" >&5 -printf %s "checking for --disable-safety... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-safety" >&5 +printf %s "checking for --enable-safety... " >&6; } # Check whether --enable-safety was given. if test ${enable_safety+y} then : - enableval=$enable_safety; if test "x$enable_safety" = xyes + enableval=$enable_safety; if test "x$disable_safety" = xyes then : - disable_safety=no + enable_safety=no else $as_nop - disable_safety=yes + enable_safety=yes fi else $as_nop - disable_safety=no + enable_safety=no fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_safety" >&5 -printf "%s\n" "$disable_safety" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_safety" >&5 +printf "%s\n" "$enable_safety" >&6; } -if test "$disable_safety" = "no" +if test "$enable_safety" = "yes" then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fstack-protector-strong" >&5 printf %s "checking whether C compiler accepts -fstack-protector-strong... " >&6; } @@ -9769,6 +9769,201 @@ else $as_nop printf "%s\n" "$as_me: WARNING: -Wtrampolines not supported" >&2;} fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wconversion" >&5 +printf %s "checking whether C compiler accepts -Wconversion... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wconversion+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wconversion" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wconversion=yes +else $as_nop + ax_cv_check_cflags__Werror__Wconversion=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wconversion" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wconversion" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wconversion" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wconversion not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wconversion not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wimplicit-fallthrough" >&5 +printf %s "checking whether C compiler accepts -Wimplicit-fallthrough... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wimplicit_fallthrough+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wimplicit-fallthrough" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=yes +else $as_nop + ax_cv_check_cflags__Werror__Wimplicit_fallthrough=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wimplicit_fallthrough" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wimplicit-fallthrough not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wimplicit-fallthrough not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Werror=format-security" >&5 +printf %s "checking whether C compiler accepts -Werror=format-security... " >&6; } +if test ${ax_cv_check_cflags__Werror__Werror_format_security+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Werror=format-security" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Werror_format_security=yes +else $as_nop + ax_cv_check_cflags__Werror__Werror_format_security=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Werror_format_security" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Werror_format_security" >&6; } +if test "x$ax_cv_check_cflags__Werror__Werror_format_security" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Werror=format-security not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Werror=format-security not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wbidi-chars=any" >&5 +printf %s "checking whether C compiler accepts -Wbidi-chars=any... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wbidi_chars_any+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wbidi-chars=any" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wbidi_chars_any=yes +else $as_nop + ax_cv_check_cflags__Werror__Wbidi_chars_any=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wbidi_chars_any" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wbidi_chars_any" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wbidi_chars_any" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wbidi-chars=any not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wbidi-chars=any not supported" >&2;} +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -Wall" >&5 +printf %s "checking whether C compiler accepts -Wall... " >&6; } +if test ${ax_cv_check_cflags__Werror__Wall+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -Wall" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__Wall=yes +else $as_nop + ax_cv_check_cflags__Werror__Wall=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__Wall" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__Wall" >&6; } +if test "x$ax_cv_check_cflags__Werror__Wall" = xyes +then : + CFLAGS_NODIST="$CFLAGS_NODIST -Wall" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: -Wall not supported" >&5 +printf "%s\n" "$as_me: WARNING: -Wall not supported" >&2;} +fi + fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-slower-safety" >&5 diff --git a/configure.ac b/configure.ac index ab39f46..5fc7e08 100644 --- a/configure.ac +++ b/configure.ac @@ -2500,16 +2500,21 @@ AS_VAR_IF([with_strict_overflow], [yes], # Enable flags that warn and protect for potential security vulnerabilities. # These flags should be enabled by default for all builds. -AC_MSG_CHECKING([for --disable-safety]) +AC_MSG_CHECKING([for --enable-safety]) AC_ARG_ENABLE([safety], - [AS_HELP_STRING([--disable-safety], [disable usage of the security compiler options with no performance overhead])], - [AS_VAR_IF([enable_safety], [yes], [disable_safety=no], [disable_safety=yes])], [disable_safety=no]) -AC_MSG_RESULT([$disable_safety]) + [AS_HELP_STRING([--enable-safety], [enable usage of the security compiler options with no performance overhead])], + [AS_VAR_IF([disable_safety], [yes], [enable_safety=no], [enable_safety=yes])], [enable_safety=no]) +AC_MSG_RESULT([$enable_safety]) -if test "$disable_safety" = "no" +if test "$enable_safety" = "yes" then AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [CFLAGS_NODIST="$CFLAGS_NODIST -fstack-protector-strong"], [AC_MSG_WARN([-fstack-protector-strong not supported])], [-Werror]) AX_CHECK_COMPILE_FLAG([-Wtrampolines], [CFLAGS_NODIST="$CFLAGS_NODIST -Wtrampolines"], [AC_MSG_WARN([-Wtrampolines not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wconversion], [CFLAGS_NODIST="$CFLAGS_NODIST -Wconversion"], [AC_MSG_WARN([-Wconversion not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [CFLAGS_NODIST="$CFLAGS_NODIST -Wimplicit-fallthrough"], [AC_MSG_WARN([-Wimplicit-fallthrough not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Werror=format-security], [CFLAGS_NODIST="$CFLAGS_NODIST -Werror=format-security"], [AC_MSG_WARN([-Werror=format-security not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wbidi-chars=any], [CFLAGS_NODIST="$CFLAGS_NODIST -Wbidi-chars=any"], [AC_MSG_WARN([-Wbidi-chars=any not supported])], [-Werror]) + AX_CHECK_COMPILE_FLAG([-Wall], [CFLAGS_NODIST="$CFLAGS_NODIST -Wall"], [AC_MSG_WARN([-Wall not supported])], [-Werror]) fi AC_MSG_CHECKING([for --enable-slower-safety]) |