diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | .hgignore | 3 | ||||
| -rw-r--r-- | Makefile.pre.in | 31 | ||||
| -rw-r--r-- | Misc/NEWS | 3 | ||||
| -rw-r--r-- | README | 29 | ||||
| -rwxr-xr-x | configure | 82 | ||||
| -rw-r--r-- | configure.ac | 45 |
7 files changed, 189 insertions, 7 deletions
@@ -6,6 +6,9 @@ *.pyo *.rej *~ +*.gc?? +*.profclang? +*.profraw Doc/build/ Doc/tools/docutils/ Doc/tools/jinja2/ @@ -45,6 +45,9 @@ libpython*.so* *.pyd *.cover *~ +*.gc?? +*.profclang? +*.profraw Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/Makefile.pre.in b/Makefile.pre.in index 5989143..4082313 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -42,6 +42,11 @@ SVNVERSION= @SVNVERSION@ HGVERSION= @HGVERSION@ HGTAG= @HGTAG@ HGBRANCH= @HGBRANCH@ +PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ +PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ +LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ +LLVM_PROF_FILE=@LLVM_PROF_FILE@ +LLVM_PROF_ERR=@LLVM_PROF_ERR@ GNULD= @GNULD@ @@ -204,8 +209,7 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@ TCLTK_LIBS= @TCLTK_LIBS@ # The task to run while instrument when building the profile-opt target -PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck -#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py +PROFILE_TASK=-m test.regrtest >/dev/null 2>&1 # === Definitions added by makesetup === @@ -420,27 +424,38 @@ LIBRARY_OBJS= \ all: build_all build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks -# Compile a binary with gcc profile guided optimization. +# Compile a binary with profile guided optimization. profile-opt: + @if [ $(LLVM_PROF_ERR) == yes ]; then \ + echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ + echo "Please add it to PATH and run ./configure again" ;\ + exit 1;\ + fi @echo "Building with support for profile generation:" $(MAKE) clean + $(MAKE) profile-removal $(MAKE) build_all_generate_profile - @echo "Running benchmark to generate profile data:" $(MAKE) profile-removal + @echo "Running code to generate profile data (this can take a while):" $(MAKE) run_profile_task + $(MAKE) build_all_merge_profile @echo "Rebuilding with profile guided optimizations:" $(MAKE) clean $(MAKE) build_all_use_profile + $(MAKE) profile-removal build_all_generate_profile: - $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov" + $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" run_profile_task: : # FIXME: can't run for a cross build - ./$(BUILDPYTHON) $(PROFILE_TASK) + $(LLVM_PROF_FILE) ./$(BUILDPYTHON) $(PROFILE_TASK) || true + +build_all_merge_profile: + $(LLVM_PROF_MERGER) build_all_use_profile: - $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use" + $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_USE_FLAG)" coverage: @echo "Building with support for coverage checking:" @@ -1330,9 +1345,11 @@ clean: pycremoval find build -name 'fficonfig.h' -exec rm -f {} ';' || true find build -name 'fficonfig.py' -exec rm -f {} ';' || true -rm -f Lib/lib2to3/*Grammar*.pickle + -rm -rf build profile-removal: find . -name '*.gc??' -exec rm -f {} ';' + find . -name '*.profclang?' -exec rm -f {} ';' clobber: clean profile-removal -rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ @@ -150,6 +150,9 @@ Library Build ----- +- Issue #24915: When doing a PGO build, the test suite is now used instead of + pybench; Clang support was also added as part off this work. + - Issue #24986: It is now possible to build Python on Windows without errors when external libraries are not available. @@ -173,6 +173,11 @@ rebuilt. In this case, you may have to run make again to correctly build your desired target. The interpreter executable is built in the top level directory. +If you need an optimized version of Python, you type "make profile-opt" +in the top level directory. This will rebuild the interpreter executable +using Profile Guided Optimization (PGO). For more details, see the +section below. + Once you have built a Python interpreter, see the subsections below on testing and installation. If you run into trouble, see the next section. @@ -185,6 +190,30 @@ guidance of the setup.py script, which is run by Make after the interpreter has been built. +Profile Guided Optimization +--------------------------- + +PGO takes advantage of recent versions of the GCC or Clang compilers. +If ran, the "profile-opt" rule will do several steps. + +First, the entire Python directory is cleaned of temporary files that +may resulted in a previous compilation. + +Then, an instrumented version of the interpreter is built, using suitable +compiler flags for each flavour. Note that this is just an intermediary +step and the binary resulted after this step is not good for real life +workloads, as it has profiling instructions embedded inside. + +After this instrumented version of the interpreter is built, the Makefile +will automatically run a training workload. This is necessary in order to +profile the interpreter execution. Note also that any output, both stdout +and stderr, that may appear at this step is supressed. + +Finally, the last step is to rebuild the interpreter, using the information +collected in the previous one. The end result will be a the Python binary +that is optimized and suitable for distribution or production installation. + + Troubleshooting --------------- @@ -661,6 +661,12 @@ LDSHARED SO LIBTOOL_CRUFT OTHER_LIBTOOL_OPT +LLVM_PROF_FOUND +LLVM_PROF_ERR +LLVM_PROF_FILE +LLVM_PROF_MERGER +PGO_PROF_USE_FLAG +PGO_PROF_GEN_FLAG UNIVERSAL_ARCH_FLAGS BASECFLAGS OPT @@ -6326,6 +6332,82 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext CFLAGS=$save_CFLAGS fi + +# Enable PGO flags. +# Extract the first word of "llvm-profdata", so it can be a program name with args. +set dummy llvm-profdata; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$LLVM_PROF_FOUND"; then + ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_LLVM_PROF_FOUND="found" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found" +fi +fi +LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND +if test -n "$LLVM_PROF_FOUND"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5 +$as_echo "$LLVM_PROF_FOUND" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + + # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X). diff --git a/configure.ac b/configure.ac index 2c998b1..decf0ec 100644 --- a/configure.ac +++ b/configure.ac @@ -1352,6 +1352,51 @@ then CFLAGS=$save_CFLAGS fi + +# Enable PGO flags. +AC_SUBST(PGO_PROF_GEN_FLAG) +AC_SUBST(PGO_PROF_USE_FLAG) +AC_SUBST(LLVM_PROF_MERGER) +AC_SUBST(LLVM_PROF_FILE) +AC_SUBST(LLVM_PROF_ERR) +AC_SUBST(LLVM_PROF_FOUND) +AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found) +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + + # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X). |