Fixes Issue #27983: Cause lack of llvm-profdata tool when using clang as

required for PGO linking to be a configure time error rather than
make time when --with-optimizations is enabled.  Also improve our
ability to find the llvm-profdata tool on MacOS and some Linuxes.

3 files changed, 219 insertions, 21 deletions

diff --git a/Misc/NEWS b/Misc/NEWS
index 71598b7..de45fc9 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,11 @@ What's New in Python 3.6.0 beta 1
 Core and Builtins
 -----------------
 
+- Issue #27983: Cause lack of llvm-profdata tool when using clang as
+  required for PGO linking to be a configure time error rather than
+  make time when --with-optimizations is enabled.  Also improve our
+  ability to find the llvm-profdata tool on MacOS and some Linuxes.
+
 - Issue #26307: The profile-opt build now applys PGO to the built-in modules.
 
 - Issue #27078: Added BUILD_STRING opcode.  Optimized f-strings evaluation.
diff --git a/configure b/configure
index 94a7435..0be0850 100755
--- a/configure
+++ b/configure
@@ -668,6 +668,11 @@ CFLAGS_NODIST
 BASECFLAGS
 OPT
 LLVM_PROF_FOUND
+target_os
+target_vendor
+target_cpu
+target
+LLVM_PROFDATA
 LLVM_PROF_ERR
 LLVM_PROF_FILE
 LLVM_PROF_MERGER
@@ -1462,6 +1467,7 @@ _ACEOF
 System types:
   --build=BUILD     configure for building on BUILD [guessed]
   --host=HOST       cross-compile to build programs to run on HOST [BUILD]
+  --target=TARGET   configure for building compilers for TARGET [HOST]
 _ACEOF
 fi
 
@@ -6497,9 +6503,11 @@ if test "$Py_OPT" = 'true' ; then
       ;;
     esac
   DEF_MAKE_ALL_RULE="profile-opt"
+  REQUIRE_PGO="yes"
   DEF_MAKE_RULE="build_all"
 else
   DEF_MAKE_ALL_RULE="build_all"
+  REQUIRE_PGO="no"
   DEF_MAKE_RULE="all"
 fi
 
@@ -6558,25 +6566,84 @@ fi
 
 
 
+# Make this work on systems where llvm tools are not installed with their
+# normal names in the default $PATH (ie: Ubuntu).  They exist under the
+# non-suffixed name in their versioned llvm directory.
+llvm_bin_dir=''
+llvm_path="${PATH}"
+if test "${CC}" = "clang"
+then
+  clang_bin=`which clang`
+  # Some systems install clang elsewhere as a symlink to the real path
+  # which is where the related llvm tools are located.
+  if test -L "${clang_bin}"
+  then
+    clang_dir=`dirname "${clang_bin}"`
+    clang_bin=`readlink "${clang_bin}"`
+    llvm_bin_dir="${clang_dir}/"`dirname "${clang_bin}"`
+    llvm_path="${llvm_path}${PATH_SEPARATOR}${llvm_bin_dir}"
+  fi
+fi
 
-# Extract the first word of "llvm-profdata", so it can be a program name with args.
-set dummy llvm-profdata; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking target system type" >&5
+$as_echo_n "checking target system type... " >&6; }
+if ${ac_cv_target+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test "x$target_alias" = x; then
+  ac_cv_target=$ac_cv_host
+else
+  ac_cv_target=`$SHELL "$ac_aux_dir/config.sub" $target_alias` ||
+    as_fn_error $? "$SHELL $ac_aux_dir/config.sub $target_alias failed" "$LINENO" 5
+fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_target" >&5
+$as_echo "$ac_cv_target" >&6; }
+case $ac_cv_target in
+*-*-*) ;;
+*) as_fn_error $? "invalid value of canonical target" "$LINENO" 5;;
+esac
+target=$ac_cv_target
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_target
+shift
+target_cpu=$1
+target_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+target_os=$*
+IFS=$ac_save_IFS
+case $target_os in *\ *) target_os=`echo "$target_os" | sed 's/ /-/g'`;; esac
+
+
+# The aliases save the names the user supplied, while $host etc.
+# will get canonicalized.
+test -n "$target_alias" &&
+  test "$program_prefix$program_suffix$program_transform_name" = \
+    NONENONEs,x,x, &&
+  program_prefix=${target_alias}-
+# Extract the first word of "$target_alias-llvm-profdata", so it can be a program name with args.
+set dummy $target_alias-llvm-profdata; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
 $as_echo_n "checking for $ac_word... " >&6; }
-if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
+if ${ac_cv_path_LLVM_PROFDATA+:} false; then :
   $as_echo_n "(cached) " >&6
 else
-  if test -n "$LLVM_PROF_FOUND"; then
-  ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
-else
-as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
-for as_dir in $PATH
+  case $LLVM_PROFDATA in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_LLVM_PROFDATA="$LLVM_PROFDATA" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in ${llvm_path}
 do
   IFS=$as_save_IFS
   test -z "$as_dir" && as_dir=.
     for ac_exec_ext in '' $ac_executable_extensions; do
   if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
-    ac_cv_prog_LLVM_PROF_FOUND="found"
+    ac_cv_path_LLVM_PROFDATA="$as_dir/$ac_word$ac_exec_ext"
     $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
     break 2
   fi
@@ -6584,30 +6651,105 @@ done
   done
 IFS=$as_save_IFS
 
-  test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
+  ;;
+esac
 fi
+LLVM_PROFDATA=$ac_cv_path_LLVM_PROFDATA
+if test -n "$LLVM_PROFDATA"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROFDATA" >&5
+$as_echo "$LLVM_PROFDATA" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+if test -z "$ac_cv_path_LLVM_PROFDATA"; then
+  if test "$build" = "$target"; then
+    ac_pt_LLVM_PROFDATA=$LLVM_PROFDATA
+    # Extract the first word of "llvm-profdata", so it can be a program name with args.
+set dummy llvm-profdata; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_ac_pt_LLVM_PROFDATA+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  case $ac_pt_LLVM_PROFDATA in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_ac_pt_LLVM_PROFDATA="$ac_pt_LLVM_PROFDATA" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in ${llvm_path}
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_ac_pt_LLVM_PROFDATA="$as_dir/$ac_word$ac_exec_ext"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_path_ac_pt_LLVM_PROFDATA" && ac_cv_path_ac_pt_LLVM_PROFDATA="''"
+  ;;
+esac
 fi
-LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
-if test -n "$LLVM_PROF_FOUND"; then
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
-$as_echo "$LLVM_PROF_FOUND" >&6; }
+ac_pt_LLVM_PROFDATA=$ac_cv_path_ac_pt_LLVM_PROFDATA
+if test -n "$ac_pt_LLVM_PROFDATA"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_LLVM_PROFDATA" >&5
+$as_echo "$ac_pt_LLVM_PROFDATA" >&6; }
 else
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
 $as_echo "no" >&6; }
 fi
 
+    LLVM_PROFDATA=$ac_pt_LLVM_PROFDATA
+  else
+    LLVM_PROFDATA="''"
+  fi
+else
+  LLVM_PROFDATA="$ac_cv_path_LLVM_PROFDATA"
+fi
 
+
+if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}"
+then
+  LLVM_PROF_FOUND="found"
+else
+  LLVM_PROF_FOUND="not-found"
+fi
+if test "$ac_sys_system" = "Darwin" -a "${LLVM_PROF_FOUND}" = "not-found"
+then
+  found_llvm_profdata=`/usr/bin/xcrun -find llvm-profdata 2>/dev/null`
+  if test -n "${found_llvm_profdata}"
+  then
+    # llvm-profdata isn't directly in $PATH in some cases.
+    # https://apple.stackexchange.com/questions/197053/
+    LLVM_PROFDATA='/usr/bin/xcrun llvm-profdata'
+    LLVM_PROF_FOUND=found
+    { $as_echo "$as_me:${as_lineno-$LINENO}: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&5
+$as_echo "$as_me: llvm-profdata found via xcrun: ${LLVM_PROFDATA}" >&6;}
+  fi
+fi
 LLVM_PROF_ERR=no
 case $CC in
   *clang*)
     # Any changes made here should be reflected in the GCC+Darwin case below
     PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
     PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
-    LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+    LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr"
     LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
     if test $LLVM_PROF_FOUND = not-found
     then
       LLVM_PROF_ERR=yes
+      if test "${REQUIRE_PGO}" = "yes"
+      then
+        as_fn_error $? "llvm-profdata is required for a --with-optimizations build but could not be found." "$LINENO" 5
+      fi
     fi
     ;;
   *gcc*)
@@ -6615,11 +6757,15 @@ case $CC in
       Darwin*)
         PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
         PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
-        LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+        LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr"
         LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
-        if test $LLVM_PROF_FOUND = not-found
+        if test "${LLVM_PROF_FOUND}" = "not-found"
         then
           LLVM_PROF_ERR=yes
+          if test "${REQUIRE_PGO}" = "yes"
+	  then
+	    as_fn_error $? "llvm-profdata is required for a --with-optimizations build but could not be found." "$LINENO" 5
+	  fi
         fi
         ;;
       *)
diff --git a/configure.ac b/configure.ac
index cd598f0..8a87945 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1307,9 +1307,11 @@ if test "$Py_OPT" = 'true' ; then
       ;;
     esac
   DEF_MAKE_ALL_RULE="profile-opt"
+  REQUIRE_PGO="yes"
   DEF_MAKE_RULE="build_all"
 else
   DEF_MAKE_ALL_RULE="build_all"
+  REQUIRE_PGO="no"
   DEF_MAKE_RULE="all"
 fi
 
@@ -1359,19 +1361,60 @@ AC_SUBST(PGO_PROF_USE_FLAG)
 AC_SUBST(LLVM_PROF_MERGER)
 AC_SUBST(LLVM_PROF_FILE)
 AC_SUBST(LLVM_PROF_ERR)
+# Make this work on systems where llvm tools are not installed with their
+# normal names in the default $PATH (ie: Ubuntu).  They exist under the
+# non-suffixed name in their versioned llvm directory.
+llvm_bin_dir=''
+llvm_path="${PATH}"
+if test "${CC}" = "clang"
+then
+  clang_bin=`which clang`
+  # Some systems install clang elsewhere as a symlink to the real path
+  # which is where the related llvm tools are located.
+  if test -L "${clang_bin}"
+  then
+    clang_dir=`dirname "${clang_bin}"`
+    clang_bin=`readlink "${clang_bin}"`
+    llvm_bin_dir="${clang_dir}/"`dirname "${clang_bin}"`
+    llvm_path="${llvm_path}${PATH_SEPARATOR}${llvm_bin_dir}"
+  fi
+fi
+AC_SUBST(LLVM_PROFDATA)
+AC_PATH_TARGET_TOOL(LLVM_PROFDATA, llvm-profdata, '', ${llvm_path})
 AC_SUBST(LLVM_PROF_FOUND)
-AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
+if test -n "${LLVM_PROFDATA}" -a -x "${LLVM_PROFDATA}"
+then
+  LLVM_PROF_FOUND="found"
+else
+  LLVM_PROF_FOUND="not-found"
+fi
+if test "$ac_sys_system" = "Darwin" -a "${LLVM_PROF_FOUND}" = "not-found"
+then
+  found_llvm_profdata=`/usr/bin/xcrun -find llvm-profdata 2>/dev/null`
+  if test -n "${found_llvm_profdata}"
+  then
+    # llvm-profdata isn't directly in $PATH in some cases.
+    # https://apple.stackexchange.com/questions/197053/
+    LLVM_PROFDATA='/usr/bin/xcrun llvm-profdata'
+    LLVM_PROF_FOUND=found
+    AC_MSG_NOTICE([llvm-profdata found via xcrun: ${LLVM_PROFDATA}])
+  fi
+fi
 LLVM_PROF_ERR=no
 case $CC in
   *clang*)
     # Any changes made here should be reflected in the GCC+Darwin case below
     PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
     PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
-    LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+    LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr"
     LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
     if test $LLVM_PROF_FOUND = not-found
     then
       LLVM_PROF_ERR=yes
+      if test "${REQUIRE_PGO}" = "yes"
+      then
+        AC_MSG_ERROR([llvm-profdata is required for a --with-optimizations build but could not be found.])
+      fi
     fi
     ;;
   *gcc*)
@@ -1379,11 +1422,15 @@ case $CC in
       Darwin*)
         PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
         PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
-        LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+        LLVM_PROF_MERGER="${LLVM_PROFDATA} merge -output=code.profclangd *.profclangr"
         LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
-        if test $LLVM_PROF_FOUND = not-found
+        if test "${LLVM_PROF_FOUND}" = "not-found"
         then
           LLVM_PROF_ERR=yes
+          if test "${REQUIRE_PGO}" = "yes"
+	  then
+	    AC_MSG_ERROR([llvm-profdata is required for a --with-optimizations build but could not be found.])
+	  fi
         fi
         ;;
       *)