Fix tests to not hang on Windows with bad .py assoc

For systems where the association for .py files is not to an actual
Python interpreter, those few cases where we need to run a Python script
directly as a program don't work.  This could be because the association
was never set up, or because some other program (e.g. Visual Studio Code)
has taken it over. In some cases may appear to "hang" because the
alternate program is waiting for user interaction

runtest.py now has a mechanism to check (thanks to Brett Cannon for
providing this incantation). It isn't super precise (looks for the
substring "py" in the queried association), but should work out.
It sets an environment variable which the test framework can read
and as a result set a flag which individual tests can read.

Two tests in scons-time which had previously been set to skip-if-win32
now look at this flag instead.  Three tests in sconsign now also look at
this flag. This allows a clean run on my dev box with VS Code having
taken over the .py association.

Various things can break if the environment used to fire off
Windows processes doesn't contain %UserProfile%. Added this to the
short list of passthrough env vars.  Apparently an environment without
this value is now considered invalid (it blew up the erroneously
launched VS Code, but we've apparently been lucky it hasn't blown
up more things - believe there was also a report of a problem
with the Visual Studio setup scripts).

A little extra cleanup:
- a couple of Py2-isms were cleaned out (Script/Main.py and in the test
  framework)
- The paths to look for site-scons were rewritten (part of this was
  another Py2-ism), and the system path changed a bit - the old path is
  still checked, and the manpage updated to reflect this.
- runtest.py dropped the unused whereis functions.
- the three sconsign tests now use f-string formatting, mostly as an
  experiment to see how easy it is to convert.

Fixes #4053

Signed-off-by: Mats Wichmann <mats@linux.com>

3 files changed, 92 insertions, 67 deletions

diff --git a/test/sconsign/script/SConsignFile.py b/test/sconsign/script/SConsignFile.py
index 4e29ef0..569e5e6 100644
--- a/test/sconsign/script/SConsignFile.py
+++ b/test/sconsign/script/SConsignFile.py
@@ -31,15 +31,20 @@ using the signatures in an SConsignFile().
 import TestSCons
 import TestSConsign
 
-_python_ = TestSCons._python_
+from TestSCons import _python_
+from TestCmd import NEED_HELPER
 
 test = TestSConsign.TestSConsign(match = TestSConsign.match_re)
+
+if NEED_HELPER:
+    test.skip_test("Test host cannot directly execute scripts, skipping test\n")
+
 test.subdir('sub1', 'sub2')
 
 fake_cc_py = test.workpath('fake_cc.py')
 fake_link_py = test.workpath('fake_link.py')
 
-test.write(fake_cc_py, r"""#!%(_python_)s
+test.write(fake_cc_py, fr"""#!{_python_}
 import os
 import re
 import sys
@@ -66,21 +71,23 @@ def process(infp, outfp):
             outfp.write(line)
 
 with open(sys.argv[2], 'w') as outf, open(sys.argv[3], 'r') as ifp:
-    outf.write('fake_cc.py:  %%s\n' %% sys.argv)
+    outf.write('fake_cc.py:  %s\n' % sys.argv)
     process(ifp, outf)
 
 sys.exit(0)
-""" % locals())
+"""
+)
 
-test.write(fake_link_py, r"""#!%(_python_)s
+test.write(fake_link_py, fr"""#!{_python_}
 import sys
 
 with open(sys.argv[1], 'w') as outf, open(sys.argv[2], 'r') as ifp:
-    outf.write('fake_link.py:  %%s\n' %% sys.argv)
+    outf.write('fake_link.py:  %s\n' % sys.argv)
     outf.write(ifp.read())
 
 sys.exit(0)
-""" % locals())
+"""
+)
 
 test.chmod(fake_cc_py, 0o755)
 test.chmod(fake_link_py, 0o755)
@@ -96,24 +103,27 @@ sub2_hello_obj  = 'sub2/hello.obj'
 sub2_inc1_h     = 'sub2/inc1.h'
 sub2_inc2_h     = 'sub2/inc2.h'
 
-test.write(['SConstruct'], """\
+test.write(['SConstruct'], f"""\
 SConsignFile()
 env1 = Environment(
     PROGSUFFIX='.exe',
     OBJSUFFIX='.obj',
-    CCCOM=[[r'%(fake_cc_py)s', 'sub2', '$TARGET', '$SOURCE']],
-    LINKCOM=[[r'%(fake_link_py)s', '$TARGET', '$SOURCE']],
+    CCCOM=[[r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']],
+    LINKCOM=[[r'{fake_link_py}', '$TARGET', '$SOURCE']],
 )
 env1.PrependENVPath('PATHEXT', '.PY')
 env1.Program('sub1/hello.c')
 env2 = env1.Clone(CPPPATH=['sub2'])
 env2.Program('sub2/hello.c')
-""" % locals())
+"""
+)
 # TODO in the above, we would normally want to run a python program
-# using "our python" as in:
-#    CCCOM=[[r'%(_python_)s', r'%(fake_cc_py)s', 'sub2', '$TARGET', '$SOURCE']],
-#    LINKCOM=[[r'%(_python_)s', r'%(fake_link_py)s', '$TARGET', '$SOURCE']],
-# however we're looking at dependencies with sconsign, so that breaks things
+# using "our python" like this:
+#    CCCOM=[[r'{_python_}', r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']],
+#    LINKCOM=[[r'{_python_}', r'{fake_link_py}', '$TARGET', '$SOURCE']],
+# however we're looking at dependencies with sconsign, so that breaks things.
+# It still breaks things on Windows if something else is registered as the
+# handler for .py files, as Visual Studio Code installs itself.
 
 test.write(['sub1', 'hello.c'], r"""
 sub1/hello.c
@@ -131,11 +141,11 @@ main(int argc, char *argv[])
 }
 """)
 
-test.write(['sub2', 'inc1.h'], r"""\
+test.write(['sub2', 'inc1.h'], r"""
 #define STRING1 "inc1.h"
 """)
 
-test.write(['sub2', 'inc2.h'], r"""\
+test.write(['sub2', 'inc2.h'], r"""
 #define STRING2 "inc2.h"
 """)
 
diff --git a/test/sconsign/script/Signatures.py b/test/sconsign/script/Signatures.py
index ced5b44..b5db3f9 100644
--- a/test/sconsign/script/Signatures.py
+++ b/test/sconsign/script/Signatures.py
@@ -32,10 +32,14 @@ value of Decider('timestamp-newer').
 import TestSCons
 import TestSConsign
 
-_python_ = TestSCons._python_
+from TestSCons import _python_
+from TestCmd import NEED_HELPER
 
 test = TestSConsign.TestSConsign(match = TestSConsign.match_re)
 
+if NEED_HELPER:
+    test.skip_test("Test host cannot directly execute scripts, skipping test\n")
+
 # Note:  We don't use os.path.join() representations of the file names
 # in the expected output because paths in the .sconsign files are
 # canonicalized to use / as the separator.
@@ -58,7 +62,7 @@ test.subdir('sub1', 'sub2')
 fake_cc_py = test.workpath('fake_cc.py')
 fake_link_py = test.workpath('fake_link.py')
 
-test.write(fake_cc_py, r"""#!%(_python_)s
+test.write(fake_cc_py, fr"""#!{_python_}
 import os
 import re
 import sys
@@ -85,26 +89,28 @@ def process(infp, outfp):
             outfp.write(line)
 
 with open(sys.argv[2], 'w') as outf, open(sys.argv[3], 'r') as ifp:
-    outf.write('fake_cc.py:  %%s\n' %% sys.argv)
+    outf.write('fake_cc.py:  %s\n' % sys.argv)
     process(ifp, outf)
 
 sys.exit(0)
-""" % locals())
+"""
+)
 
-test.write(fake_link_py, r"""#!%(_python_)s
+test.write(fake_link_py, fr"""#!{_python_}
 import sys
 
 with open(sys.argv[1], 'w') as outf, open(sys.argv[2], 'r') as ifp:
-    outf.write('fake_link.py:  %%s\n' %% sys.argv)
+    outf.write('fake_link.py:  %s\n' % sys.argv)
     outf.write(ifp.read())
 
 sys.exit(0)
-""" % locals())
+"""
+)
 
 test.chmod(fake_cc_py, 0o755)
 test.chmod(fake_link_py, 0o755)
 
-test.write('SConstruct', """
+test.write('SConstruct', f"""
 SConsignFile(None)
 Decider('timestamp-newer')
 env1 = Environment(
@@ -113,19 +119,22 @@ env1 = Environment(
     # Specify the command lines with lists-of-lists so
     # finding the implicit dependencies works even with
     # spaces in the fake_*_py path names.
-    CCCOM=[[r'%(fake_cc_py)s', 'sub2', '$TARGET', '$SOURCE']],
-    LINKCOM=[[r'%(fake_link_py)s', '$TARGET', '$SOURCE']],
+    CCCOM=[[r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']],
+    LINKCOM=[[r'{fake_link_py}', '$TARGET', '$SOURCE']],
 )
 env1.PrependENVPath('PATHEXT', '.PY')
 env1.Program('sub1/hello.c')
 env2 = env1.Clone(CPPPATH=['sub2'])
 env2.Program('sub2/hello.c')
-""" % locals())
+"""
+)
 # TODO in the above, we would normally want to run a python program
-# using "our python" as in:
-#    CCCOM=[[r'%(_python_)s', r'%(fake_cc_py)s', 'sub2', '$TARGET', '$SOURCE']],
-#    LINKCOM=[[r'%(_python_)s', r'%(fake_link_py)s', '$TARGET', '$SOURCE']],
-# however we're looking at dependencies with sconsign, so that breaks things
+# using "our python" like this:
+#    CCCOM=[[r'{_python_}', r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']],
+#    LINKCOM=[[r'{_python_}', r'{fake_link_py}', '$TARGET', '$SOURCE']],
+# however we're looking at dependencies with sconsign, so that breaks things.
+# It still breaks things on Windows if something else is registered as the
+# handler for .py files, as Visual Studio Code installs itself.
 
 test.write(['sub1', 'hello.c'], r"""\
 sub1/hello.c
@@ -154,7 +163,7 @@ date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d'
 
 database_name = test.get_sconsignname()
 
-test.run_sconsign(arguments = "-e hello.exe -e hello.obj sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-e hello.exe -e hello.obj sub1/{database_name}",
          stdout = r"""hello.exe: %(sig_re)s \d+ \d+
         %(sub1_hello_obj)s: %(sig_re)s \d+ \d+
         fake_link\.py: None \d+ \d+
@@ -165,7 +174,7 @@ hello.obj: %(sig_re)s \d+ \d+
         %(sig_re)s \[.*\]
 """ % locals())
 
-test.run_sconsign(arguments = "-e hello.exe -e hello.obj -r sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-e hello.exe -e hello.obj -r sub1/{database_name}",
          stdout = r"""hello.exe: %(sig_re)s '%(date_re)s' \d+
         %(sub1_hello_obj)s: %(sig_re)s '%(date_re)s' \d+
         fake_link\.py: None '%(date_re)s' \d+
diff --git a/test/sconsign/script/no-SConsignFile.py b/test/sconsign/script/no-SConsignFile.py
index bdd878b..f06a2d0 100644
--- a/test/sconsign/script/no-SConsignFile.py
+++ b/test/sconsign/script/no-SConsignFile.py
@@ -22,9 +22,6 @@
 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 """
 Verify that the sconsign script works when using an individual
@@ -34,10 +31,14 @@ Verify that the sconsign script works when using an individual
 import TestSCons
 import TestSConsign
 
-_python_ = TestSCons._python_
+from TestSCons import _python_
+from TestCmd import NEED_HELPER
 
 test = TestSConsign.TestSConsign(match = TestSConsign.match_re)
 
+if NEED_HELPER:
+    test.skip_test("Test host cannot directly execute scripts, skipping test\n")
+
 test.subdir('sub1', 'sub2')
 
 database_name = test.get_sconsignname()
@@ -55,7 +56,7 @@ database_name = test.get_sconsignname()
 fake_cc_py = test.workpath('fake_cc.py')
 fake_link_py = test.workpath('fake_link.py')
 
-test.write(fake_cc_py, r"""#!%(_python_)s
+test.write(fake_cc_py, fr"""#!{_python_}
 import os
 import re
 import sys
@@ -82,21 +83,23 @@ def process(infp, outfp):
             outfp.write(line)
 
 with open(sys.argv[2], 'w') as outf, open(sys.argv[3], 'r') as ifp:
-    outf.write('fake_cc.py:  %%s\n' %% sys.argv)
+    outf.write('fake_cc.py:  %s\n' % sys.argv)
     process(ifp, outf)
 
 sys.exit(0)
-""" % locals())
+"""
+)
 
-test.write(fake_link_py, r"""#!%(_python_)s
+test.write(fake_link_py, fr"""#!{_python_}
 import sys
 
 with open(sys.argv[1], 'w') as outf, open(sys.argv[2], 'r') as ifp:
-    outf.write('fake_link.py:  %%s\n' %% sys.argv)
+    outf.write('fake_link.py:  %s\n' % sys.argv)
     outf.write(ifp.read())
 
 sys.exit(0)
-""" % locals())
+"""
+)
 
 test.chmod(fake_cc_py, 0o755)
 test.chmod(fake_link_py, 0o755)
@@ -112,7 +115,7 @@ sub2_hello_obj  = 'sub2/hello.obj'
 sub2_inc1_h     = 'sub2/inc1.h'
 sub2_inc2_h     = 'sub2/inc2.h'
 
-test.write(['SConstruct'], """
+test.write(['SConstruct'], f"""
 SConsignFile(None)
 env1 = Environment(
     PROGSUFFIX='.exe',
@@ -120,35 +123,38 @@ env1 = Environment(
     # Specify the command lines with lists-of-lists so
     # finding the implicit dependencies works even with
     # spaces in the fake_*_py path names.
-    CCCOM=[[r'%(fake_cc_py)s', 'sub2', '$TARGET', '$SOURCE']],
-    LINKCOM=[[r'%(fake_link_py)s', '$TARGET', '$SOURCE']],
+    CCCOM=[[r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']],
+    LINKCOM=[[r'{fake_link_py}', '$TARGET', '$SOURCE']],
 )
 env1.PrependENVPath('PATHEXT', '.PY')
 env1.Program('sub1/hello.c')
 env2 = env1.Clone(CPPPATH=['sub2'])
 env2.Program('sub2/hello.c')
-""" % locals())
+"""
+)
 # TODO in the above, we would normally want to run a python program
-# using "our python" as in:
-#    CCCOM=[[r'%(_python_)s', r'%(fake_cc_py)s', 'sub2', '$TARGET', '$SOURCE']],
-#    LINKCOM=[[r'%(_python_)s', r'%(fake_link_py)s', '$TARGET', '$SOURCE']],
-# however we're looking at dependencies with sconsign, so that breaks things
-
-test.write(['sub1', 'hello.c'], r"""\
+# using "our python" like this:
+#    CCCOM=[[r'{_python_}', r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']],
+#    LINKCOM=[[r'{_python_}', r'{fake_link_py}', '$TARGET', '$SOURCE']],
+# however we're looking at dependencies with sconsign, so that breaks things.
+# It still breaks things on Windows if something else is registered as the
+# handler for .py files, as Visual Studio Code installs itself.
+
+test.write(['sub1', 'hello.c'], r"""
 sub1/hello.c
 """)
 
-test.write(['sub2', 'hello.c'], r"""\
+test.write(['sub2', 'hello.c'], r"""
 #include <inc1.h>
 #include <inc2.h>
 sub2/hello.c
 """)
 
-test.write(['sub2', 'inc1.h'], r"""\
+test.write(['sub2', 'inc1.h'], r"""
 #define STRING1 "inc1.h"
 """)
 
-test.write(['sub2', 'inc2.h'], r"""\
+test.write(['sub2', 'inc2.h'], r"""
 #define STRING2 "inc2.h"
 """)
 
@@ -167,9 +173,9 @@ hello.obj: %(sig_re)s \d+ \d+
         %(sig_re)s \[.*\]
 """ % locals()
 
-test.run_sconsign(arguments = "sub1/{}".format(database_name), stdout=expect)
+test.run_sconsign(arguments = f"sub1/{database_name}", stdout=expect)
 
-test.run_sconsign(arguments = "--raw sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"--raw sub1/{database_name}",
          stdout = r"""hello.c: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2}
 hello.exe: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2}
         %(sub1_hello_obj)s: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_id': 2}
@@ -181,7 +187,7 @@ hello.obj: {'csig': '%(sig_re)s', 'timestamp': \d+L?, 'size': \d+L?, '_version_i
         %(sig_re)s \[.*\]
 """ % locals())
 
-test.run_sconsign(arguments = "-v sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-v sub1/{database_name}",
          stdout = r"""hello.c:
     csig: %(sig_re)s
     timestamp: \d+
@@ -216,7 +222,7 @@ hello.obj:
     action: %(sig_re)s \[.*\]
 """ % locals())
 
-test.run_sconsign(arguments = "-c -v sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-c -v sub1/{database_name}",
          stdout = r"""hello.c:
     csig: %(sig_re)s
 hello.exe:
@@ -225,7 +231,7 @@ hello.obj:
     csig: %(sig_re)s
 """ % locals())
 
-test.run_sconsign(arguments = "-s -v sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-s -v sub1/{database_name}",
          stdout = r"""hello.c:
     size: \d+
 hello.exe:
@@ -234,7 +240,7 @@ hello.obj:
     size: \d+
 """ % locals())
 
-test.run_sconsign(arguments = "-t -v sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-t -v sub1/{database_name}",
          stdout = r"""hello.c:
     timestamp: \d+
 hello.exe:
@@ -243,14 +249,14 @@ hello.obj:
     timestamp: \d+
 """ % locals())
 
-test.run_sconsign(arguments = "-e hello.obj sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-e hello.obj sub1/{database_name}",
          stdout = r"""hello.obj: %(sig_re)s \d+ \d+
         %(sub1_hello_c)s: %(sig_re)s \d+ \d+
         fake_cc\.py: %(sig_re)s \d+ \d+
         %(sig_re)s \[.*\]
 """ % locals())
 
-test.run_sconsign(arguments = "-e hello.obj -e hello.exe -e hello.obj sub1/{}".format(database_name),
+test.run_sconsign(arguments = f"-e hello.obj -e hello.exe -e hello.obj sub1/{database_name}",
          stdout = r"""hello.obj: %(sig_re)s \d+ \d+
         %(sub1_hello_c)s: %(sig_re)s \d+ \d+
         fake_cc\.py: %(sig_re)s \d+ \d+
@@ -265,7 +271,7 @@ hello.obj: %(sig_re)s \d+ \d+
         %(sig_re)s \[.*\]
 """ % locals())
 
-test.run_sconsign(arguments = "sub2/{}".format(database_name),
+test.run_sconsign(arguments = f"sub2/{database_name}",
          stdout = r"""hello.c: %(sig_re)s \d+ \d+
 hello.exe: %(sig_re)s \d+ \d+
         %(sub2_hello_obj)s: %(sig_re)s \d+ \d+
@@ -293,7 +299,7 @@ inc2.h: %(sig_re)s \d+ \d+
 #        inc2.h: %(sig_re)s \d+ \d+
 #""" % locals())
 
-test.run_sconsign(arguments = "-e hello.obj sub2/{} sub1/{}".format(database_name, database_name),
+test.run_sconsign(arguments = f"-e hello.obj sub2/{database_name} sub1/{database_name}",
          stdout = r"""hello.obj: %(sig_re)s \d+ \d+
         %(sub2_hello_c)s: %(sig_re)s \d+ \d+
         %(sub2_inc1_h)s: %(sig_re)s \d+ \d+