bpo-36044: Reduce number of unit tests run for PGO build (GH-14702)

Reduce the number of unit tests run for the PGO generation task.  This
speeds up the task by a factor of about 15x.  Running the full unit test
suite is slow.  This change may result in a slightly less optimized build
since not as many code branches will be executed.  If you are willing to
wait for the much slower build, the old behavior can be restored using
'./configure [..] PROFILE_TASK="-m test --pgo-extended"'.  We make no
guarantees as to which PGO task set produces a faster build.  Users who
care should run their own relevant benchmarks as results can depend on
the environment, workload, and compiler tool chain.

3 files changed, 61 insertions, 1 deletions

diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index 9f1bf68..c8fedc7 100644
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -264,7 +264,9 @@ def _create_parser():
                        help='only write the name of test cases that will be run'
                             ' , don\'t execute them')
     group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
-                       help='enable Profile Guided Optimization training')
+                       help='enable Profile Guided Optimization (PGO) training')
+    group.add_argument('--pgo-extended', action='store_true',
+                       help='enable extended PGO training (slower training)')
     group.add_argument('--fail-env-changed', action='store_true',
                        help='if a test file alters the environment, mark '
                             'the test as failed')
@@ -344,6 +346,8 @@ def _parse_args(args, **kwargs):
         parser.error("-G/--failfast needs either -v or -W")
     if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
         parser.error("--pgo/-v don't go together!")
+    if ns.pgo_extended:
+        ns.pgo = True  # pgo_extended implies pgo
 
     if ns.nowindows:
         print("Warning: the --nowindows (-n) option is deprecated. "
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index e227425..78b6790 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -17,6 +17,7 @@ from test.libregrtest.runtest import (
     INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN,
     PROGRESS_MIN_TIME, format_test_result, is_failed)
 from test.libregrtest.setup import setup_tests
+from test.libregrtest.pgo import setup_pgo_tests
 from test.libregrtest.utils import removepy, count, format_duration, printlist
 from test import support
 
@@ -214,6 +215,9 @@ class Regrtest:
 
         removepy(self.tests)
 
+        # add default PGO tests if no tests are specified
+        setup_pgo_tests(self.ns)
+
         stdtests = STDTESTS[:]
         nottests = NOTTESTS.copy()
         if self.ns.exclude:
diff --git a/Lib/test/libregrtest/pgo.py b/Lib/test/libregrtest/pgo.py
new file mode 100644
index 0000000..327f193
--- /dev/null
+++ b/Lib/test/libregrtest/pgo.py
@@ -0,0 +1,52 @@
+# Set of tests run by default if --pgo is specified.  The tests below were
+# chosen based on the following criteria: either they exercise a commonly used
+# C extension module or type, or they run some relatively typical Python code.
+# Long running tests should be avoided because the PGO instrumented executable
+# runs slowly.
+PGO_TESTS = [
+    'test_array',
+    'test_base64',
+    'test_binascii',
+    'test_binop',
+    'test_bisect',
+    'test_bytes',
+    'test_cmath',
+    'test_codecs',
+    'test_collections',
+    'test_complex',
+    'test_dataclasses',
+    'test_datetime',
+    'test_decimal',
+    'test_difflib',
+    'test_embed',
+    'test_float',
+    'test_fstring',
+    'test_functools',
+    'test_generators',
+    'test_hashlib',
+    'test_heapq',
+    'test_int',
+    'test_itertools',
+    'test_json',
+    'test_long',
+    'test_math',
+    'test_memoryview',
+    'test_operator',
+    'test_ordered_dict',
+    'test_pickle',
+    'test_pprint',
+    'test_re',
+    'test_set',
+    'test_statistics',
+    'test_struct',
+    'test_tabnanny',
+    'test_time',
+    'test_unicode',
+    'test_xml_etree',
+    'test_xml_etree_c',
+]
+
+def setup_pgo_tests(ns):
+    if not ns.args and not ns.pgo_extended:
+        # run default set of tests for PGO training
+        ns.args = PGO_TESTS[:]