diff options
| -rw-r--r-- | Lib/distutils/command/build.py | 9 | ||||
| -rw-r--r-- | Lib/distutils/command/build_ext.py | 57 | ||||
| -rw-r--r-- | Lib/distutils/tests/test_build_ext.py | 56 | ||||
| -rw-r--r-- | Misc/NEWS | 3 | ||||
| -rw-r--r-- | Modules/Setup.dist | 1 | ||||
| -rw-r--r-- | setup.py | 19 |
6 files changed, 117 insertions, 28 deletions
diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py index cfc15cf..337dd0b 100644 --- a/Lib/distutils/command/build.py +++ b/Lib/distutils/command/build.py @@ -36,6 +36,8 @@ class build(Command): "(default: %s)" % get_platform()), ('compiler=', 'c', "specify the compiler type"), + ('parallel=', 'j', + "number of parallel build jobs"), ('debug', 'g', "compile extensions and libraries with debugging information"), ('force', 'f', @@ -65,6 +67,7 @@ class build(Command): self.debug = None self.force = 0 self.executable = None + self.parallel = None def finalize_options(self): if self.plat_name is None: @@ -116,6 +119,12 @@ class build(Command): if self.executable is None: self.executable = os.path.normpath(sys.executable) + if isinstance(self.parallel, str): + try: + self.parallel = int(self.parallel) + except ValueError: + raise DistutilsOptionError("parallel should be an integer") + def run(self): # Run all relevant sub-commands. This will be some subset of: # - build_py - pure Python modules diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 3ab2d04..08449e1 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -4,7 +4,10 @@ Implements the Distutils 'build_ext' command, for building extension modules (currently limited to C extensions, should accommodate C++ extensions ASAP).""" -import sys, os, re +import contextlib +import os +import re +import sys from distutils.core import Command from distutils.errors import * from distutils.sysconfig import customize_compiler, get_python_version @@ -85,6 +88,8 @@ class build_ext(Command): "forcibly build everything (ignore file timestamps)"), ('compiler=', 'c', "specify the compiler type"), + ('parallel=', 'j', + "number of parallel build jobs"), ('swig-cpp', None, "make SWIG create C++ files (default is C)"), ('swig-opts=', None, @@ -124,6 +129,7 @@ class build_ext(Command): self.swig_cpp = None self.swig_opts = None self.user = None + self.parallel = None def finalize_options(self): from distutils import sysconfig @@ -134,6 +140,7 @@ class build_ext(Command): ('compiler', 'compiler'), ('debug', 'debug'), ('force', 'force'), + ('parallel', 'parallel'), ('plat_name', 'plat_name'), ) @@ -274,6 +281,12 @@ class build_ext(Command): self.library_dirs.append(user_lib) self.rpath.append(user_lib) + if isinstance(self.parallel, str): + try: + self.parallel = int(self.parallel) + except ValueError: + raise DistutilsOptionError("parallel should be an integer") + def run(self): from distutils.ccompiler import new_compiler @@ -442,15 +455,45 @@ class build_ext(Command): def build_extensions(self): # First, sanity-check the 'extensions' list self.check_extensions_list(self.extensions) + if self.parallel: + self._build_extensions_parallel() + else: + self._build_extensions_serial() + + def _build_extensions_parallel(self): + workers = self.parallel + if self.parallel is True: + workers = os.cpu_count() # may return None + try: + from concurrent.futures import ThreadPoolExecutor + except ImportError: + workers = None + + if workers is None: + self._build_extensions_serial() + return + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(self.build_extension, ext) + for ext in self.extensions] + for ext, fut in zip(self.extensions, futures): + with self._filter_build_errors(ext): + fut.result() + + def _build_extensions_serial(self): for ext in self.extensions: - try: + with self._filter_build_errors(ext): self.build_extension(ext) - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) + + @contextlib.contextmanager + def _filter_build_errors(self, ext): + try: + yield + except (CCompilerError, DistutilsError, CompileError) as e: + if not ext.optional: + raise + self.warn('building extension "%s" failed: %s' % + (ext.name, e)) def build_extension(self, ext): sources = ext.sources diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index b9f407f..366ffbe 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -37,6 +37,9 @@ class BuildExtTestCase(TempdirManager, from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + def build_ext(self, *args, **kwargs): + return build_ext(*args, **kwargs) + def test_build_ext(self): global ALREADY_TESTED copy_xxmodule_c(self.tmp_dir) @@ -44,7 +47,7 @@ class BuildExtTestCase(TempdirManager, xx_ext = Extension('xx', [xx_c]) dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]}) dist.package_dir = self.tmp_dir - cmd = build_ext(dist) + cmd = self.build_ext(dist) fixup_build_ext(cmd) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -91,7 +94,7 @@ class BuildExtTestCase(TempdirManager, def test_solaris_enable_shared(self): dist = Distribution({'name': 'xx'}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) old = sys.platform sys.platform = 'sunos' # fooling finalize_options @@ -113,7 +116,7 @@ class BuildExtTestCase(TempdirManager, def test_user_site(self): import site dist = Distribution({'name': 'xx'}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) # making sure the user option is there options = [name for name, short, lable in @@ -144,14 +147,14 @@ class BuildExtTestCase(TempdirManager, # with the optional argument. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() self.assertRaises((UnknownFileError, CompileError), cmd.run) # should raise an error modules = [Extension('foo', ['xxx'], optional=True)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() cmd.run() # should pass @@ -160,7 +163,7 @@ class BuildExtTestCase(TempdirManager, # etc.) are in the include search path. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.finalize_options() from distutils import sysconfig @@ -172,14 +175,14 @@ class BuildExtTestCase(TempdirManager, # make sure cmd.libraries is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.libraries = 'my_lib, other_lib lastlib' cmd.finalize_options() self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib']) # make sure cmd.library_dirs is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep cmd.finalize_options() self.assertIn('my_lib_dir', cmd.library_dirs) @@ -187,7 +190,7 @@ class BuildExtTestCase(TempdirManager, # make sure rpath is turned into a list # if it's a string - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.rpath = 'one%stwo' % os.pathsep cmd.finalize_options() self.assertEqual(cmd.rpath, ['one', 'two']) @@ -196,32 +199,32 @@ class BuildExtTestCase(TempdirManager, # make sure define is turned into 2-tuples # strings if they are ','-separated strings - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.define = 'one,two' cmd.finalize_options() self.assertEqual(cmd.define, [('one', '1'), ('two', '1')]) # make sure undef is turned into a list of # strings if they are ','-separated strings - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.undef = 'one,two' cmd.finalize_options() self.assertEqual(cmd.undef, ['one', 'two']) # make sure swig_opts is turned into a list - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.swig_opts = None cmd.finalize_options() self.assertEqual(cmd.swig_opts, []) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.swig_opts = '1 2' cmd.finalize_options() self.assertEqual(cmd.swig_opts, ['1', '2']) def test_check_extensions_list(self): dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.finalize_options() #'extensions' option must be a list of Extension instances @@ -270,7 +273,7 @@ class BuildExtTestCase(TempdirManager, def test_get_source_files(self): modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.ensure_finalized() self.assertEqual(cmd.get_source_files(), ['xxx']) @@ -279,7 +282,7 @@ class BuildExtTestCase(TempdirManager, # should not be overriden by a compiler instance # when the command is run dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.compiler = 'unix' cmd.ensure_finalized() cmd.run() @@ -292,7 +295,7 @@ class BuildExtTestCase(TempdirManager, ext = Extension('foo', [c_file], optional=False) dist = Distribution({'name': 'xx', 'ext_modules': [ext]}) - cmd = build_ext(dist) + cmd = self.build_ext(dist) fixup_build_ext(cmd) cmd.ensure_finalized() self.assertEqual(len(cmd.get_outputs()), 1) @@ -355,7 +358,7 @@ class BuildExtTestCase(TempdirManager, #etree_ext = Extension('lxml.etree', [etree_c]) #dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]}) dist = Distribution() - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.inplace = 1 cmd.distribution.package_dir = {'': 'src'} cmd.distribution.packages = ['lxml', 'lxml.html'] @@ -462,7 +465,7 @@ class BuildExtTestCase(TempdirManager, 'ext_modules': [deptarget_ext] }) dist.package_dir = self.tmp_dir - cmd = build_ext(dist) + cmd = self.build_ext(dist) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -481,8 +484,19 @@ class BuildExtTestCase(TempdirManager, self.fail("Wrong deployment target during compilation") +class ParallelBuildExtTestCase(BuildExtTestCase): + + def build_ext(self, *args, **kwargs): + build_ext = super().build_ext(*args, **kwargs) + build_ext.parallel = True + return build_ext + + def test_suite(): - return unittest.makeSuite(BuildExtTestCase) + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(BuildExtTestCase)) + suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase)) + return suite if __name__ == '__main__': - support.run_unittest(test_suite()) + support.run_unittest(__name__) @@ -139,6 +139,9 @@ Core and Builtins Library ------- +- Issue #5309: distutils' build and build_ext commands now accept a ``-j`` + option to enable parallel building of extension modules. + - Issue #22448: Improve canceled timer handles cleanup to prevent unbound memory usage. Patch by Joshua Moore-Oliva. diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 01fb85f..ee84df4 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -118,6 +118,7 @@ _collections _collectionsmodule.c # Container types itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _stat _stat.c # stat.h interface +time timemodule.c # time module # access to ISO C locale support _locale _localemodule.c # -lintl @@ -25,6 +25,11 @@ cflags = sysconfig.get_config_var('CFLAGS') py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist +class Dummy: + """Hack for parallel build""" + ProcessPoolExecutor = None +sys.modules['concurrent.futures.process'] = Dummy + def get_platform(): # cross build if "_PYTHON_HOST_PLATFORM" in os.environ: @@ -174,6 +179,8 @@ class PyBuildExt(build_ext): build_ext.__init__(self, dist) self.failed = [] self.failed_on_import = [] + if '-j' in os.environ.get('MAKEFLAGS', ''): + self.parallel = True def build_extensions(self): @@ -253,6 +260,9 @@ class PyBuildExt(build_ext): build_ext.build_extensions(self) + for ext in self.extensions: + self.check_extension_import(ext) + longest = max([len(e.name) for e in self.extensions]) if self.failed or self.failed_on_import: all_failed = self.failed + self.failed_on_import @@ -305,6 +315,15 @@ class PyBuildExt(build_ext): (ext.name, sys.exc_info()[1])) self.failed.append(ext.name) return + + def check_extension_import(self, ext): + # Don't try to import an extension that has failed to compile + if ext.name in self.failed: + self.announce( + 'WARNING: skipping import check for failed build "%s"' % + ext.name, level=1) + return + # Workaround for Mac OS X: The Carbon-based modules cannot be # reliably imported into a command-line Python if 'Carbon' in ext.extra_link_args: |