path: root/Lib/packaging/command/build_ext.py

"""Build extension modules."""

import os
import re
import sys
import site
import sysconfig

from packaging.util import get_platform
from packaging.command.cmd import Command
from packaging.errors import (CCompilerError, CompileError, PackagingError,
                               PackagingPlatformError, PackagingSetupError)
from packaging.compiler import customize_compiler, show_compilers
from packaging.util import newer_group
from packaging.compiler.extension import Extension
from packaging import logger

if os.name == 'nt':
    from packaging.compiler.msvccompiler import get_build_version
    MSVC_VERSION = int(get_build_version())

# An extension name is just a dot-separated list of Python NAMEs (ie.
# the same as a fully-qualified module name).
extension_name_re = re.compile \
    (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$')


class build_ext(Command):

    description = "build C/C++ extension modules (compile/link to build directory)"

    # XXX thoughts on how to deal with complex command-line options like
    # these, i.e. how to make it so fancy_getopt can suck them off the
    # command line and turn them into the appropriate
    # lists of tuples of what-have-you.
    #   - each command needs a callback to process its command-line options
    #   - Command.__init__() needs access to its share of the whole
    #     command line (must ultimately come from
    #     Distribution.parse_command_line())
    #   - it then calls the current command class' option-parsing
    #     callback to deal with weird options like -D, which have to
    #     parse the option text and churn out some custom data
    #     structure
    #   - that data structure (in this case, a list of 2-tuples)
    #     will then be present in the command object by the time
    #     we get to finalize_options() (i.e. the constructor
    #     takes care of both command-line and client options
    #     in between initialize_options() and finalize_options())

    sep_by = " (separated by '%s')" % os.pathsep
    user_options = [
        ('build-lib=', 'b',
         "directory for compiled extension modules"),
        ('build-temp=', 't',
         "directory for temporary files (build by-products)"),
        ('plat-name=', 'p',
         "platform name to cross-compile for, if supported "
         "(default: %s)" % get_platform()),
        ('inplace', 'i',
         "ignore build-lib and put compiled extensions into the source " +
         "directory alongside your pure Python modules"),
        ('user', None,
         "add user include, library and rpath"),
        ('include-dirs=', 'I',
         "list of directories to search for header files" + sep_by),
        ('define=', 'D',
         "C preprocessor macros to define"),
        ('undef=', 'U',
         "C preprocessor macros to undefine"),
        ('libraries=', 'l',
         "external C libraries to link with"),
        ('library-dirs=', 'L',
         "directories to search for external C libraries" + sep_by),
        ('rpath=', 'R',
         "directories to search for shared C libraries at runtime"),
        ('link-objects=', 'O',
         "extra explicit link objects to include in the link"),
        ('debug', 'g',
         "compile/link with debugging information"),
        ('force', 'f',
         "forcibly build everything (ignore file timestamps)"),
        ('compiler=', 'c',
         "specify the compiler type"),
        ('swig-opts=', None,
         "list of SWIG command-line options"),
        ('swig=', None,
         "path to the SWIG executable"),
        ]

    boolean_options = ['inplace', 'debug', 'force', 'user']


    help_options = [
        ('help-compiler', None,
         "list available compilers", show_compilers),
        ]

    def initialize_options(self):
        self.extensions = None
        self.build_lib = None
        self.plat_name = None
        self.build_temp = None
        self.inplace = False
        self.package = None

        self.include_dirs = None
        self.define = None
        self.undef = None
        self.libraries = None
        self.library_dirs = None
        self.rpath = None
        self.link_objects = None
        self.debug = None
        self.force = None
        self.compiler = None
        self.swig = None
        self.swig_opts = None
        self.user = None

    def finalize_options(self):
        self.set_undefined_options('build',
                                   'build_lib', 'build_temp', 'compiler',
                                   'debug', 'force', 'plat_name')

        if self.package is None:
            self.package = self.distribution.ext_package

        # Ensure that the list of extensions is valid, i.e. it is a list of
        # Extension objects.
        self.extensions = self.distribution.ext_modules
        if self.extensions:
            if not isinstance(self.extensions, (list, tuple)):
                type_name = (self.extensions is None and 'None'
                            or type(self.extensions).__name__)
                raise PackagingSetupError(
                    "'ext_modules' must be a sequence of Extension instances,"
                    " not %s" % (type_name,))
            for i, ext in enumerate(self.extensions):
                if isinstance(ext, Extension):
                    continue                # OK! (assume type-checking done
                                            # by Extension constructor)
                type_name = (ext is None and 'None' or type(ext).__name__)
                raise PackagingSetupError(
                    "'ext_modules' item %d must be an Extension instance,"
                    " not %s" % (i, type_name))

        # Make sure Python's include directories (for Python.h, pyconfig.h,
        # etc.) are in the include search path.
        py_include = sysconfig.get_path('include')
        plat_py_include = sysconfig.get_path('platinclude')
        if self.include_dirs is None:
            self.include_dirs = self.distribution.include_dirs or []
        if isinstance(self.include_dirs, str):
            self.include_dirs = self.include_dirs.split(os.pathsep)

        # Put the Python "system" include dir at the end, so that
        # any local include dirs take precedence.
        self.include_dirs.append(py_include)
        if plat_py_include != py_include:
            self.include_dirs.append(plat_py_include)

        self.ensure_string_list('libraries')

        # Life is easier if we're not forever checking for None, so
        # simplify these options to empty lists if unset
        if self.libraries is None:
            self.libraries = []
        if self.library_dirs is None:
            self.library_dirs = []
        elif isinstance(self.library_dirs, str):
            self.library_dirs = self.library_dirs.split(os.pathsep)

        if self.rpath is None:
            self.rpath = []
        elif isinstance(self.rpath, str):
            self.rpath = self.rpath.split(os.pathsep)

        # for extensions under windows use different directories
        # for Release and Debug builds.
        # also Python's library directory must be appended to library_dirs
        if os.name == 'nt':
            # the 'libs' directory is for binary installs - we assume that
            # must be the *native* platform.  But we don't really support
            # cross-compiling via a binary install anyway, so we let it go.
            self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs'))
            if self.debug:
                self.build_temp = os.path.join(self.build_temp, "Debug")
            else:
                self.build_temp = os.path.join(self.build_temp, "Release")

            # Append the source distribution include and library directories,
            # this allows distutils on windows to work in the source tree
            self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC'))
            if MSVC_VERSION == 9:
                # Use the .lib files for the correct architecture
                if self.plat_name == 'win32':
                    suffix = ''
                else:
                    # win-amd64 or win-ia64
                    suffix = self.plat_name[4:]
                new_lib = os.path.join(sys.exec_prefix, 'PCbuild')
                if suffix:
                    new_lib = os.path.join(new_lib, suffix)
                self.library_dirs.append(new_lib)

            elif MSVC_VERSION == 8:
                self.library_dirs.append(os.path.join(sys.exec_prefix,
                                         'PC', 'VS8.0'))
            elif MSVC_VERSION == 7:
                self.library_dirs.append(os.path.join(sys.exec_prefix,
                                         'PC', 'VS7.1'))
            else:
                self.library_dirs.append(os.path.join(sys.exec_prefix,
                                         'PC', 'VC6'))

        # OS/2 (EMX) doesn't support Debug vs Release builds, but has the
        # import libraries in its "Config" subdirectory
        if os.name == 'os2':
            self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config'))

        # for extensions under Cygwin and AtheOS Python's library directory must be
        # appended to library_dirs
        if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos':
            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
                # building third party extensions
                self.library_dirs.append(os.path.join(sys.prefix, "lib",
                                  "python" + sysconfig.get_python_version(),
                                                      "config"))
            else:
                # building python standard extensions
                self.library_dirs.append(os.curdir)

        # for extensions under Linux or Solaris with a shared Python library,
        # Python's library directory must be appended to library_dirs
        sysconfig.get_config_var('Py_ENABLE_SHARED')
        if (sys.platform.startswith(('linux', 'gnu', 'sunos'))
            and sysconfig.get_config_var('Py_ENABLE_SHARED')):
            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
                # building third party extensions
                self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
            else:
                # building python standard extensions
                self.library_dirs.append(os.curdir)

        # The argument parsing will result in self.define being a string, but
        # it has to be a list of 2-tuples.  All the preprocessor symbols
        # specified by the 'define' option will be set to '1'.  Multiple
        # symbols can be separated with commas.

        if self.define:
            defines = self.define.split(',')
            self.define = [(symbol, '1') for symbol in defines]

        # The option for macros to undefine is also a string from the
        # option parsing, but has to be a list.  Multiple symbols can also
        # be separated with commas here.
        if self.undef:
            self.undef = self.undef.split(',')

        if self.swig_opts is None:
            self.swig_opts = []
        else:
            self.swig_opts = self.swig_opts.split(' ')

        # Finally add the user include and library directories if requested
        if self.user:
            user_include = os.path.join(site.USER_BASE, "include")
            user_lib = os.path.join(site.USER_BASE, "lib")
            if os.path.isdir(user_include):
                self.include_dirs.append(user_include)
            if os.path.isdir(user_lib):
                self.library_dirs.append(user_lib)
                self.rpath.append(user_lib)

    def run(self):
        from packaging.compiler import new_compiler

        if not self.extensions:
            return

        # If we were asked to build any C/C++ libraries, make sure that the
        # directory where we put them is in the library search path for
        # linking extensions.
        if self.distribution.has_c_libraries():
            build_clib = self.get_finalized_command('build_clib')
            self.libraries.extend(build_clib.get_library_names() or [])
            self.library_dirs.append(build_clib.build_clib)

        # Setup the CCompiler object that we'll use to do all the
        # compiling and linking
        self.compiler_obj = new_compiler(compiler=self.compiler,
                                         dry_run=self.dry_run,
                                         force=self.force)

        customize_compiler(self.compiler_obj)
        # If we are cross-compiling, init the compiler now (if we are not
        # cross-compiling, init would not hurt, but people may rely on
        # late initialization of compiler even if they shouldn't...)
        if os.name == 'nt' and self.plat_name != get_platform():
            self.compiler_obj.initialize(self.plat_name)

        # And make sure that any compile/link-related options (which might
        # come from the command line or from the setup script) are set in
        # that CCompiler object -- that way, they automatically apply to
        # all compiling and linking done here.
        if self.include_dirs is not None:
            self.compiler_obj.set_include_dirs(self.include_dirs)
        if self.define is not None:
            # 'define' option is a list of (name,value) tuples
            for name, value in self.define:
                self.compiler_obj.define_macro(name, value)
        if self.undef is not None:
            for macro in self.undef:
                self.compiler_obj.undefine_macro(macro)
        if self.libraries is not None:
            self.compiler_obj.set_libraries(self.libraries)
        if self.library_dirs is not None:
            self.compiler_obj.set_library_dirs(self.library_dirs)
        if self.rpath is not None:
            self.compiler_obj.set_runtime_library_dirs(self.rpath)
        if self.link_objects is not None:
            self.compiler_obj.set_link_objects(self.link_objects)

        # Now actually compile and link everything.
        self.build_extensions()

    def get_source_files(self):
        filenames = []

        # Wouldn't it be neat if we knew the names of header files too...
        for ext in self.extensions:
            filenames.extend(ext.sources)

        return filenames

    def get_outputs(self):
        # And build the list of output (built) filenames.  Note that this
        # ignores the 'inplace' flag, and assumes everything goes in the
        # "build" tree.
        outputs = []
        for ext in self.extensions:
            outputs.append(self.get_ext_fullpath(ext.name))
        return outputs

    def build_extensions(self):
        for ext in self.extensions:
            try:
                self.build_extension(ext)
            except (CCompilerError, PackagingError, CompileError) as e:
                if not ext.optional:
                    raise
                logger.warning('%s: building extension %r failed: %s',
                               self.get_command_name(), ext.name, e)

    def build_extension(self, ext):
        sources = ext.sources
        if sources is None or not isinstance(sources, (list, tuple)):
            raise PackagingSetupError(("in 'ext_modules' option (extension '%s'), " +
                   "'sources' must be present and must be " +
                   "a list of source filenames") % ext.name)
        sources = list(sources)

        ext_path = self.get_ext_fullpath(ext.name)
        depends = sources + ext.depends
        if not (self.force or newer_group(depends, ext_path, 'newer')):
            logger.debug("skipping '%s' extension (up-to-date)", ext.name)
            return
        else:
            logger.info("building '%s' extension", ext.name)

        # First, scan the sources for SWIG definition files (.i), run
        # SWIG on 'em to create .c files, and modify the sources list
        # accordingly.
        sources = self.swig_sources(sources, ext)

        # Next, compile the source code to object files.

        # XXX not honouring 'define_macros' or 'undef_macros' -- the
        # CCompiler API needs to change to accommodate this, and I
        # want to do one thing at a time!

        # Two possible sources for extra compiler arguments:
        #   - 'extra_compile_args' in Extension object
        #   - CFLAGS environment variable (not particularly
        #     elegant, but people seem to expect it and I
        #     guess it's useful)
        # The environment variable should take precedence, and
        # any sensible compiler will give precedence to later
        # command-line args.  Hence we combine them in order:
        extra_args = ext.extra_compile_args or []

        macros = ext.define_macros[:]
        for undef in ext.undef_macros:
            macros.append((undef,))

        objects = self.compiler_obj.compile(sources,
                                            output_dir=self.build_temp,
                                            macros=macros,
                                            include_dirs=ext.include_dirs,
                                            debug=self.debug,
                                            extra_postargs=extra_args,
                                            depends=ext.depends)

        # XXX -- this is a Vile HACK!
        #
        # The setup.py script for Python on Unix needs to be able to
        # get this list so it can perform all the clean up needed to
        # avoid keeping object files around when cleaning out a failed
        # build of an extension module.  Since Packaging does not
        # track dependencies, we have to get rid of intermediates to
        # ensure all the intermediates will be properly re-built.
        #
        self._built_objects = objects[:]

        # Now link the object files together into a "shared object" --
        # of course, first we have to figure out all the other things
        # that go into the mix.
        if ext.extra_objects:
            objects.extend(ext.extra_objects)
        extra_args = ext.extra_link_args or []

        # Detect target language, if not provided
        language = ext.language or self.compiler_obj.detect_language(sources)

        self.compiler_obj.link_shared_object(
            objects, ext_path,
            libraries=self.get_libraries(ext),
            library_dirs=ext.library_dirs,
            runtime_library_dirs=ext.runtime_library_dirs,
            extra_postargs=extra_args,
            export_symbols=self.get_export_symbols(ext),
            debug=self.debug,
            build_temp=self.build_temp,
            target_lang=language)


    def swig_sources(self, sources, extension):
        """Walk the list of source files in 'sources', looking for SWIG
        interface (.i) files.  Run SWIG on all that are found, and
        return a modified 'sources' list with SWIG source files replaced
        by the generated C (or C++) files.
        """
        new_sources = []
        swig_sources = []
        swig_targets = {}

        # XXX this drops generated C/C++ files into the source tree, which
        # is fine for developers who want to distribute the generated
        # source -- but there should be an option to put SWIG output in
        # the temp dir.

        if ('-c++' in self.swig_opts or '-c++' in extension.swig_opts):
            target_ext = '.cpp'
        else:
            target_ext = '.c'

        for source in sources:
            base, ext = os.path.splitext(source)
            if ext == ".i":             # SWIG interface file
                new_sources.append(base + '_wrap' + target_ext)
                swig_sources.append(source)
                swig_targets[source] = new_sources[-1]
            else:
                new_sources.append(source)

        if not swig_sources:
            return new_sources

        swig = self.swig or self.find_swig()
        swig_cmd = [swig, "-python"]
        swig_cmd.extend(self.swig_opts)

        # Do not override commandline arguments
        if not self.swig_opts:
            for o in extension.swig_opts:
                swig_cmd.append(o)

        for source in swig_sources:
            target = swig_targets[source]
            logger.info("swigging %s to %s", source, target)
            self.spawn(swig_cmd + ["-o", target, source])

        return new_sources

    def find_swig(self):
        """Return the name of the SWIG executable.  On Unix, this is
        just "swig" -- it should be in the PATH.  Tries a bit harder on
        Windows.
        """

        if os.name == "posix":
            return "swig"
        elif os.name == "nt":

            # Look for SWIG in its standard installation directory on
            # Windows (or so I presume!).  If we find it there, great;
            # if not, act like Unix and assume it's in the PATH.
            for vers in ("1.3", "1.2", "1.1"):
                fn = os.path.join("c:\\swig%s" % vers, "swig.exe")
                if os.path.isfile(fn):
                    return fn
            else:
                return "swig.exe"

        elif os.name == "os2":
            # assume swig available in the PATH.
            return "swig.exe"

        else:
            raise PackagingPlatformError(("I don't know how to find (much less run) SWIG "
                   "on platform '%s'") % os.name)

    # -- Name generators -----------------------------------------------
    # (extension names, filenames, whatever)
    def get_ext_fullpath(self, ext_name):
        """Returns the path of the filename for a given extension.

        The file is located in `build_lib` or directly in the package
        (inplace option).
        """
        fullname = self.get_ext_fullname(ext_name)
        modpath = fullname.split('.')
        filename = self.get_ext_filename(modpath[-1])

        if not self.inplace:
            # no further work needed
            # returning :
            #   build_dir/package/path/filename
            filename = os.path.join(*modpath[:-1]+[filename])
            return os.path.join(self.build_lib, filename)

        # the inplace option requires to find the package directory
        # using the build_py command for that
        package = '.'.join(modpath[0:-1])
        build_py = self.get_finalized_command('build_py')
        package_dir = os.path.abspath(build_py.get_package_dir(package))

        # returning
        #   package_dir/filename
        return os.path.join(package_dir, filename)

    def get_ext_fullname(self, ext_name):
        """Returns the fullname of a given extension name.

        Adds the `package.` prefix"""
        if self.package is None:
            return ext_name
        else:
            return self.package + '.' + ext_name

    def get_ext_filename(self, ext_name):
        r"""Convert the name of an extension (eg. "foo.bar") into the name
        of the file from which it will be loaded (eg. "foo/bar.so", or
        "foo\bar.pyd").
        """
        ext_path = ext_name.split('.')
        # OS/2 has an 8 character module (extension) limit :-(
        if os.name == "os2":
            ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
        # extensions in debug_mode are named 'module_d.pyd' under windows
        so_ext = sysconfig.get_config_var('SO')
        if os.name == 'nt' and self.debug:
            return os.path.join(*ext_path) + '_d' + so_ext
        return os.path.join(*ext_path) + so_ext

    def get_export_symbols(self, ext):
        """Return the list of symbols that a shared extension has to
        export.  This either uses 'ext.export_symbols' or, if it's not
        provided, "init" + module_name.  Only relevant on Windows, where
        the .pyd file (DLL) must export the module "init" function.
        """
        initfunc_name = "PyInit_" + ext.name.split('.')[-1]
        if initfunc_name not in ext.export_symbols:
            ext.export_symbols.append(initfunc_name)
        return ext.export_symbols

    def get_libraries(self, ext):
        """Return the list of libraries to link against when building a
        shared extension.  On most platforms, this is just 'ext.libraries';
        on Windows and OS/2, we add the Python library (eg. python20.dll).
        """
        # The python library is always needed on Windows.  For MSVC, this
        # is redundant, since the library is mentioned in a pragma in
        # pyconfig.h that MSVC groks.  The other Windows compilers all seem
        # to need it mentioned explicitly, though, so that's what we do.
        # Append '_d' to the python import library on debug builds.
        if sys.platform == "win32":
            from packaging.compiler.msvccompiler import MSVCCompiler
            if not isinstance(self.compiler_obj, MSVCCompiler):
                template = "python%d%d"
                if self.debug:
                    template = template + '_d'
                pythonlib = template % sys.version_info[:2]
                # don't extend ext.libraries, it may be shared with other
                # extensions, it is a reference to the original list
                return ext.libraries + [pythonlib]
            else:
                return ext.libraries
        elif sys.platform == "os2emx":
            # EMX/GCC requires the python library explicitly, and I
            # believe VACPP does as well (though not confirmed) - AIM Apr01
            template = "python%d%d"
            # debug versions of the main DLL aren't supported, at least
            # not at this time - AIM Apr01
            #if self.debug:
            #    template = template + '_d'
            pythonlib = template % sys.version_info[:2]
            # don't extend ext.libraries, it may be shared with other
            # extensions, it is a reference to the original list
            return ext.libraries + [pythonlib]
        elif sys.platform[:6] == "cygwin":
            template = "python%d.%d"
            pythonlib = template % sys.version_info[:2]
            # don't extend ext.libraries, it may be shared with other
            # extensions, it is a reference to the original list
            return ext.libraries + [pythonlib]
        elif sys.platform[:6] == "atheos":
            template = "python%d.%d"
            pythonlib = template % sys.version_info[:2]
            # Get SHLIBS from Makefile
            extra = []
            for lib in sysconfig.get_config_var('SHLIBS').split():
                if lib.startswith('-l'):
                    extra.append(lib[2:])
                else:
                    extra.append(lib)
            # don't extend ext.libraries, it may be shared with other
            # extensions, it is a reference to the original list
            return ext.libraries + [pythonlib, "m"] + extra

        elif sys.platform == 'darwin':
            # Don't use the default code below
            return ext.libraries

        else:
            if sysconfig.get_config_var('Py_ENABLE_SHARED'):
                template = 'python%d.%d' + sys.abiflags
                pythonlib = template % sys.version_info[:2]
                return ext.libraries + [pythonlib]
            else:
                return ext.libraries