diff options
| author | Ronald Oussoren <ronaldoussoren@mac.com> | 2020-11-22 10:18:40 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-11-22 10:18:40 (GMT) |
| commit | e8b1c038b14b5fc8120aab62c9bf5fb840274cb6 (patch) | |
| tree | e4b3ac9e3a17883c78d284c2f1552820ae098106 | |
| parent | 0aab3522b259c40abf1f070c71aa7b914c1239b5 (diff) | |
| download | cpython-e8b1c038b14b5fc8120aab62c9bf5fb840274cb6.zip cpython-e8b1c038b14b5fc8120aab62c9bf5fb840274cb6.tar.gz cpython-e8b1c038b14b5fc8120aab62c9bf5fb840274cb6.tar.bz2 | |
[3.9] bpo-41100: Support macOS 11 and Apple Silicon (GH-22855) (GH-23295)
* [3.9] bpo-41100: Support macOS 11 and Apple Silicon (GH-22855)
Co-authored-by: Lawrence D’Anna <lawrence_danna@apple.com>
* Add support for macOS 11 and Apple Silicon (aka arm64)
As a side effect of this work use the system copy of libffi on macOS, and remove the vendored copy
* Support building on recent versions of macOS while deploying to older versions
This allows building installers on macOS 11 while still supporting macOS 10.9..
(cherry picked from commit 41761933c1c30bb6003b65eef1ba23a83db4eae4)
Co-authored-by: Ronald Oussoren <ronaldoussoren@mac.com>
* Back port of changes to _decimal to support arm64
* temp_dir is in test.support in 3.9
28 files changed, 1592 insertions, 346 deletions
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 1112990..37975fe 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -113,6 +113,26 @@ def _get_system_version(): return _SYSTEM_VERSION +_SYSTEM_VERSION_TUPLE = None +def _get_system_version_tuple(): + """ + Return the macOS system version as a tuple + + The return value is safe to use to compare + two version numbers. + """ + global _SYSTEM_VERSION_TUPLE + if _SYSTEM_VERSION_TUPLE is None: + osx_version = _get_system_version() + if osx_version: + try: + _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + _SYSTEM_VERSION_TUPLE = () + + return _SYSTEM_VERSION_TUPLE + + def _remove_original_values(_config_vars): """Remove original unmodified values for testing""" # This is needed for higher-level cross-platform tests of get_platform. @@ -162,14 +182,18 @@ def _supports_universal_builds(): # builds, in particular -isysroot and -arch arguments to the compiler. This # is in support of allowing 10.4 universal builds to run on 10.3.x systems. - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' + osx_version = _get_system_version_tuple() return bool(osx_version >= (10, 4)) if osx_version else False +def _supports_arm64_builds(): + """Returns True if arm64 builds are supported on this system""" + # There are two sets of systems supporting macOS/arm64 builds: + # 1. macOS 11 and later, unconditionally + # 2. macOS 10.15 with Xcode 12.2 or later + # For now the second category is ignored. + osx_version = _get_system_version_tuple() + return osx_version >= (11, 0) if osx_version else False + def _find_appropriate_compiler(_config_vars): """Find appropriate C compiler for extension module builds""" @@ -361,6 +385,12 @@ def compiler_fixup(compiler_so, cc_args): except ValueError: break + elif not _supports_arm64_builds(): + # Look for "-arch arm64" and drop that + for idx in reversed(range(len(compiler_so))): + if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64": + del compiler_so[idx:idx+2] + if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, # see also distutils.sysconfig @@ -511,6 +541,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b05..1c3f8fd 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ import os from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b35269..a1bac26 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 5e47e07..5a32e03 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -492,7 +492,7 @@ class BuildExtTestCase(TempdirManager, # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 770e2c5..05568f2 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1034,6 +1034,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index a5c35df..bd953b5 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -245,7 +245,7 @@ class PlatformTest(unittest.TestCase): self.assertEqual(res[1], ('', '', '')) if sys.byteorder == 'little': - self.assertIn(res[2], ('i386', 'x86_64')) + self.assertIn(res[2], ('i386', 'x86_64', 'arm64')) else: self.assertEqual(res[2], 'PowerPC') diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index be121ae..f4edb8b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1905,6 +1905,233 @@ class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): assert_python_ok(*args, PATH=path) +@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") +class TestPosixWeaklinking(unittest.TestCase): + # These test cases verify that weak linking support on macOS works + # as expected. These cases only test new behaviour introduced by weak linking, + # regular behaviour is tested by the normal test cases. + # + # See the section on Weak Linking in Mac/README.txt for more information. + def setUp(self): + import sysconfig + import platform + + config_vars = sysconfig.get_config_vars() + self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] } + self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split(".")) + + def _verify_available(self, name): + if name not in self.available: + raise unittest.SkipTest(f"{name} not weak-linked") + + def test_pwritev(self): + self._verify_available("HAVE_PWRITEV") + if self.mac_ver >= (10, 16): + self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") + self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + + else: + self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") + self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + + def test_stat(self): + self._verify_available("HAVE_FSTATAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FSTATAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FSTATAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.stat("file", dir_fd=0) + + def test_access(self): + self._verify_available("HAVE_FACCESSAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FACCESSAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FACCESSAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.access("file", os.R_OK, dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): + os.access("file", os.R_OK, follow_symlinks=False) + + with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"): + os.access("file", os.R_OK, effective_ids=True) + + def test_chmod(self): + self._verify_available("HAVE_FCHMODAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHMODAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FCHMODAT", posix._have_functions) + self.assertIn("HAVE_LCHMOD", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chmod("file", 0o644, dir_fd=0) + + def test_chown(self): + self._verify_available("HAVE_FCHOWNAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHOWNAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions) + self.assertIn("HAVE_LCHOWN", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chown("file", 0, 0, dir_fd=0) + + def test_link(self): + self._verify_available("HAVE_LINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_LINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_LINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"): + os.link("source", "target", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) + + # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag + with support.temp_dir() as base_path: + link_path = os.path.join(base_path, "link") + target_path = os.path.join(base_path, "target") + source_path = os.path.join(base_path, "source") + + with open(source_path, "w") as fp: + fp.write("data") + + os.symlink("target", link_path) + + # Calling os.link should fail in the link(2) call, and + # should not reject *follow_symlinks* (to match the + # behaviour you'd get when building on a platform without + # linkat) + with self.assertRaises(FileExistsError): + os.link(source_path, link_path, follow_symlinks=True) + + with self.assertRaises(FileExistsError): + os.link(source_path, link_path, follow_symlinks=False) + + + def test_listdir_scandir(self): + self._verify_available("HAVE_FDOPENDIR") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FDOPENDIR", posix._have_functions) + + else: + self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions) + + with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"): + os.listdir(0) + + with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"): + os.scandir(0) + + def test_mkdir(self): + self._verify_available("HAVE_MKDIRAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_MKDIRAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_MKDIRAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.mkdir("dir", dir_fd=0) + + def test_rename_replace(self): + self._verify_available("HAVE_RENAMEAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_RENAMEAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_RENAMEAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", dst_dir_fd=0) + + def test_unlink_rmdir(self): + self._verify_available("HAVE_UNLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_UNLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_UNLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.unlink("path", dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.rmdir("path", dir_fd=0) + + def test_open(self): + self._verify_available("HAVE_OPENAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_OPENAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_OPENAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.open("path", os.O_RDONLY, dir_fd=0) + + def test_readlink(self): + self._verify_available("HAVE_READLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_READLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_READLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.readlink("path", dir_fd=0) + + def test_symlink(self): + self._verify_available("HAVE_SYMLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_SYMLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.symlink("a", "b", dir_fd=0) + + def test_utime(self): + self._verify_available("HAVE_FUTIMENS") + self._verify_available("HAVE_UTIMENSAT") + if self.mac_ver >= (10, 13): + self.assertIn("HAVE_FUTIMENS", posix._have_functions) + self.assertIn("HAVE_UTIMENSAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FUTIMENS", posix._have_functions) + self.assertNotIn("HAVE_UTIMENSAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.utime("path", dir_fd=0) + + def test_main(): try: support.run_unittest( @@ -1912,6 +2139,7 @@ def test_main(): PosixGroupsTester, TestPosixSpawn, TestPosixSpawnP, + TestPosixWeaklinking ) finally: support.reap_children() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 80e43fa..6674edc 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1040,6 +1040,36 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): with self.assertRaises(ValueError): pytime_object_to_timespec(float('nan'), time_rnd) +@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") +class TestTimeWeaklinking(unittest.TestCase): + # These test cases verify that weak linking support on macOS works + # as expected. These cases only test new behaviour introduced by weak linking, + # regular behaviour is tested by the normal test cases. + # + # See the section on Weak Linking in Mac/README.txt for more information. + def test_clock_functions(self): + import sysconfig + import platform + + config_vars = sysconfig.get_config_vars() + var_name = "HAVE_CLOCK_GETTIME" + if var_name not in config_vars or not config_vars[var_name]: + raise unittest.SkipTest(f"{var_name} is not available") + + mac_ver = tuple(int(x) for x in platform.mac_ver()[0].split(".")) + + clock_names = [ + "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", + "clock_settime_ns", "clock_getres"] + + if mac_ver >= (10, 12): + for name in clock_names: + self.assertTrue(hasattr(time, name), f"time.{name} is not available") + + else: + for name in clock_names: + self.assertFalse(hasattr(time, name), f"time.{name} is available") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 2ee4e64..23508c5 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2523,11 +2523,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): support.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 2548b21..0e76d3c 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -116,7 +116,8 @@ WORKDIR = "/tmp/_py" DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -universal_opts_map = { '32-bit': ('i386', 'ppc',), +universal_opts_map = { 'universal2': ('arm64', 'x86_64'), + '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), 'intel-32': ('i386',), @@ -124,6 +125,7 @@ universal_opts_map = { '32-bit': ('i386', 'ppc',), '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { + 'universal2': '10.9', '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', @@ -190,6 +192,27 @@ EXPECTED_SHARED_LIBS = {} def internalTk(): return getDeptargetTuple() >= (10, 6) + +def tweak_tcl_build(basedir, archList): + with open("Makefile", "r") as fp: + contents = fp.readlines() + + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. + new_contents = [] + for line in contents: + if line.startswith("COMPAT_OBJS"): + # note: the space before strtod.o is intentional, + # the detection of a broken strtod results in + # "fixstrod.o" on this line. + for nm in ("strstr.o", "strtoul.o", " strtod.o"): + line = line.replace(nm, "") + new_contents.append(line) + + with open("Makefile", "w") as fp: + fp.writelines(new_contents) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -215,6 +238,9 @@ def library_recipes(): buildrecipe=build_universal_openssl, configure=None, install=None, + patches=[ + "openssl-mac-arm64.patch", + ], ), ]) @@ -231,6 +257,7 @@ def library_recipes(): '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), ], useLDFlags=False, + buildrecipe=tweak_tcl_build, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), @@ -801,6 +828,7 @@ def build_universal_openssl(basedir, archList): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], + "arm64": ["darwin64-arm64-cc"], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } diff --git a/Mac/BuildScript/openssl-mac-arm64.patch b/Mac/BuildScript/openssl-mac-arm64.patch new file mode 100644 index 0000000..11267fb --- /dev/null +++ b/Mac/BuildScript/openssl-mac-arm64.patch @@ -0,0 +1,41 @@ +diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf +--- openssl-1.1.1g-orig/Configurations/10-main.conf 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/Configurations/10-main.conf 2020-07-26 12:21:32.000000000 +0200 +@@ -1557,6 +1557,14 @@ + bn_ops => "SIXTY_FOUR_BIT_LONG", + perlasm_scheme => "macosx", + }, ++ "darwin64-arm64-cc" => { ++ inherit_from => [ "darwin-common", asm("aarch64_asm") ], ++ CFLAGS => add("-Wall"), ++ cflags => add("-arch arm64"), ++ lib_cppflags => add("-DL_ENDIAN"), ++ bn_ops => "SIXTY_FOUR_BIT_LONG", ++ perlasm_scheme => "ios64", ++ }, + + ##### GNU Hurd + "hurd-x86" => { +diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config +--- openssl-1.1.1g-orig/config 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/config 2020-07-26 12:21:59.000000000 +0200 +@@ -255,6 +255,9 @@ + ;; + x86_64) + echo "x86_64-apple-darwin${VERSION}" ++ ;; ++ arm64) ++ echo "arm64-apple-darwin${VERSION}" + ;; + *) + echo "i686-apple-darwin${VERSION}" +@@ -497,6 +500,9 @@ + else + OUT="darwin64-x86_64-cc" + fi ;; ++ x86_64-apple-darwin*) ++ OUT="darwin64-arm64-cc" ++ ;; + armv6+7-*-iphoneos) + __CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7" + __CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7" diff --git a/Mac/README.rst b/Mac/README.rst index ec7d873..f3638aa 100644 --- a/Mac/README.rst +++ b/Mac/README.rst @@ -120,6 +120,8 @@ support ppc (Xcode 4 on 10.6 and later systems). The flavor can be specified using the configure option ``--with-universal-archs=VALUE``. The following values are available: + * ``universal2``: ``arm64``, ``x86_64`` + * ``intel``: ``i386``, ``x86_64`` * ``intel-32``: ``i386`` @@ -155,6 +157,8 @@ following combinations of SDKs and universal-archs flavors are available: * 10.15 and later SDKs support ``intel-64`` only + * 11.0 and later SDKs support ``universal2`` + The makefile for a framework build will also install ``python3.x-32`` binaries when the universal architecture includes at least one 32-bit architecture (that is, for all flavors but ``64-bit`` and ``intel-64``). @@ -352,6 +356,39 @@ A framework install also installs some applications in ``/Applications/Python X. And lastly a framework installation installs files in ``/usr/local/bin``, all of them symbolic links to files in ``/Library/Frameworks/Python.framework/Versions/X.Y/bin``. +Weak linking support +==================== + +The CPython sources support building with the latest SDK while targetting deployment +to macOS 10.9. This is done through weak linking of symbols introduced in macOS +10.10 or later and checking for their availability at runtime. + +This requires the use of Apple's compiler toolchain on macOS 10.13 or later. + +The basic implementation pattern is: + +* ``HAVE_<FUNCTION>`` is a macro defined (or not) by the configure script + +* ``HAVE_<FUNCTION>_RUNTIME`` is a macro defined in the relevant source + files. This expands to a call to ``__builtin_available`` when using + a new enough Apple compiler, and to a true value otherwise. + +* Use ``HAVE_<FUNCTION>_RUNTIME`` before calling ``<function>``. This macro + *must* be used a the sole expression in an if statement:: + + if (HAVE_<FUNCTION>_RUNTIME) { + /* <function> is available */ + } + + Or: + + if (HAVE_<FUNCTION>_RUNTIME) {} else { + /* <function> is not available */ + } + + Using other patterns (such as ``!HAVE_<FUNCTION>_RUNTIME``) is not supported + by Apple's compilers. + Resources ========= diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index c8bd3ba..78813e8 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) size_t count; cpu_type_t cpu_types[1]; short flags = 0; -#ifdef __LP64__ - int ch; -#endif if ((errno = posix_spawnattr_init(spawnattr)) != 0) { err(2, "posix_spawnattr_int"); @@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) #elif defined(__ppc__) cpu_types[0] = CPU_TYPE_POWERPC; + #elif defined(__i386__) cpu_types[0] = CPU_TYPE_X86; + +#elif defined(__arm64__) + cpu_types[0] = CPU_TYPE_ARM64; + #else # error "Unknown CPU" + #endif if (posix_spawnattr_setbinpref_np(spawnattr, count, @@ -220,7 +223,8 @@ main(int argc, char **argv) { /* We're weak-linking to posix-spawnv to ensure that * an executable build on 10.5 can work on 10.4. */ - if (posix_spawn != NULL) { + + if (&posix_spawn != NULL) { posix_spawnattr_t spawnattr = NULL; setup_spawnattr(&spawnattr); diff --git a/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst b/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst new file mode 100644 index 0000000..6cbb279 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst @@ -0,0 +1,8 @@ +Add support for macOS 11 and Apple Silicon systems. + +It is now possible to build "Universal 2" binaries using +"--enable-universalsdk --with-universal-archs=universal2". + +Binaries build on later macOS versions can be deployed back to older +versions (tested up to macOS 10.9), when using the correct deployment +target. This is tested using Xcode 11 and later. diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 2abfa67..39cace5 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,6 +1,8 @@ #include "Python.h" #include "frameobject.h" +#include <stdbool.h> + #include <ffi.h> #ifdef MS_WIN32 #include <windows.h> @@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) Py_XDECREF(self->callable); Py_XDECREF(self->restype); if (self->pcl_write) - ffi_closure_free(self->pcl_write); + Py_ffi_closure_free(self->pcl_write); PyObject_GC_Del(self); } @@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(CThunk_CheckExact((PyObject *)p)); - p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), - &p->pcl_exec); + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); goto error; @@ -408,13 +409,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } -#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) - result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); +#if HAVE_FFI_PREP_CLOSURE_LOC +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME 1 +# endif + if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { + result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, + p, + p->pcl_exec); + } else +#endif + { +#if USING_APPLE_OS_LIBFFI && defined(__arm64__) + PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); + goto error; #else - result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, - p, - p->pcl_exec); +#ifdef MACOSX + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif + result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); + +#ifdef MACOSX + #pragma clang diagnostic pop +#endif + +#endif + } if (result != FFI_OK) { PyErr_Format(PyExc_RuntimeError, "ffi_prep_closure failed with %d", result); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 6030cc3..b0a36a3 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -57,6 +57,8 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include <stdbool.h> + #ifdef MS_WIN32 #include <windows.h> #include <tchar.h> @@ -64,6 +66,10 @@ #include "ctypes_dlfcn.h" #endif +#ifdef __APPLE__ +#include <mach-o/dyld.h> +#endif + #ifdef MS_WIN32 #include <malloc.h> #endif @@ -812,7 +818,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -835,14 +842,70 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# elif HAVE_FFI_PREP_CIF_VAR +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false +# endif + + /* Even on Apple-arm64 the calling convention for variadic functions conincides + * with the standard calling convention in the case that the function called + * only with its fixed arguments. Thus, we do not need a special flag to be + * set on variadic functions. We treat a function as variadic if it is called + * with a nonzero number of variadic arguments */ + bool is_variadic = (argtypecount != 0 && argcount > argtypecount); + (void) is_variadic; + +#if defined(__APPLE__) && defined(__arm64__) + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + } else { + PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); + return -1; + } + } +#endif + +#if HAVE_FFI_PREP_CIF_VAR + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } else { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } + } + } else +#endif + + { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { @@ -1212,9 +1275,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN @@ -1398,6 +1460,42 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH +static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) +{ + PyObject *name, *name2; + char *name_str; + + if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { + int r; + + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + name_str = PyBytes_AS_STRING(name2); + + r = _dyld_shared_cache_contains_path(name_str); + Py_DECREF(name2); + + if (r) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + + } else { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + } +#endif + static PyObject *py_dl_open(PyObject *self, PyObject *args) { PyObject *name, *name2; @@ -1887,6 +1985,8 @@ buffer_info(PyObject *self, PyObject *arg) return Py_BuildValue("siN", dict->format, dict->ndim, shape); } + + PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, @@ -1909,6 +2009,9 @@ PyMethodDef _ctypes_module_methods[] = { {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, #endif +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, +#endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, {"byref", byref, METH_VARARGS, byref_doc}, diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf..3f20031 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -366,6 +366,14 @@ PyObject *_ctypes_get_errobj(int **pspace); extern PyObject *ComError; #endif +#if USING_MALLOC_CLOSURE_DOT_C +void Py_ffi_closure_free(void *p); +void *Py_ffi_closure_alloc(size_t size, void** codeloc); +#else +#define Py_ffi_closure_free ffi_closure_free +#define Py_ffi_closure_alloc ffi_closure_alloc +#endif + /* Local Variables: compile-command: "python setup.py -q build install --home ~" diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index f9cdb33..4f220e4 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -89,16 +89,27 @@ static void more_core(void) /******************************************************************/ /* put the item back into the free list */ -void ffi_closure_free(void *p) +void Py_ffi_closure_free(void *p) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + ffi_closure_free(p); + return; + } +#endif ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; } /* return one item from the free list, allocating more if needed */ -void *ffi_closure_alloc(size_t ignored, void** codeloc) +void *Py_ffi_closure_alloc(size_t size, void** codeloc) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + return ffi_closure_alloc(size, codeloc); + } +#endif ITEM *item; if (!free_list) more_core(); diff --git a/Modules/_decimal/libmpdec/mpdecimal.h b/Modules/_decimal/libmpdec/mpdecimal.h index 2815a8c..5a24396 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.h +++ b/Modules/_decimal/libmpdec/mpdecimal.h @@ -121,6 +121,9 @@ const char *mpd_version(void); #elif defined(__x86_64__) #define CONFIG_64 #define ASM + #elif defined(__arm64__) + #define CONFIG_64 + #define ANSI #else #error "unknown architecture for universal build." #endif diff --git a/Modules/getpath.c b/Modules/getpath.c index a84c858..4035819 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -923,11 +923,7 @@ static PyStatus calculate_program_macos(wchar_t **abs_path_p) { char execpath[MAXPATHLEN + 1]; -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#else - unsigned long nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#endif /* On Mac OS X, if a script uses an interpreter of the form "#!/opt/python2.3/bin/python", the kernel only passes "python" diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8e14ffc..12f72f5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7,18 +7,6 @@ of the compiler used. Different compilers define their own feature test macro, e.g. '_MSC_VER'. */ -#ifdef __APPLE__ - /* - * Step 1 of support for weak-linking a number of symbols existing on - * OSX 10.4 and later, see the comment in the #ifdef __APPLE__ block - * at the end of this file for more information. - */ -# pragma weak lchown -# pragma weak statvfs -# pragma weak fstatvfs - -#endif /* __APPLE__ */ - #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -50,6 +38,127 @@ #include <stdio.h> /* needed for ctermid() */ +/* + * A number of APIs are available on macOS from a certain macOS version. + * To support building with a new SDK while deploying to older versions + * the availability test is split into two: + * - HAVE_<FUNCTION>: The configure check for compile time availability + * - HAVE_<FUNCTION>_RUNTIME: Runtime check for availability + * + * The latter is always true when not on macOS, or when using a compiler + * that does not support __has_builtin (older versions of Xcode). + * + * Due to compiler restrictions there is one valid use of HAVE_<FUNCTION>_RUNTIME: + * if (HAVE_<FUNCTION>_RUNTIME) { ... } + * + * In mixing the test with other tests or using negations will result in compile + * errors. + */ +#if defined(__APPLE__) + +#if defined(__has_builtin) && __has_builtin(__builtin_available) +# define HAVE_FSTATAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FACCESSAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FCHMODAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FCHOWNAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_LINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FDOPENDIR_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_MKDIRAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_RENAMEAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_UNLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_OPENAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_READLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_SYMLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FUTIMENS_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) +# define HAVE_UTIMENSAT_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) +# define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) + +# define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) + +#else /* Xcode 8 or earlier */ + + /* __builtin_available is not present in these compilers, but + * some of the symbols might be weak linked (10.10 SDK or later + * deploying on 10.9. + * + * Fall back to the older style of availability checking for + * symbols introduced in macOS 10.10. + */ + +# ifdef HAVE_FSTATAT +# define HAVE_FSTATAT_RUNTIME (fstatat != NULL) +# endif + +# ifdef HAVE_FACCESSAT +# define HAVE_FACCESSAT_RUNTIME (faccessat != NULL) +# endif + +# ifdef HAVE_FCHMODAT +# define HAVE_FCHMODAT_RUNTIME (fchmodat != NULL) +# endif + +# ifdef HAVE_FCHOWNAT +# define HAVE_FCHOWNAT_RUNTIME (fchownat != NULL) +# endif + +# ifdef HAVE_LINKAT +# define HAVE_LINKAT_RUNTIME (linkat != NULL) +# endif + +# ifdef HAVE_FDOPENDIR +# define HAVE_FDOPENDIR_RUNTIME (fdopendir != NULL) +# endif + +# ifdef HAVE_MKDIRAT +# define HAVE_MKDIRAT_RUNTIME (mkdirat != NULL) +# endif + +# ifdef HAVE_RENAMEAT +# define HAVE_RENAMEAT_RUNTIME (renameat != NULL) +# endif + +# ifdef HAVE_UNLINKAT +# define HAVE_UNLINKAT_RUNTIME (unlinkat != NULL) +# endif + +# ifdef HAVE_OPENAT +# define HAVE_OPENAT_RUNTIME (openat != NULL) +# endif + +# ifdef HAVE_READLINKAT +# define HAVE_READLINKAT_RUNTIME (readlinkat != NULL) +# endif + +# ifdef HAVE_SYMLINKAT +# define HAVE_SYMLINKAT_RUNTIME (symlinkat != NULL) +# endif + +#endif + +#ifdef HAVE_FUTIMESAT +/* Some of the logic for weak linking depends on this assertion */ +# error "HAVE_FUTIMESAT unexpectedly defined" +#endif + +#else +# define HAVE_FSTATAT_RUNTIME 1 +# define HAVE_FACCESSAT_RUNTIME 1 +# define HAVE_FCHMODAT_RUNTIME 1 +# define HAVE_FCHOWNAT_RUNTIME 1 +# define HAVE_LINKAT_RUNTIME 1 +# define HAVE_FDOPENDIR_RUNTIME 1 +# define HAVE_MKDIRAT_RUNTIME 1 +# define HAVE_RENAMEAT_RUNTIME 1 +# define HAVE_UNLINKAT_RUNTIME 1 +# define HAVE_OPENAT_RUNTIME 1 +# define HAVE_READLINKAT_RUNTIME 1 +# define HAVE_SYMLINKAT_RUNTIME 1 +# define HAVE_FUTIMENS_RUNTIME 1 +# define HAVE_UTIMENSAT_RUNTIME 1 +# define HAVE_PWRITEV_RUNTIME 1 +#endif + + #ifdef __cplusplus extern "C" { #endif @@ -2308,6 +2417,10 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, STRUCT_STAT st; int result; +#ifdef HAVE_FSTATAT + int fstatat_unavailable = 0; +#endif + #if !defined(MS_WINDOWS) && !defined(HAVE_FSTATAT) && !defined(HAVE_LSTAT) if (follow_symlinks_specified(function_name, follow_symlinks)) return NULL; @@ -2334,15 +2447,27 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, else #endif /* HAVE_LSTAT */ #ifdef HAVE_FSTATAT - if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) - result = fstatat(dir_fd, path->narrow, &st, + if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { + if (HAVE_FSTATAT_RUNTIME) { + result = fstatat(dir_fd, path->narrow, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - else + + } else { + fstatat_unavailable = 1; + } + } else #endif /* HAVE_FSTATAT */ result = STAT(path->narrow, &st); #endif /* MS_WINDOWS */ Py_END_ALLOW_THREADS +#ifdef HAVE_FSTATAT + if (fstatat_unavailable) { + argument_unavailable_error("stat", "dir_fd"); + return NULL; + } +#endif + if (result != 0) { return path_error(path); } @@ -2760,6 +2885,10 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int result; #endif +#ifdef HAVE_FACCESSAT + int faccessat_unavailable = 0; +#endif + #ifndef HAVE_FACCESSAT if (follow_symlinks_specified("access", follow_symlinks)) return -1; @@ -2794,17 +2923,40 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, if ((dir_fd != DEFAULT_DIR_FD) || effective_ids || !follow_symlinks) { - int flags = 0; - if (!follow_symlinks) - flags |= AT_SYMLINK_NOFOLLOW; - if (effective_ids) - flags |= AT_EACCESS; - result = faccessat(dir_fd, path->narrow, mode, flags); + + if (HAVE_FACCESSAT_RUNTIME) { + int flags = 0; + if (!follow_symlinks) + flags |= AT_SYMLINK_NOFOLLOW; + if (effective_ids) + flags |= AT_EACCESS; + result = faccessat(dir_fd, path->narrow, mode, flags); + } else { + faccessat_unavailable = 1; + } } else #endif result = access(path->narrow, mode); Py_END_ALLOW_THREADS + +#ifdef HAVE_FACCESSAT + if (faccessat_unavailable) { + if (dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("access", "dir_fd"); + return -1; + } + if (follow_symlinks_specified("access", follow_symlinks)) + return -1; + + if (effective_ids) { + argument_unavailable_error("access", "effective_ids"); + return -1; + } + /* should be unreachable */ + return -1; + } +#endif return_value = !result; #endif @@ -3002,6 +3154,7 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; + int fchmodat_unsupported = 0; #endif #if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) @@ -3037,42 +3190,56 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, if (path->fd != -1) result = fchmod(path->fd, mode); else -#endif +#endif /* HAVE_CHMOD */ #ifdef HAVE_LCHMOD if ((!follow_symlinks) && (dir_fd == DEFAULT_DIR_FD)) result = lchmod(path->narrow, mode); else -#endif +#endif /* HAVE_LCHMOD */ #ifdef HAVE_FCHMODAT if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { - /* - * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! - * The documentation specifically shows how to use it, - * and then says it isn't implemented yet. - * (true on linux with glibc 2.15, and openindiana 3.x) - * - * Once it is supported, os.chmod will automatically - * support dir_fd and follow_symlinks=False. (Hopefully.) - * Until then, we need to be careful what exception we raise. - */ - result = fchmodat(dir_fd, path->narrow, mode, - follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - /* - * But wait! We can't throw the exception without allowing threads, - * and we can't do that in this nested scope. (Macro trickery, sigh.) - */ - fchmodat_nofollow_unsupported = - result && - ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && - !follow_symlinks; + if (HAVE_FCHMODAT_RUNTIME) { + /* + * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! + * The documentation specifically shows how to use it, + * and then says it isn't implemented yet. + * (true on linux with glibc 2.15, and openindiana 3.x) + * + * Once it is supported, os.chmod will automatically + * support dir_fd and follow_symlinks=False. (Hopefully.) + * Until then, we need to be careful what exception we raise. + */ + result = fchmodat(dir_fd, path->narrow, mode, + follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); + /* + * But wait! We can't throw the exception without allowing threads, + * and we can't do that in this nested scope. (Macro trickery, sigh.) + */ + fchmodat_nofollow_unsupported = + result && + ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && + !follow_symlinks; + } else { + fchmodat_unsupported = 1; + fchmodat_nofollow_unsupported = 1; + + result = -1; + } } else -#endif +#endif /* HAVE_FHCMODAT */ result = chmod(path->narrow, mode); Py_END_ALLOW_THREADS if (result) { #ifdef HAVE_FCHMODAT + if (fchmodat_unsupported) { + if (dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("chmod", "dir_fd"); + return NULL; + } + } + if (fchmodat_nofollow_unsupported) { if (dir_fd != DEFAULT_DIR_FD) dir_fd_and_follow_symlinks_invalid("chmod", @@ -3082,10 +3249,10 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, return NULL; } else -#endif +#endif /* HAVE_FCHMODAT */ return path_error(path); } -#endif +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } @@ -3373,6 +3540,10 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, { int result; +#if defined(HAVE_FCHOWNAT) + int fchownat_unsupported = 0; +#endif + #if !(defined(HAVE_LCHOWN) || defined(HAVE_FCHOWNAT)) if (follow_symlinks_specified("chown", follow_symlinks)) return NULL; @@ -3381,19 +3552,6 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, fd_and_follow_symlinks_invalid("chown", path->fd, follow_symlinks)) return NULL; -#ifdef __APPLE__ - /* - * This is for Mac OS X 10.3, which doesn't have lchown. - * (But we still have an lchown symbol because of weak-linking.) - * It doesn't have fchownat either. So there's no possibility - * of a graceful failover. - */ - if ((!follow_symlinks) && (lchown == NULL)) { - follow_symlinks_specified("chown", follow_symlinks); - return NULL; - } -#endif - if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { return NULL; @@ -3411,14 +3569,28 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, else #endif #ifdef HAVE_FCHOWNAT - if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) + if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { + if (HAVE_FCHOWNAT_RUNTIME) { result = fchownat(dir_fd, path->narrow, uid, gid, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - else + } else { + fchownat_unsupported = 1; + } + } else #endif result = chown(path->narrow, uid, gid); Py_END_ALLOW_THREADS +#ifdef HAVE_FCHOWNAT + if (fchownat_unsupported) { + /* This would be incorrect if the current platform + * doesn't support lchown. + */ + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -3664,6 +3836,9 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #else int result; #endif +#if defined(HAVE_LINKAT) + int linkat_unavailable = 0; +#endif #ifndef HAVE_LINKAT if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD)) { @@ -3698,15 +3873,43 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #ifdef HAVE_LINKAT if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD) || - (!follow_symlinks)) - result = linkat(src_dir_fd, src->narrow, - dst_dir_fd, dst->narrow, - follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + (!follow_symlinks)) { + + if (HAVE_LINKAT_RUNTIME) { + + result = linkat(src_dir_fd, src->narrow, + dst_dir_fd, dst->narrow, + follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + + } +#ifdef __APPLE__ + else { + if (src_dir_fd == DEFAULT_DIR_FD && dst_dir_fd == DEFAULT_DIR_FD) { + /* See issue 41355: This matches the behaviour of !HAVE_LINKAT */ + result = link(src->narrow, dst->narrow); + } else { + linkat_unavailable = 1; + } + } +#endif + } else #endif /* HAVE_LINKAT */ result = link(src->narrow, dst->narrow); Py_END_ALLOW_THREADS +#ifdef HAVE_LINKAT + if (linkat_unavailable) { + /* Either or both dir_fd arguments were specified */ + if (src_dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("link", "src_dir_fd"); + } else { + argument_unavailable_error("link", "dst_dir_fd"); + } + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif /* MS_WINDOWS */ @@ -3829,6 +4032,7 @@ _posix_listdir(path_t *path, PyObject *list) errno = 0; #ifdef HAVE_FDOPENDIR if (path->fd != -1) { + if (HAVE_FDOPENDIR_RUNTIME) { /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(path->fd); if (fd == -1) @@ -3839,6 +4043,11 @@ _posix_listdir(path_t *path, PyObject *list) Py_BEGIN_ALLOW_THREADS dirp = fdopendir(fd); Py_END_ALLOW_THREADS + } else { + PyErr_SetString(PyExc_TypeError, + "listdir: path should be string, bytes, os.PathLike or None, not int"); + return NULL; + } } else #endif @@ -4152,6 +4361,9 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=e965f68377e9b1ce]*/ { int result; +#ifdef HAVE_MKDIRAT + int mkdirat_unavailable = 0; +#endif if (PySys_Audit("os.mkdir", "Oii", path->object, mode, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4168,9 +4380,14 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_MKDIRAT_RUNTIME) { result = mkdirat(dir_fd, path->narrow, mode); - else + + } else { + mkdirat_unavailable = 1; + } + } else #endif #if defined(__WATCOMC__) && !defined(__QNX__) result = mkdir(path->narrow); @@ -4178,6 +4395,14 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) result = mkdir(path->narrow, mode); #endif Py_END_ALLOW_THREADS + +#if HAVE_MKDIRAT + if (mkdirat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result < 0) return path_error(path); #endif /* MS_WINDOWS */ @@ -4287,6 +4512,10 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is const char *function_name = is_replace ? "replace" : "rename"; int dir_fd_specified; +#ifdef HAVE_RENAMEAT + int renameat_unavailable = 0; +#endif + #ifdef MS_WINDOWS BOOL result; int flags = is_replace ? MOVEFILE_REPLACE_EXISTING : 0; @@ -4326,13 +4555,25 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is Py_BEGIN_ALLOW_THREADS #ifdef HAVE_RENAMEAT - if (dir_fd_specified) - result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); - else + if (dir_fd_specified) { + if (HAVE_RENAMEAT_RUNTIME) { + result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); + } else { + renameat_unavailable = 1; + } + } else #endif result = rename(src->narrow, dst->narrow); Py_END_ALLOW_THREADS + +#ifdef HAVE_RENAMEAT + if (renameat_unavailable) { + argument_unavailable_error(function_name, "src_dir_fd and dst_dir_fd"); + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif @@ -4408,6 +4649,9 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) /*[clinic end generated code: output=080eb54f506e8301 input=38c8b375ca34a7e2]*/ { int result; +#ifdef HAVE_UNLINKAT + int unlinkat_unavailable = 0; +#endif if (PySys_Audit("os.rmdir", "Oi", path->object, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4420,14 +4664,26 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) result = !RemoveDirectoryW(path->wide); #else #ifdef HAVE_UNLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_UNLINKAT_RUNTIME) { result = unlinkat(dir_fd, path->narrow, AT_REMOVEDIR); - else + } else { + unlinkat_unavailable = 1; + result = -1; + } + } else #endif result = rmdir(path->narrow); #endif Py_END_ALLOW_THREADS +#ifdef HAVE_UNLINKAT + if (unlinkat_unavailable) { + argument_unavailable_error("rmdir", "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -4571,6 +4827,9 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) /*[clinic end generated code: output=621797807b9963b1 input=d7bcde2b1b2a2552]*/ { int result; +#ifdef HAVE_UNLINKAT + int unlinkat_unavailable = 0; +#endif if (PySys_Audit("os.remove", "Oi", path->object, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4584,15 +4843,27 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) result = !Py_DeleteFileW(path->wide); #else #ifdef HAVE_UNLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_UNLINKAT_RUNTIME) { + result = unlinkat(dir_fd, path->narrow, 0); - else + } else { + unlinkat_unavailable = 1; + } + } else #endif /* HAVE_UNLINKAT */ result = unlink(path->narrow); #endif _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS +#ifdef HAVE_UNLINKAT + if (unlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -4763,7 +5034,16 @@ typedef struct { static int utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) { -#ifdef HAVE_UTIMENSAT +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + if (HAVE_UTIMENSAT_RUNTIME) { + int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; + UTIME_TO_TIMESPEC; + return utimensat(dir_fd, path, time, flags); + } else { + errno = ENOSYS; + return -1; + } +#elif defined(HAVE_UTIMENSAT) int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; UTIME_TO_TIMESPEC; return utimensat(dir_fd, path, time, flags); @@ -4790,11 +5070,30 @@ static int utime_fd(utime_t *ut, int fd) { #ifdef HAVE_FUTIMENS + + if (HAVE_FUTIMENS_RUNTIME) { + UTIME_TO_TIMESPEC; return futimens(fd, time); -#else + + } else +#ifndef HAVE_FUTIMES + { + /* Not sure if this can happen */ + PyErr_SetString( + PyExc_RuntimeError, + "neither futimens nor futimes are supported" + " on this system"); + return -1; + } +#endif + +#endif +#ifdef HAVE_FUTIMES + { UTIME_TO_TIMEVAL; return futimes(fd, time); + } #endif } @@ -4813,11 +5112,27 @@ static int utime_nofollow_symlinks(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT - UTIME_TO_TIMESPEC; - return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); -#else + if (HAVE_UTIMENSAT_RUNTIME) { + UTIME_TO_TIMESPEC; + return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); + } else +#ifndef HAVE_LUTIMES + { + /* Not sure if this can happen */ + PyErr_SetString( + PyExc_RuntimeError, + "neither utimensat nor lutimes are supported" + " on this system"); + return -1; + } +#endif +#endif + +#ifdef HAVE_LUTIMES + { UTIME_TO_TIMEVAL; return lutimes(path, time); + } #endif } @@ -4828,7 +5143,15 @@ utime_nofollow_symlinks(utime_t *ut, const char *path) static int utime_default(utime_t *ut, const char *path) { -#ifdef HAVE_UTIMENSAT +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + if (HAVE_UTIMENSAT_RUNTIME) { + UTIME_TO_TIMESPEC; + return utimensat(DEFAULT_DIR_FD, path, time, 0); + } else { + UTIME_TO_TIMEVAL; + return utimes(path, time); + } +#elif defined(HAVE_UTIMENSAT) UTIME_TO_TIMESPEC; return utimensat(DEFAULT_DIR_FD, path, time, 0); #elif defined(HAVE_UTIMES) @@ -5037,9 +5360,10 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, #endif #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) - if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) + if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { result = utime_dir_fd(&utime, dir_fd, path->narrow, follow_symlinks); - else + + } else #endif #if defined(HAVE_FUTIMES) || defined(HAVE_FUTIMENS) @@ -5052,6 +5376,14 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, Py_END_ALLOW_THREADS +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + /* See utime_dir_fd implementation */ + if (result == -1 && errno == ENOSYS) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result < 0) { /* see previous comment about not putting filename in error here */ posix_error(); @@ -5450,6 +5782,9 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg } if (setsid) { +#ifdef HAVE_POSIX_SPAWN_SETSID_RUNTIME + if (HAVE_POSIX_SPAWN_SETSID_RUNTIME) { +#endif #ifdef POSIX_SPAWN_SETSID all_flags |= POSIX_SPAWN_SETSID; #elif defined(POSIX_SPAWN_SETSID_NP) @@ -5458,6 +5793,14 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg argument_unavailable_error(func_name, "setsid"); return -1; #endif + +#ifdef HAVE_POSIX_SPAWN_SETSID_RUNTIME + } else { + argument_unavailable_error(func_name, "setsid"); + return -1; + } +#endif /* HAVE_POSIX_SPAWN_SETSID_RUNTIME */ + } if (setsigmask) { @@ -8068,16 +8411,30 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) #if defined(HAVE_READLINK) char buffer[MAXPATHLEN+1]; ssize_t length; +#ifdef HAVE_READLINKAT + int readlinkat_unavailable = 0; +#endif Py_BEGIN_ALLOW_THREADS #ifdef HAVE_READLINKAT - if (dir_fd != DEFAULT_DIR_FD) - length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); - else + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_READLINKAT_RUNTIME) { + length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); + } else { + readlinkat_unavailable = 1; + } + } else #endif length = readlink(path->narrow, buffer, MAXPATHLEN); Py_END_ALLOW_THREADS +#ifdef HAVE_READLINKAT + if (readlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (length < 0) { return path_error(path); } @@ -8273,6 +8630,9 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, static int windows_has_symlink_unprivileged_flag = TRUE; #else int result; +#ifdef HAVE_SYMLINKAT + int symlinkat_unavailable = 0; +#endif #endif if (PySys_Audit("os.symlink", "OOi", src->object, dst->object, @@ -8335,14 +8695,25 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } Py_BEGIN_ALLOW_THREADS -#if HAVE_SYMLINKAT - if (dir_fd != DEFAULT_DIR_FD) - result = symlinkat(src->narrow, dir_fd, dst->narrow); - else +#ifdef HAVE_SYMLINKAT + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_SYMLINKAT_RUNTIME) { + result = symlinkat(src->narrow, dir_fd, dst->narrow); + } else { + symlinkat_unavailable = 1; + } + } else #endif result = symlink(src->narrow, dst->narrow); Py_END_ALLOW_THREADS +#ifdef HAVE_SYMLINKAT + if (symlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif @@ -8613,6 +8984,9 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) { int fd; int async_err = 0; +#ifdef HAVE_OPENAT + int openat_unavailable = 0; +#endif #ifdef O_CLOEXEC int *atomic_flag_works = &_Py_open_cloexec_works; @@ -8637,9 +9011,15 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) fd = _wopen(path->wide, flags, mode); #else #ifdef HAVE_OPENAT - if (dir_fd != DEFAULT_DIR_FD) - fd = openat(dir_fd, path->narrow, flags, mode); - else + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_OPENAT_RUNTIME) { + fd = openat(dir_fd, path->narrow, flags, mode); + + } else { + openat_unavailable = 1; + fd = -1; + } + } else #endif /* HAVE_OPENAT */ fd = open(path->narrow, flags, mode); #endif /* !MS_WINDOWS */ @@ -8647,6 +9027,13 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); _Py_END_SUPPRESS_IPH +#ifdef HAVE_OPENAT + if (openat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return -1; + } +#endif + if (fd < 0) { if (!async_err) PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path->object); @@ -9229,12 +9616,25 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else do { +#ifdef __APPLE__ +/* This entire function will be removed from the module dict when the API + * is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +#endif Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH n = preadv(fd, iov, cnt, offset); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif iov_cleanup(iov, buf, cnt); @@ -9856,6 +10256,15 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else + +#ifdef __APPLE__ +/* This entire function will be removed from the module dict when the API + * is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +#endif do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH @@ -9863,6 +10272,11 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif iov_cleanup(iov, buf, cnt); @@ -10752,13 +11166,6 @@ os_statvfs_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS #ifdef HAVE_FSTATVFS if (path->fd != -1) { -#ifdef __APPLE__ - /* handle weak-linking on Mac OS X 10.3 */ - if (fstatvfs == NULL) { - fd_specified("statvfs", path->fd); - return NULL; - } -#endif result = fstatvfs(path->fd, &st); } else @@ -12832,12 +13239,17 @@ DirEntry_fetch_stat(PyObject *module, DirEntry *self, int follow_symlinks) const char *path = PyBytes_AS_STRING(ub); if (self->dir_fd != DEFAULT_DIR_FD) { #ifdef HAVE_FSTATAT + if (HAVE_FSTATAT_RUNTIME) { result = fstatat(self->dir_fd, path, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); -#else + } else + +#endif /* HAVE_FSTATAT */ + { + Py_DECREF(ub); PyErr_SetString(PyExc_NotImplementedError, "can't fetch stat"); return NULL; -#endif /* HAVE_FSTATAT */ + } } else #endif @@ -13623,7 +14035,8 @@ os_scandir_impl(PyObject *module, path_t *path) #else /* POSIX */ errno = 0; #ifdef HAVE_FDOPENDIR - if (path->fd != -1) { + if (iterator->path.fd != -1) { + if (HAVE_FDOPENDIR_RUNTIME) { /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(path->fd); if (fd == -1) @@ -13632,6 +14045,11 @@ os_scandir_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS iterator->dirp = fdopendir(fd); Py_END_ALLOW_THREADS + } else { + PyErr_SetString(PyExc_TypeError, + "scandir: path should be string, bytes, os.PathLike or None, not int"); + return NULL; + } } else #endif @@ -14702,137 +15120,210 @@ all_ins(PyObject *m) } -static const char * const have_functions[] = { + +#define PROBE(name, test) \ + static int name(void) \ + { \ + if (test) { \ + return 1; \ + } else { \ + return 0; \ + } \ + } + +#ifdef HAVE_FSTATAT +PROBE(probe_fstatat, HAVE_FSTATAT_RUNTIME) +#endif + +#ifdef HAVE_FACCESSAT +PROBE(probe_faccessat, HAVE_FACCESSAT_RUNTIME) +#endif + +#ifdef HAVE_FCHMODAT +PROBE(probe_fchmodat, HAVE_FCHMODAT_RUNTIME) +#endif + +#ifdef HAVE_FCHOWNAT +PROBE(probe_fchownat, HAVE_FCHOWNAT_RUNTIME) +#endif + +#ifdef HAVE_LINKAT +PROBE(probe_linkat, HAVE_LINKAT_RUNTIME) +#endif + +#ifdef HAVE_FDOPENDIR +PROBE(probe_fdopendir, HAVE_FDOPENDIR_RUNTIME) +#endif + +#ifdef HAVE_MKDIRAT +PROBE(probe_mkdirat, HAVE_MKDIRAT_RUNTIME) +#endif + +#ifdef HAVE_RENAMEAT +PROBE(probe_renameat, HAVE_RENAMEAT_RUNTIME) +#endif + +#ifdef HAVE_UNLINKAT +PROBE(probe_unlinkat, HAVE_UNLINKAT_RUNTIME) +#endif + +#ifdef HAVE_OPENAT +PROBE(probe_openat, HAVE_OPENAT_RUNTIME) +#endif + +#ifdef HAVE_READLINKAT +PROBE(probe_readlinkat, HAVE_READLINKAT_RUNTIME) +#endif + +#ifdef HAVE_SYMLINKAT +PROBE(probe_symlinkat, HAVE_SYMLINKAT_RUNTIME) +#endif + +#ifdef HAVE_FUTIMENS +PROBE(probe_futimens, HAVE_FUTIMENS_RUNTIME) +#endif + +#ifdef HAVE_UTIMENSAT +PROBE(probe_utimensat, HAVE_UTIMENSAT_RUNTIME) +#endif + + + + +static const struct have_function { + const char * const label; + int (*probe)(void); +} have_functions[] = { #ifdef HAVE_FACCESSAT - "HAVE_FACCESSAT", + { "HAVE_FACCESSAT", probe_faccessat }, #endif #ifdef HAVE_FCHDIR - "HAVE_FCHDIR", + { "HAVE_FCHDIR", NULL }, #endif #ifdef HAVE_FCHMOD - "HAVE_FCHMOD", + { "HAVE_FCHMOD", NULL }, #endif #ifdef HAVE_FCHMODAT - "HAVE_FCHMODAT", + { "HAVE_FCHMODAT", probe_fchmodat }, #endif #ifdef HAVE_FCHOWN - "HAVE_FCHOWN", + { "HAVE_FCHOWN", NULL }, #endif #ifdef HAVE_FCHOWNAT - "HAVE_FCHOWNAT", + { "HAVE_FCHOWNAT", probe_fchownat }, #endif #ifdef HAVE_FEXECVE - "HAVE_FEXECVE", + { "HAVE_FEXECVE", NULL }, #endif #ifdef HAVE_FDOPENDIR - "HAVE_FDOPENDIR", + { "HAVE_FDOPENDIR", probe_fdopendir }, #endif #ifdef HAVE_FPATHCONF - "HAVE_FPATHCONF", + { "HAVE_FPATHCONF", NULL }, #endif #ifdef HAVE_FSTATAT - "HAVE_FSTATAT", + { "HAVE_FSTATAT", probe_fstatat }, #endif #ifdef HAVE_FSTATVFS - "HAVE_FSTATVFS", + { "HAVE_FSTATVFS", NULL }, #endif #if defined HAVE_FTRUNCATE || defined MS_WINDOWS - "HAVE_FTRUNCATE", + { "HAVE_FTRUNCATE", NULL }, #endif #ifdef HAVE_FUTIMENS - "HAVE_FUTIMENS", + { "HAVE_FUTIMENS", probe_futimens }, #endif #ifdef HAVE_FUTIMES - "HAVE_FUTIMES", + { "HAVE_FUTIMES", NULL }, #endif #ifdef HAVE_FUTIMESAT - "HAVE_FUTIMESAT", + { "HAVE_FUTIMESAT", NULL }, #endif #ifdef HAVE_LINKAT - "HAVE_LINKAT", + { "HAVE_LINKAT", probe_linkat }, #endif #ifdef HAVE_LCHFLAGS - "HAVE_LCHFLAGS", + { "HAVE_LCHFLAGS", NULL }, #endif #ifdef HAVE_LCHMOD - "HAVE_LCHMOD", + { "HAVE_LCHMOD", NULL }, #endif #ifdef HAVE_LCHOWN - "HAVE_LCHOWN", + { "HAVE_LCHOWN", NULL }, #endif #ifdef HAVE_LSTAT - "HAVE_LSTAT", + { "HAVE_LSTAT", NULL }, #endif #ifdef HAVE_LUTIMES - "HAVE_LUTIMES", + { "HAVE_LUTIMES", NULL }, #endif #ifdef HAVE_MEMFD_CREATE - "HAVE_MEMFD_CREATE", + { "HAVE_MEMFD_CREATE", NULL }, #endif #ifdef HAVE_MKDIRAT - "HAVE_MKDIRAT", + { "HAVE_MKDIRAT", probe_mkdirat }, #endif #ifdef HAVE_MKFIFOAT - "HAVE_MKFIFOAT", + { "HAVE_MKFIFOAT", NULL }, #endif #ifdef HAVE_MKNODAT - "HAVE_MKNODAT", + { "HAVE_MKNODAT", NULL }, #endif #ifdef HAVE_OPENAT - "HAVE_OPENAT", + { "HAVE_OPENAT", probe_openat }, #endif #ifdef HAVE_READLINKAT - "HAVE_READLINKAT", + { "HAVE_READLINKAT", probe_readlinkat }, #endif #ifdef HAVE_RENAMEAT - "HAVE_RENAMEAT", + { "HAVE_RENAMEAT", probe_renameat }, #endif #ifdef HAVE_SYMLINKAT - "HAVE_SYMLINKAT", + { "HAVE_SYMLINKAT", probe_symlinkat }, #endif #ifdef HAVE_UNLINKAT - "HAVE_UNLINKAT", + { "HAVE_UNLINKAT", probe_unlinkat }, #endif #ifdef HAVE_UTIMENSAT - "HAVE_UTIMENSAT", + { "HAVE_UTIMENSAT", probe_utimensat }, #endif #ifdef MS_WINDOWS - "MS_WINDOWS", + { "MS_WINDOWS", NULL }, #endif - NULL + { NULL, NULL } }; @@ -14841,6 +15332,23 @@ posixmodule_exec(PyObject *m) { _posixstate *state = get_posix_state(m); +#if defined(HAVE_PWRITEV) + if (HAVE_PWRITEV_RUNTIME) {} else { + PyObject* dct = PyModule_GetDict(m); + + if (dct == NULL) { + return -1; + } + + if (PyDict_DelItemString(dct, "pwritev") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "preadv") == -1) { + PyErr_Clear(); + } + } +#endif + /* Initialize environ dictionary */ PyObject *v = convertenviron(); Py_XINCREF(v); @@ -14953,44 +15461,6 @@ posixmodule_exec(PyObject *m) PyModule_AddObject(m, "uname_result", (PyObject *)UnameResultType); state->UnameResultType = (PyObject *)UnameResultType; -#ifdef __APPLE__ - /* - * Step 2 of weak-linking support on Mac OS X. - * - * The code below removes functions that are not available on the - * currently active platform. - * - * This block allow one to use a python binary that was build on - * OSX 10.4 on OSX 10.3, without losing access to new APIs on - * OSX 10.4. - */ -#ifdef HAVE_FSTATVFS - if (fstatvfs == NULL) { - if (PyObject_DelAttrString(m, "fstatvfs") == -1) { - return -1; - } - } -#endif /* HAVE_FSTATVFS */ - -#ifdef HAVE_STATVFS - if (statvfs == NULL) { - if (PyObject_DelAttrString(m, "statvfs") == -1) { - return -1; - } - } -#endif /* HAVE_STATVFS */ - -# ifdef HAVE_LCHOWN - if (lchown == NULL) { - if (PyObject_DelAttrString(m, "lchown") == -1) { - return -1; - } - } -#endif /* HAVE_LCHOWN */ - - -#endif /* __APPLE__ */ - if ((state->billion = PyLong_FromLong(1000000000)) == NULL) return -1; #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -15020,14 +15490,17 @@ posixmodule_exec(PyObject *m) if (!list) { return -1; } - for (const char * const *trace = have_functions; *trace; trace++) { - PyObject *unicode = PyUnicode_DecodeASCII(*trace, strlen(*trace), NULL); + for (const struct have_function *trace = have_functions; trace->label; trace++) { + PyObject *unicode; + if (trace->probe && !trace->probe()) continue; + unicode = PyUnicode_DecodeASCII(trace->label, strlen(trace->label), NULL); if (!unicode) return -1; if (PyList_Append(list, unicode)) return -1; Py_DECREF(unicode); } + PyModule_AddObject(m, "_have_functions", list); return 0; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index eb192c5..80eab30 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -51,6 +51,15 @@ #define _Py_tzname tzname #endif +#if defined(__APPLE__ ) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +# endif +#endif +#ifndef HAVE_CLOCK_GETTIME_RUNTIME +# define HAVE_CLOCK_GETTIME_RUNTIME 1 +#endif + #define SEC_TO_NS (1000 * 1000 * 1000) /* Forward declarations */ @@ -149,6 +158,16 @@ perf_counter(_Py_clock_info_t *info) } #ifdef HAVE_CLOCK_GETTIME + +#ifdef __APPLE__ +/* + * The clock_* functions will be removed from the module + * dict entirely when the C API is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#endif + static PyObject * time_clock_gettime(PyObject *self, PyObject *args) { @@ -297,6 +316,11 @@ PyDoc_STRVAR(clock_getres_doc, "clock_getres(clk_id) -> floating point number\n\ \n\ Return the resolution (precision) of the specified clock clk_id."); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif /* HAVE_CLOCK_GETRES */ #ifdef HAVE_PTHREAD_GETCPUCLOCKID @@ -1162,31 +1186,35 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #if defined(HAVE_CLOCK_GETTIME) \ && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) struct timespec ts; + + if (HAVE_CLOCK_GETTIME_RUNTIME) { + #ifdef CLOCK_PROF - const clockid_t clk_id = CLOCK_PROF; - const char *function = "clock_gettime(CLOCK_PROF)"; + const clockid_t clk_id = CLOCK_PROF; + const char *function = "clock_gettime(CLOCK_PROF)"; #else - const clockid_t clk_id = CLOCK_PROCESS_CPUTIME_ID; - const char *function = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)"; + const clockid_t clk_id = CLOCK_PROCESS_CPUTIME_ID; + const char *function = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)"; #endif - if (clock_gettime(clk_id, &ts) == 0) { - if (info) { - struct timespec res; - info->implementation = function; - info->monotonic = 1; - info->adjustable = 0; - if (clock_getres(clk_id, &res)) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; + if (clock_gettime(clk_id, &ts) == 0) { + if (info) { + struct timespec res; + info->implementation = function; + info->monotonic = 1; + info->adjustable = 0; + if (clock_getres(clk_id, &res)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; } - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - } - if (_PyTime_FromTimespec(tp, &ts) < 0) { - return -1; + if (_PyTime_FromTimespec(tp, &ts) < 0) { + return -1; + } + return 0; } - return 0; } #endif @@ -1390,6 +1418,16 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #elif defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) #define HAVE_THREAD_TIME + +#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +static int +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) + __attribute__((availability(macos, introduced=10.12))) + __attribute__((availability(ios, introduced=10.0))) + __attribute__((availability(tvos, introduced=10.0))) + __attribute__((availability(watchos, introduced=3.0))); +#endif + static int _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) { @@ -1421,6 +1459,15 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif #ifdef HAVE_THREAD_TIME +#ifdef __APPLE__ +/* + * The clock_* functions will be removed from the module + * dict entirely when the C API is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#endif + static PyObject * time_thread_time(PyObject *self, PyObject *unused) { @@ -1451,6 +1498,11 @@ PyDoc_STRVAR(thread_time_ns_doc, \n\ Thread time for profiling as nanoseconds:\n\ sum of the kernel and user-space CPU time."); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif @@ -1500,9 +1552,19 @@ time_get_clock_info(PyObject *self, PyObject *args) } #ifdef HAVE_THREAD_TIME else if (strcmp(name, "thread_time") == 0) { - if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + +#ifdef __APPLE__ + if (HAVE_CLOCK_GETTIME_RUNTIME) { +#endif + if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + return NULL; + } +#ifdef __APPLE__ + } else { + PyErr_SetString(PyExc_ValueError, "unknown clock"); return NULL; } +#endif } #endif else { @@ -1783,68 +1845,116 @@ if it is -1, mktime() should guess based on the date and time.\n"); static int time_exec(PyObject *module) { +#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) + if (HAVE_CLOCK_GETTIME_RUNTIME) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + if (dct == NULL) { + return -1; + } + + if (PyDict_DelItemString(dct, "clock_gettime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_getres") == -1) { + PyErr_Clear(); + } + } +#endif +#if defined(__APPLE__) && defined(HAVE_THREAD_TIME) + if (HAVE_CLOCK_GETTIME_RUNTIME) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + + if (PyDict_DelItemString(dct, "thread_time") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { + PyErr_Clear(); + } + } +#endif /* Set, or reset, module variables like time.timezone */ if (init_timezone(module) < 0) { return -1; } #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) + if (HAVE_CLOCK_GETTIME_RUNTIME) { #ifdef CLOCK_REALTIME - if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { + return -1; + } #endif + #ifdef CLOCK_MONOTONIC - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { - return -1; - } + + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { + return -1; + } + #endif #ifdef CLOCK_MONOTONIC_RAW - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { + return -1; + } #endif + #ifdef CLOCK_HIGHRES - if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { + return -1; + } #endif #ifdef CLOCK_PROCESS_CPUTIME_ID - if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { + return -1; + } #endif + #ifdef CLOCK_THREAD_CPUTIME_ID - if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { + return -1; + } #endif #ifdef CLOCK_PROF - if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { + return -1; + } #endif #ifdef CLOCK_BOOTTIME - if (PyModule_AddIntMacro(module, CLOCK_BOOTTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_BOOTTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_TAI - if (PyModule_AddIntMacro(module, CLOCK_TAI) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_TAI) < 0) { + return -1; + } #endif #ifdef CLOCK_UPTIME - if (PyModule_AddIntMacro(module, CLOCK_UPTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_UPTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_UPTIME_RAW - if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { - return -1; - } + + if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { + return -1; + } #endif + } #endif /* defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) */ diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 4736930..a212f69 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -25,6 +25,16 @@ # include <sanitizer/msan_interface.h> #endif +#if defined(__APPLE__) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME __builtin_available(macOS 10.12, iOS 10.10, tvOS 10.0, watchOS 3.0, *) +# endif +#endif +#ifndef HAVE_GETENTRYPY_GETRANDOM_RUNTIME +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME 1 +#endif + + #ifdef Py_DEBUG int _Py_HashSecret_Initialized = 0; #else @@ -208,6 +218,16 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) error. getentropy() is retried if it failed with EINTR: interrupted by a signal. */ + +#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +static int +py_getentropy(char *buffer, Py_ssize_t size, int raise) + __attribute__((availability(macos,introduced=10.12))) + __attribute__((availability(ios,introduced=10.0))) + __attribute__((availability(tvos,introduced=10.0))) + __attribute__((availability(watchos,introduced=3.0))); +#endif + static int py_getentropy(char *buffer, Py_ssize_t size, int raise) { @@ -498,19 +518,21 @@ pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise) #else #if defined(PY_GETRANDOM) || defined(PY_GETENTROPY) + if (HAVE_GETENTRYPY_GETRANDOM_RUNTIME) { #ifdef PY_GETRANDOM - res = py_getrandom(buffer, size, blocking, raise); + res = py_getrandom(buffer, size, blocking, raise); #else - res = py_getentropy(buffer, size, raise); + res = py_getentropy(buffer, size, raise); #endif - if (res < 0) { - return -1; - } - if (res == 1) { - return 0; - } - /* getrandom() or getentropy() function is not available: failed with - ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ + if (res < 0) { + return -1; + } + if (res == 1) { + return 0; + } + /* getrandom() or getentropy() function is not available: failed with + ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ + } /* end of availability block */ #endif return dev_urandom(buffer, size, raise); diff --git a/Python/pytime.c b/Python/pytime.c index b121b43..89d63e0 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -5,6 +5,12 @@ #if defined(__APPLE__) #include <mach/mach_time.h> /* mach_absolute_time(), mach_timebase_info() */ + +#if defined(__APPLE__) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +# endif +#endif #endif #define _PyTime_check_mul_overflow(a, b) \ @@ -683,15 +689,22 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) #else /* MS_WINDOWS */ int err; -#ifdef HAVE_CLOCK_GETTIME +#if defined(HAVE_CLOCK_GETTIME) struct timespec ts; -#else +#endif + +#if !defined(HAVE_CLOCK_GETTIME) || defined(__APPLE__) struct timeval tv; #endif assert(info == NULL || raise); #ifdef HAVE_CLOCK_GETTIME + +#ifdef HAVE_CLOCK_GETTIME_RUNTIME + if (HAVE_CLOCK_GETTIME_RUNTIME) { +#endif + err = clock_gettime(CLOCK_REALTIME, &ts); if (err) { if (raise) { @@ -715,7 +728,14 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->resolution = 1e-9; } } -#else /* HAVE_CLOCK_GETTIME */ + +#ifdef HAVE_CLOCK_GETTIME_RUNTIME + } else { +#endif + +#endif + +#if !defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_GETTIME_RUNTIME) /* test gettimeofday() */ err = gettimeofday(&tv, (struct timezone *)NULL); @@ -735,6 +755,11 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->monotonic = 0; info->adjustable = 1; } + +#if defined(HAVE_CLOCK_GETTIME_RUNTIME) && defined(HAVE_CLOCK_GETTIME) + } /* end of availibity block */ +#endif + #endif /* !HAVE_CLOCK_GETTIME */ #endif /* !MS_WINDOWS */ return 0; @@ -1510,8 +1510,8 @@ Optional Packages: specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", - "intel-64", or "all") see Mac/README.rst + ("universal2", "32-bit", "64-bit", "3-way", "intel", + "intel-32", "intel-64", or "all") see Mac/README.rst --with-framework-name=FRAMEWORK specify the name for the python framework on macOS only valid when --enable-framework is set. see @@ -6954,7 +6954,7 @@ fi -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= @@ -7481,6 +7481,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -7502,7 +7507,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 ;; esac @@ -9359,7 +9364,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac else @@ -9369,9 +9374,12 @@ fi ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac @@ -12014,6 +12022,31 @@ $as_echo "no" >&6; } fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _dyld_shared_cache_contains_path" >&5 +$as_echo_n "checking for _dyld_shared_cache_contains_path... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <mach-o/dyld.h> +int +main () +{ +void *x=_dyld_shared_cache_contains_path + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH 1" >>confdefs.h + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking for memfd_create" >&5 $as_echo_n "checking for memfd_create... " >&6; } diff --git a/configure.ac b/configure.ac index 29acb8e..c968d14 100644 --- a/configure.ac +++ b/configure.ac @@ -218,7 +218,7 @@ AC_ARG_WITH(universal-archs, AS_HELP_STRING([--with-universal-archs=ARCH], [specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") + ("universal2", "32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") see Mac/README.rst]), [ UNIVERSAL_ARCHS="$withval" @@ -1587,7 +1587,7 @@ AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= AC_SUBST(UNIVERSAL_ARCH_FLAGS) @@ -1888,6 +1888,11 @@ yes) LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -1909,7 +1914,7 @@ yes) ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - AC_MSG_ERROR([proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way]) + AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) ;; esac @@ -2486,7 +2491,7 @@ case $ac_sys_system/$ac_sys_release in MACOSX_DEFAULT_ARCH="ppc" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac else @@ -2496,9 +2501,12 @@ case $ac_sys_system/$ac_sys_release in ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac @@ -3777,6 +3785,12 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no) ]) +AC_MSG_CHECKING(for _dyld_shared_cache_contains_path) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <mach-o/dyld.h>]], [[void *x=_dyld_shared_cache_contains_path]])], + [AC_DEFINE(HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH, 1, Define if you have the '_dyld_shared_cache_contains_path' function.) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no) +]) AC_MSG_CHECKING(for memfd_create) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/pyconfig.h.in b/pyconfig.h.in index c9589cd..f39858d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -778,6 +778,9 @@ /* Define if you have the 'prlimit' functions. */ #undef HAVE_PRLIMIT +/* Define if you have the '_dyld_shared_cache_contains_path' function. */ +#undef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH + /* Define to 1 if you have the <process.h> header file. */ #undef HAVE_PROCESS_H @@ -216,6 +216,13 @@ def is_macosx_sdk_path(path): or path.startswith('/Library/') ) +def grep_headers_for(function, headers): + for header in headers: + with open(header, 'r', errors='surrogateescape') as f: + if function in f.read(): + return True + return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -2078,43 +2085,17 @@ class PyBuildExt(build_ext): library_dirs=added_lib_dirs)) return True - def configure_ctypes_darwin(self, ext): - # Darwin (OS X) uses preconfigured files, in - # the Modules/_ctypes/libffi_osx directory. - ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', - '_ctypes', 'libffi_osx')) - sources = [os.path.join(ffi_srcdir, p) - for p in ['ffi.c', - 'x86/darwin64.S', - 'x86/x86-darwin.S', - 'x86/x86-ffi_darwin.c', - 'x86/x86-ffi64.c', - 'powerpc/ppc-darwin.S', - 'powerpc/ppc-darwin_closure.S', - 'powerpc/ppc-ffi_darwin.c', - 'powerpc/ppc64-darwin_closure.S', - ]] - - # Add .S (preprocessed assembly) to C compiler source extensions. - self.compiler.src_extensions.append('.S') - - include_dirs = [os.path.join(ffi_srcdir, 'include'), - os.path.join(ffi_srcdir, 'powerpc')] - ext.include_dirs.extend(include_dirs) - ext.sources.extend(sources) - return True - def configure_ctypes(self, ext): - if not self.use_system_libffi: - if MACOS: - return self.configure_ctypes_darwin(ext) - print('INFO: Could not locate ffi libs and/or headers') - return False return True def detect_ctypes(self): # Thomas Heller's _ctypes module - self.use_system_libffi = False + + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS): + self.use_system_libffi = True + else: + self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") + include_dirs = [] extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] extra_link_args = [] @@ -2127,11 +2108,9 @@ class PyBuildExt(build_ext): if MACOS: sources.append('_ctypes/malloc_closure.c') - sources.append('_ctypes/darwin/dlfcn_simple.c') + extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') - # XXX Is this still needed? - # extra_link_args.extend(['-read_only_relocs', 'warning']) elif HOST_PLATFORM == 'sunos5': # XXX This shouldn't be necessary; it appears that some @@ -2161,31 +2140,48 @@ class PyBuildExt(build_ext): sources=['_ctypes/_ctypes_test.c'], libraries=['m'])) + ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") + ffi_lib = None + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): - return - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') - - ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] - if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) - if ffi_inc is not None: - ffi_h = ffi_inc[0] + '/ffi.h' + ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") + + if not ffi_inc: + if os.path.exists(ffi_in_sdk): + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + found = find_file('ffi.h', [], ffi_inc_dirs) + if found: + ffi_inc = found[0] + if ffi_inc: + ffi_h = ffi_inc + '/ffi.h' if not os.path.exists(ffi_h): ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: for lib_name in ('ffi', 'ffi_pic'): if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break if ffi_inc and ffi_lib: - ext.include_dirs.extend(ffi_inc) + ffi_headers = glob(os.path.join(ffi_inc, '*.h')) + if grep_headers_for('ffi_prep_cif_var', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") + if grep_headers_for('ffi_prep_closure_loc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") + if grep_headers_for('ffi_closure_alloc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True |