From e9f5038505720300b9a9e9311cbd7fd6a1d8a393 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 26 Oct 2023 14:48:16 -0600 Subject: Fix library literal prefix code * Beef up _stripixes unittests * rename LIBLITERAL to LIBLITERALPREFIX * Initialize in Platform code, takes an error if not set when StringSubber.expand() is called with a string containing LIBLITERALPREFIX * Fix some bad typing markup. * Add docs Signed-off-by: Mats Wichmann --- CHANGES.txt | 3 + RELEASE.txt | 3 + SCons/Defaults.py | 4 +- SCons/Defaults.xml | 137 +++++++++++++++++++++++++++++++++----------- SCons/Environment.py | 16 ++++-- SCons/EnvironmentTests.py | 40 ++++++++----- SCons/Platform/Platform.xml | 12 ++-- SCons/Platform/cygwin.py | 5 +- SCons/Platform/os2.py | 5 +- SCons/Platform/posix.py | 5 +- SCons/Platform/win32.py | 5 +- SCons/Subst.py | 10 ++-- SCons/Tool/dmd.py | 2 +- SCons/Tool/ldc.py | 2 +- SCons/Tool/link.py | 2 +- test/Libs/LIBLITERAL.py | 8 +-- 16 files changed, 180 insertions(+), 79 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b9c4144..a0e40d0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -188,6 +188,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER avoid the GNU linker issuing warnings for its absence. - Eliminate more http: references (mostly in comments/docstrings where they really weren't harmful). A few links labeled dead with no alt. + - Add a LIBLITERALPREFIX variable which can be set to the linker's + prefix for considering a library argument unmodified (e.g. for the + GNU linker, the ':' in '-l:libfoo.a'). From Jonathon Reinhart: diff --git a/RELEASE.txt b/RELEASE.txt index 764cc40..a05527c 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -23,6 +23,9 @@ NEW FUNCTIONALITY - MSVC: If necessary, automatically define VSCMD_SKIP_SENDTELEMETRY for VS2019 and later on arm64 hosts when using an arm (32-bit) build of python to prevent a powershell error pop-up window (powershell dll not found). +- Add a LIBLITERALPREFIX variable which can be set to the linker's + prefix for considering a library argument unmodified (e.g. for the + GNU linker, the ':' in '-l:libfoo.a'). DEPRECATED FUNCTIONALITY ------------------------ diff --git a/SCons/Defaults.py b/SCons/Defaults.py index 49e9cc4..32e9da3 100644 --- a/SCons/Defaults.py +++ b/SCons/Defaults.py @@ -459,8 +459,8 @@ def _stripixes( prefix: str, items, suffix: str, - stripprefixes: str, - stripsuffixes: str, + stripprefixes: List[str], + stripsuffixes: List[str], env, literal_prefix: str = "", c: Callable[[list], list] = None, diff --git a/SCons/Defaults.xml b/SCons/Defaults.xml index b892f2e..3ff7359 100644 --- a/SCons/Defaults.xml +++ b/SCons/Defaults.xml @@ -250,7 +250,7 @@ as the result will be non-portable and the directories will not be searched by the dependency scanner. &cv-CPPPATH; should be a list of path strings, or a single string, not a pathname list joined by -Python's os.sep. +Python's os.pathsep. @@ -473,7 +473,7 @@ when the &cv-link-_LIBDIRFLAGS; variable is automatically generated. An automatically-generated construction variable containing the linker command-line options for specifying libraries to be linked with the resulting target. -The value of &cv-link-_LIBFLAGS; is created +The value of &cv-_LIBFLAGS; is created by respectively prepending and appending &cv-link-LIBLINKPREFIX; and &cv-link-LIBLINKSUFFIX; to each filename in &cv-link-LIBS;. @@ -510,7 +510,7 @@ The list of directories that will be searched for libraries specified by the &cv-link-LIBS; &consvar;. &cv-LIBPATH; should be a list of path strings, or a single string, not a pathname list joined by -Python's os.sep. +Python's os.pathsep. +For each &Builder; call that causes linking with libraries, +&SCons; will add the libraries in the setting of &cv-LIBS; +in effect at that moment to the dependecy graph +as dependencies of the target being generated. + + + +The library list will transformed to command line +arguments through the automatically-generated +&cv-link-_LIBFLAGS; &consvar; which is constructed by respectively prepending and appending the values of the -&cv-LIBLINKPREFIX; and &cv-LIBLINKSUFFIX; &consvars; -to each library name in &cv-LIBS;. -Library name strings should not include a -path component, instead the compiler will be -directed to look for libraries in the paths -specified by &cv-link-LIBPATH;. +&cv-link-LIBLINKPREFIX; and &cv-link-LIBLINKSUFFIX; &consvars; +to each library name. -Any command lines you define that need -the &cv-LIBS; library list should -include &cv-_LIBFLAGS;: +Any command lines you define yourself that need +the libraries from &cv-LIBS; should include &cv-_LIBFLAGS; +(as well as &cv-link-_LIBDIRFLAGS;) +rather than &cv-LIBS;. +For example: env = Environment(LINKCOM="my_linker $_LIBDIRFLAGS $_LIBFLAGS -o $TARGET $SOURCE") + + + + -If you add a -File -object to the -&cv-LIBS; -list, the name of that file will be added to -&cv-_LIBFLAGS;, -and thus to the link line, as-is, without -&cv-LIBLINKPREFIX; -or -&cv-LIBLINKSUFFIX;. -For example: +If the linker supports command line syntax directing +that the argument specifying a library should be +searched for literally (without modification), +&cv-LIBLITERALPREFIX; can be set to that indicator. +For example, the GNU linker follows this rule: + +-l:foo searches the library path +for a filename called foo, +without converting it to +libfoo.so or +libfoo.a. + +If &cv-LIBLITERALPREFIX; is set, +&SCons; will not transform a string-valued entry in +&cv-link-LIBS; that starts with that string. +The entry will still be surrounded with +&cv-link-LIBLINKPREFIX; and &cv-link-LIBLINKSUFFIX; +on the command line. +This is useful, for example, +in directing that a static library +be used when both a static and dynamic library are available +and linker policy is to prefer dynamic libraries. +Compared to the example in &cv-link-LIBS;, - -env.Append(LIBS=File('/tmp/mylib.so')) +env.Append(LIBS=":libmylib.a") - -In all cases, scons will add dependencies from the executable program to -all the libraries in this list. +will let the linker select that specific (static) +library name if found in the library search path. +This differs from using a +File object +to specify the static library, +as the latter bypasses the library search path entirely. diff --git a/SCons/Environment.py b/SCons/Environment.py index 6327d86..64d38b0 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -37,6 +37,7 @@ import re import shlex from collections import UserDict, deque from subprocess import PIPE, DEVNULL +from typing import Optional import SCons.Action import SCons.Builder @@ -679,7 +680,7 @@ class SubstitutionEnvironment: def lvars(self): return {} - def subst(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: bool=False): + def subst(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: Optional[dict] = None): """Recursively interpolates construction variables from the Environment into the specified string, returning the expanded result. Construction variables are specified by a $ prefix @@ -705,9 +706,11 @@ class SubstitutionEnvironment: nkw[k] = v return nkw - def subst_list(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: bool=False): - """Calls through to SCons.Subst.scons_subst_list(). See - the documentation for that function.""" + def subst_list(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: Optional[dict] = None): + """Calls through to SCons.Subst.scons_subst_list(). + + See the documentation for that function. + """ gvars = self.gvars() lvars = self.lvars() lvars['__env__'] = self @@ -716,9 +719,10 @@ class SubstitutionEnvironment: return SCons.Subst.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv, overrides=overrides) def subst_path(self, path, target=None, source=None): - """Substitute a path list, turning EntryProxies into Nodes - and leaving Nodes (and other objects) as-is.""" + """Substitute a path list. + Turns EntryProxies into Nodes, leaving Nodes (and other objects) as-is. + """ if not is_List(path): path = [path] diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index b1b6a0d..c40439a 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -1528,29 +1528,43 @@ def exists(env): # function is in Defaults.py, tested here to use TestEnvironment def test__stripixes(self) -> None: """Test _stripixes()""" - # LIBPREFIXES and LIBSUFFIXES are stripped, - # except if an entry begins with LIBLITERAL + # LIBPREFIXES and LIBSUFFIXES are stripped, except if an entry + # begins with LIBLITERALPREFIX. Check this with and without that + # argument being passed, and whether or not LIBLITERALPREFIX is + # explicitly set. e = self.TestEnvironment( PRE='pre', SUF='suf', LIST=['xxx-a', 'b.yyy', 'zzxxx-c.yyy'], LIBPREFIXES=['xxx-'], LIBSUFFIXES=['.yyy'], - LIBLITERAL='zz', ) - x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__, LIBLITERAL)} $)') - self.assertEqual(x, 'preasuf prebsuf prezzxxx-c.yyysuf') - # Test that setting literal_prefix (in this case LIBLITERAL) - # same as os.pathsep disables the literal protection - e['LIBLITERAL'] = os.pathsep - x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__, LIBLITERAL)} $)') - self.assertEqual(x, 'preasuf prebsuf prezzxxx-csuf') + # e['LIBLITERALPREFIX'] = '' + with self.subTest(): + x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__, LIBLITERALPREFIX)} $)') + self.assertEqual('preasuf prebsuf prezzxxx-csuf', x) + + with self.subTest(): + x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__)} $)') + self.assertEqual('preasuf prebsuf prezzxxx-csuf', x) + + # add it to the env: + e['LIBLITERALPREFIX'] = 'zz' - # Test that setting not settingliteral_prefix doesn't fail - x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__)} $)') - self.assertEqual(x, 'preasuf prebsuf prezzxxx-csuf') + with self.subTest(): + x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__, LIBLITERALPREFIX)} $)') + self.assertEqual('preasuf prebsuf prezzxxx-c.yyysuf', x) + + with self.subTest(): + x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__)} $)') + self.assertEqual('preasuf prebsuf prezzxxx-csuf', x) + # And special case: LIBLITERALPREFIX is the same as os.pathsep: + e['LIBLITERALPREFIX'] = os.pathsep + with self.subTest(): + x = e.subst('$( ${_stripixes(PRE, LIST, SUF, LIBPREFIXES, LIBSUFFIXES,__env__, LIBLITERALPREFIX)} $)') + self.assertEqual('preasuf prebsuf prezzxxx-csuf', x) def test_gvars(self) -> None: diff --git a/SCons/Platform/Platform.xml b/SCons/Platform/Platform.xml index c449fa5..dc9ed79 100644 --- a/SCons/Platform/Platform.xml +++ b/SCons/Platform/Platform.xml @@ -49,7 +49,8 @@ to reflect the names of the libraries they create. -A list of all legal prefixes for library file names. +A list of all legal prefixes for library file names +on the current platform. When searching for library dependencies, SCons will look for files with these prefixes, the base library name, @@ -75,6 +76,7 @@ to reflect the names of the libraries they create. A list of all legal suffixes for library file names. +on the current platform. When searching for library dependencies, SCons will look for files with prefixes from the &cv-link-LIBPREFIXES; list, the base library name, @@ -129,7 +131,7 @@ else: platform argument to &f-link-Environment;). - Should be considered immutable. + Should be considered immutable. &cv-HOST_OS; is not currently used by &SCons;, but the option is reserved to do so in future @@ -177,7 +179,7 @@ else: and x86_64 for 64-bit hosts. - Should be considered immutable. + Should be considered immutable. &cv-HOST_ARCH; is not currently used by other platforms, but the option is reserved to do so in future @@ -305,7 +307,7 @@ an alternate command line so the invoked tool will make use of the contents of the temporary file. If you need to replace the default tempfile object, the callable should take into account the settings of -&cv-link-MAXLINELENGTH;, +&cv-link-MAXLINELENGTH;, &cv-link-TEMPFILEPREFIX;, &cv-link-TEMPFILESUFFIX;, &cv-link-TEMPFILEARGJOIN;, @@ -367,7 +369,7 @@ The directory to create the long-lines temporary file in. -The default argument escape function is +The default argument escape function is SCons.Subst.quote_spaces. If you need to apply extra operations on a command argument (to fix Windows slashes, normalize paths, etc.) diff --git a/SCons/Platform/cygwin.py b/SCons/Platform/cygwin.py index c62a668..2353763 100644 --- a/SCons/Platform/cygwin.py +++ b/SCons/Platform/cygwin.py @@ -47,8 +47,9 @@ def generate(env) -> None: env['PROGSUFFIX'] = '.exe' env['SHLIBPREFIX'] = '' env['SHLIBSUFFIX'] = '.dll' - env['LIBPREFIXES'] = [ '$LIBPREFIX', '$SHLIBPREFIX', '$IMPLIBPREFIX' ] - env['LIBSUFFIXES'] = [ '$LIBSUFFIX', '$SHLIBSUFFIX', '$IMPLIBSUFFIX' ] + env['LIBPREFIXES'] = ['$LIBPREFIX', '$SHLIBPREFIX', '$IMPLIBPREFIX'] + env['LIBSUFFIXES'] = ['$LIBSUFFIX', '$SHLIBSUFFIX', '$IMPLIBSUFFIX'] + env['LIBLITERAPPREFIX'] = ':' env['TEMPFILE'] = TempFileMunge env['TEMPFILEPREFIX'] = '@' env['MAXLINELENGTH'] = 2048 diff --git a/SCons/Platform/os2.py b/SCons/Platform/os2.py index 7394aa8..72bb034 100644 --- a/SCons/Platform/os2.py +++ b/SCons/Platform/os2.py @@ -43,8 +43,9 @@ def generate(env) -> None: env['LIBSUFFIX'] = '.lib' env['SHLIBPREFIX'] = '' env['SHLIBSUFFIX'] = '.dll' - env['LIBPREFIXES'] = '$LIBPREFIX' - env['LIBSUFFIXES'] = [ '$LIBSUFFIX', '$SHLIBSUFFIX' ] + env['LIBPREFIXES'] = ['$LIBPREFIX'] + env['LIBSUFFIXES'] = ['$LIBSUFFIX', '$SHLIBSUFFIX'] + env['LIBLITERAPPREFIX'] = '' env['HOST_OS'] = 'os2' env['HOST_ARCH'] = win32.get_architecture().arch diff --git a/SCons/Platform/posix.py b/SCons/Platform/posix.py index 55b00b4..b655b77 100644 --- a/SCons/Platform/posix.py +++ b/SCons/Platform/posix.py @@ -93,8 +93,9 @@ def generate(env) -> None: env['LIBSUFFIX'] = '.a' env['SHLIBPREFIX'] = '$LIBPREFIX' env['SHLIBSUFFIX'] = '.so' - env['LIBPREFIXES'] = [ '$LIBPREFIX' ] - env['LIBSUFFIXES'] = [ '$LIBSUFFIX', '$SHLIBSUFFIX' ] + env['LIBPREFIXES'] = ['$LIBPREFIX'] + env['LIBSUFFIXES'] = ['$LIBSUFFIX', '$SHLIBSUFFIX'] + env['LIBLITERALPREFIX'] = '' env['HOST_OS'] = 'posix' env['HOST_ARCH'] = platform.machine() env['PSPAWN'] = pspawn diff --git a/SCons/Platform/win32.py b/SCons/Platform/win32.py index da9e574..b145823 100644 --- a/SCons/Platform/win32.py +++ b/SCons/Platform/win32.py @@ -420,8 +420,9 @@ def generate(env): env['LIBSUFFIX'] = '.lib' env['SHLIBPREFIX'] = '' env['SHLIBSUFFIX'] = '.dll' - env['LIBPREFIXES'] = [ '$LIBPREFIX' ] - env['LIBSUFFIXES'] = [ '$LIBSUFFIX' ] + env['LIBPREFIXES'] = ['$LIBPREFIX'] + env['LIBSUFFIXES'] = ['$LIBSUFFIX'] + env['LIBLITERALPREFIX'] = '' env['PSPAWN'] = piped_spawn env['SPAWN'] = spawn env['SHELL'] = cmd_interp diff --git a/SCons/Subst.py b/SCons/Subst.py index 4046ca6..b04ebe5 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -26,6 +26,7 @@ import collections import re from inspect import signature, Parameter +from typing import Optional import SCons.Errors from SCons.Util import is_String, is_Sequence @@ -448,11 +449,12 @@ class StringSubber: This serves as a wrapper for splitting up a string into separate tokens. """ + def sub_match(match): + return self.conv(self.expand(match.group(1), lvars)) + if is_String(args) and not isinstance(args, CmdStringHolder): args = str(args) # In case it's a UserString. try: - def sub_match(match): - return self.conv(self.expand(match.group(1), lvars)) result = _dollar_exps.sub(sub_match, args) except TypeError: # If the internal conversion routine doesn't return @@ -805,7 +807,7 @@ _separate_args = re.compile(r'(%s|\s+|[^\s$]+|\$)' % _dollar_exps_str) _space_sep = re.compile(r'[\t ]+(?![^{]*})') -def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None, overrides: bool=False): +def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None, overrides: Optional[dict] = None): """Expand a string or list containing construction variable substitutions. @@ -887,7 +889,7 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ return result -def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None,overrides: bool=False): +def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None, overrides: Optional[dict] = None): """Substitute construction variables in a string (or list or other object) and separate the arguments into a command list. diff --git a/SCons/Tool/dmd.py b/SCons/Tool/dmd.py index 1a173ad..2ac84d0 100644 --- a/SCons/Tool/dmd.py +++ b/SCons/Tool/dmd.py @@ -138,7 +138,7 @@ def generate(env) -> None: env['DLIBLINKPREFIX'] = '' if env['PLATFORM'] == 'win32' else '-L-l' env['DLIBLINKSUFFIX'] = '.lib' if env['PLATFORM'] == 'win32' else '' - env['_DLIBFLAGS'] = '${_stripixes(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__, LIBLITERAL)}' + env['_DLIBFLAGS'] = '${_stripixes(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__, LIBLITERALPREFIX)}' env['DLIBDIRPREFIX'] = '-L-L' env['DLIBDIRSUFFIX'] = '' diff --git a/SCons/Tool/ldc.py b/SCons/Tool/ldc.py index b557134..6ee022a 100644 --- a/SCons/Tool/ldc.py +++ b/SCons/Tool/ldc.py @@ -115,7 +115,7 @@ def generate(env) -> None: env['DLIBLINKPREFIX'] = '' if env['PLATFORM'] == 'win32' else '-L-l' env['DLIBLINKSUFFIX'] = '.lib' if env['PLATFORM'] == 'win32' else '' # env['_DLIBFLAGS'] = '${_concat(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, __env__, RDirs, TARGET, SOURCE)}' - env['_DLIBFLAGS'] = '${_stripixes(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__, LIBLITERAL)}' + env['_DLIBFLAGS'] = '${_stripixes(DLIBLINKPREFIX, LIBS, DLIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__, LIBLITERALPREFIX)}' env['DLIBDIRPREFIX'] = '-L-L' env['DLIBDIRSUFFIX'] = '' diff --git a/SCons/Tool/link.py b/SCons/Tool/link.py index bfebc5d..cd8b2f8 100644 --- a/SCons/Tool/link.py +++ b/SCons/Tool/link.py @@ -55,7 +55,7 @@ def generate(env) -> None: env['LINKCOM'] = '$LINK -o $TARGET $LINKFLAGS $__RPATH $SOURCES $_LIBDIRFLAGS $_LIBFLAGS' env['LIBDIRPREFIX'] = '-L' env['LIBDIRSUFFIX'] = '' - env['_LIBFLAGS'] = '${_stripixes(LIBLINKPREFIX, LIBS, LIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__, LIBLITERAL)}' + env['_LIBFLAGS'] = '${_stripixes(LIBLINKPREFIX, LIBS, LIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__, LIBLITERALPREFIX)}' env['LIBLINKPREFIX'] = '-l' env['LIBLINKSUFFIX'] = '' diff --git a/test/Libs/LIBLITERAL.py b/test/Libs/LIBLITERAL.py index e99fa1f..ebd32b6 100644 --- a/test/Libs/LIBLITERAL.py +++ b/test/Libs/LIBLITERAL.py @@ -24,7 +24,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -Test LIBLITERAL behavior. +Test LIBLITERALPREFIX behavior. Build a static library and shared library of the same name, and make sure the requested syntax selects the static one. @@ -46,11 +46,11 @@ if sys.platform == 'win32': test.skip_test("Functionality not available for msvc, skipping test.\n") test.write('SConstruct', """\ -LIBLITERAL = ":" -env = Environment(LIBLITERAL=LIBLITERAL, LIBPATH=".") +LIBLITERALPREFIX = ":" +env = Environment(LIBLITERALPREFIX=LIBLITERALPREFIX, LIBPATH=".") lib = env.Library(target='foo', source='foo.c') shlib = env.SharedLibrary(target='foo', source='shfoo.c') -env.Program(target='prog', source=['prog.c'], LIBS=f"{LIBLITERAL}libfoo.a") +env.Program(target='prog', source=['prog.c'], LIBS=f"{LIBLITERALPREFIX}libfoo.a") """) test.write('shfoo.c', r""" -- cgit v0.12