From cbe5d046c82d6f3c3e9952ffdbcf457be1dccb72 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 16 Oct 2022 12:43:12 -0600 Subject: Java scanner don't split JAVACLASSPATH on space The scanner now splits JAVACLASSPATH before searching only if it is passed as a scalar (string); it no longer recurses to split elements of a list value - this is more consistent with other SCons usage. No longer splits on space, rather splits on os.pathsep, as considerably more useful for a search-path variable. The latter change avoids breaking paths with an embedded space (usually from Windows). This is a "breaking change" from the new behavior introduced in 4.4 where a JAVACLASSPATH like "aaa bbb ccc" would have been split into ["aaa", "bbb", "ccc"] - now it will be left unchanged in the scanner. However a JAVACLASSPATH like "aaa;bbb;ccc" will now be split into ["aaa", "bbb", "ccc"]. This behavior only affects the scanner - there is no change in how JAVACLASSPATH gets transformed into the "-classpath ARG" argument when calling JDK elements. The former behavior was presumably unintended, and definitely broke on a space-containing path (e.g. "My Classes". The unit test is expanded to test for a path string with spaces, and for a path string containing a search-path separator. Documentation tweaks: explain better how JAVACLASSPATH (and the other two Java path variables) can be specified, and whether or not SCons changes them before calling the JDK commands. Added note that JAVABOOTCLASSPATH is no longer useful, and a note about the side effect that SCons always supplies a "-sourcepath" argument when calling javac. Fixes #4243 Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 +++ RELEASE.txt | 10 ++++ SCons/Scanner/Java.py | 57 +++++++++--------- SCons/Scanner/JavaTests.py | 43 +++++++++++--- SCons/Tool/jar.xml | 4 +- SCons/Tool/javac.xml | 143 +++++++++++++++++++++++++++++++-------------- SCons/Tool/javah.xml | 4 +- SCons/Tool/rmic.xml | 28 +++++---- doc/user/java.xml | 49 +++++----------- 9 files changed, 220 insertions(+), 125 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9cd653..c5d55a9 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -40,6 +40,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - doc: EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and SConscriptChdir are now listed as Global functions only; the Environment versions still work but are not documented. + - The Java scanner processing of JAVACLASSPATH for dependencies was + changed to split on os.pathsep instead of space, to match usage of + passing a path string like "xxx:yyy:zzz". This is not portable - + passing a POSIX-style path string (with ':') won't work on Windows + (';') - which is now documented with a hint to use a list instead + to be portable. Splitting on space broke paths with embedded spaces. + Fixes #4243. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 8ad105b..a8541e6 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -43,6 +43,10 @@ FIXES - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress message were adjusted. +- The Java scanner processing of JAVACLASSPATH for dependencies (introduced + in 4.4.0) is adjusted to split on the system's search path separator + instead of on space - the latter disallowed passing a completed search + string as a scalar value, and broke on paths with embedded spaces. IMPROVEMENTS ------------ @@ -74,6 +78,12 @@ DOCUMENTATION SConscriptChdir are now listed as Global functions only. - Updated the docs for Glob. - Updated SHELL_ENV_GENERATORS description and added versionadded indicator. +- JAVABOOTCLASSPATH, JAVACLASSPATH and JAVASOURCEPATH better document the + acceptable syntax for values, and how they will be interpreted, + including that JAVACLASSPATH will be scanned for dependencies. + Added note on the possibly surprising feature that SCons always passes + -sourcepath when calling javac, which affects how the class path is + used when finding sources. DEVELOPMENT ----------- diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index ab1f4e6..0c6df37 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -21,56 +21,58 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import os +import os import SCons.Node import SCons.Node.FS import SCons.Scanner -import SCons.Util +from SCons.Util import flatten, is_String -def _subst_libs(env, libs): - """ - Substitute environment variables and split into list. +def _subst_paths(env, paths) -> list: + """Return a list of substituted path elements. + + If *paths* is a string, it is split on the search-path separator + (this makes the interpretation system-specitic - this is warned about + in the manpage). This helps support behavior like pulling in the + external ``CLASSPATH`` and setting it directly into ``JAVACLASSPATH``. + Otherwise, substitution is done on string-valued list elements + but not splitting. """ - if SCons.Util.is_String(libs): - libs = env.subst(libs) - if SCons.Util.is_String(libs): - libs = libs.split() - elif SCons.Util.is_Sequence(libs): - _libs = [] - for lib in libs: - _libs += _subst_libs(env, lib) - libs = _libs + if is_String(paths): + paths = env.subst(paths) + if SCons.Util.is_String(paths): + paths = paths.split(os.pathsep) else: - # libs is an object (Node, for example) - libs = [libs] - return libs + paths = flatten(paths) + paths = [env.subst(path) if is_String(path) else path for path in paths] + return paths -def _collect_classes(list, dirname, files): +def _collect_classes(classlist, dirname, files): for fname in files: - if os.path.splitext(fname)[1] == ".class": - list.append(os.path.join(str(dirname), fname)) + if fname.endswith(".class"): + classlist.append(os.path.join(str(dirname), fname)) -def scan(node, env, libpath=()): +def scan(node, env, libpath=()) -> list: """Scan for files on the JAVACLASSPATH. - The classpath can contain: + JAVACLASSPATH path can contain: - Explicit paths to JAR/Zip files - Wildcards (*) - Directories which contain classes in an unnamed package - Parent directories of the root package for classes in a named package - Class path entries that are neither directories nor archives (.zip or JAR files) nor the asterisk (*) wildcard character are ignored. - """ + Class path entries that are neither directories nor archives (.zip + or JAR files) nor the asterisk (*) wildcard character are ignored. + """ classpath = env.get('JAVACLASSPATH', []) - classpath = _subst_libs(env, classpath) + classpath = _subst_paths(env, classpath) result = [] for path in classpath: - if SCons.Util.is_String(path) and "*" in path: + if is_String(path) and "*" in path: libs = env.Glob(path) else: libs = [path] @@ -89,8 +91,7 @@ def scan(node, env, libpath=()): def JavaScanner(): - return SCons.Scanner.Base(scan, 'JavaScanner', - skeys=['.java']) + return SCons.Scanner.Base(scan, 'JavaScanner', skeys=['.java']) # Local Variables: # tab-width:4 diff --git a/SCons/Scanner/JavaTests.py b/SCons/Scanner/JavaTests.py index 9fb39ce..77cd560 100644 --- a/SCons/Scanner/JavaTests.py +++ b/SCons/Scanner/JavaTests.py @@ -33,18 +33,25 @@ import SCons.Warnings test = TestCmd.TestCmd(workdir = '') -test.subdir('com') files = [ 'bootclasspath.jar', 'classpath.jar', 'Test.class', - 'com/Test.class' ] for fname in files: test.write(fname, "\n") +test.subdir('com') +test.subdir('java space') +subfiles = [ + 'com/Test.class', + 'java space/Test.class' +] + +for fname in subfiles: + test.write(fname.split('/'), "\n") class DummyEnvironment(collections.UserDict): def __init__(self,**kw): @@ -52,7 +59,7 @@ class DummyEnvironment(collections.UserDict): self.data.update(kw) self.fs = SCons.Node.FS.FS(test.workpath('')) self['ENV'] = {} - + def Dictionary(self, *args): return self.data @@ -80,7 +87,7 @@ class DummyEnvironment(collections.UserDict): def File(self, filename): return self.fs.File(filename) - + def Glob(self, path): return self.fs.Glob(path) @@ -111,8 +118,7 @@ def deps_match(self, deps, headers): class JavaScannerEmptyClasspath(unittest.TestCase): def runTest(self): path = [] - env = DummyEnvironment(JAVASUFFIXES=['.java'], - JAVACLASSPATH=path) + env = DummyEnvironment(JAVASUFFIXES=['.java'], JAVACLASSPATH=path) s = SCons.Scanner.Java.JavaScanner() deps = s(DummyNode('dummy'), env) expected = [] @@ -145,10 +151,33 @@ class JavaScannerDirClasspath(unittest.TestCase): JAVACLASSPATH=[test.workpath()]) s = SCons.Scanner.Java.JavaScanner() deps = s(DummyNode('dummy'), env) - expected = ['Test.class', 'com/Test.class'] + expected = ['Test.class', 'com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + + +class JavaScannerNamedDirClasspath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment( + JAVASUFFIXES=['.java'], + JAVACLASSPATH=[test.workpath('com'), test.workpath('java space')], + ) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['com/Test.class', 'java space/Test.class'] deps_match(self, deps, expected) +class JavaScannerSearchPathClasspath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment( + JAVASUFFIXES=['.java'], + JAVACLASSPATH=os.pathsep.join([test.workpath('com'), test.workpath('java space')]), + ) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + if __name__ == "__main__": unittest.main() diff --git a/SCons/Tool/jar.xml b/SCons/Tool/jar.xml index 1713495..403aacb 100644 --- a/SCons/Tool/jar.xml +++ b/SCons/Tool/jar.xml @@ -1,6 +1,8 @@ @@ -206,7 +187,7 @@ public class AdditionalClass3 Will not only tell you reliably that the .class files - in the classes subdirectory + in the classes subdirectory are up-to-date: @@ -250,7 +231,7 @@ public class AdditionalClass3 variable to specify the version in use. With Java 1.6, the one-liner example can then be defined like this: - + Java('classes', 'src', JAVAVERSION='1.6') @@ -280,8 +261,8 @@ Java('classes', 'src', JAVAVERSION='1.6') -Java(target = 'classes', source = 'src') -Jar(target = 'test.jar', source = 'classes') +Java(target='classes', source='src') +Jar(target='test.jar', source='classes') public class Example1 @@ -344,10 +325,10 @@ public class Example3 -prog1_class_files = Java(target = 'classes', source = 'prog1') -prog2_class_files = Java(target = 'classes', source = 'prog2') -Jar(target = 'prog1.jar', source = prog1_class_files) -Jar(target = 'prog2.jar', source = prog2_class_files) +prog1_class_files = Java(target='classes', source='prog1') +prog2_class_files = Java(target='classes', source='prog2') +Jar(target='prog1.jar', source=prog1_class_files) +Jar(target='prog2.jar', source=prog2_class_files) public class Example1 @@ -418,8 +399,8 @@ public class Example4 -classes = Java(target = 'classes', source = 'src/pkg/sub') -JavaH(target = 'native', source = classes) +classes = Java(target='classes', source='src/pkg/sub') +JavaH(target='native', source=classes) package pkg.sub; @@ -642,8 +623,8 @@ public class Example3 -classes = Java(target = 'classes', source = 'src/pkg/sub') -RMIC(target = 'outdir', source = classes) +classes = Java(target='classes', source='src/pkg/sub') +RMIC(target='outdir', source=classes) package pkg.sub; -- cgit v0.12 From 984e3f58b9f2c33cc551e2a3d2962c5f6420b3c8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 18 Oct 2022 11:33:36 -0600 Subject: Java scanner: fixes per review comments Signed-off-by: Mats Wichmann --- RELEASE.txt | 13 +++++++++---- SCons/Scanner/Java.py | 10 +++++++++- SCons/Tool/javac.xml | 27 ++++++++++++++------------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index a8541e6..8bdbe43 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -43,10 +43,15 @@ FIXES - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress message were adjusted. -- The Java scanner processing of JAVACLASSPATH for dependencies (introduced - in 4.4.0) is adjusted to split on the system's search path separator - instead of on space - the latter disallowed passing a completed search - string as a scalar value, and broke on paths with embedded spaces. +- The Java Scanner processing of JAVACLASSPATH for dependencies (behavior + that was introduced in SCons 4.4.0) is adjusted to split on the system's + search path separator instead of on a space. The previous behavior meant + that a path containing spaces (e.g. r"C:\somepath\My Classes") would + lead to unexpected errors. If the split-on-space behavior is desired, + pre-split the value: instead of: env["JAVACLASSPATH"] = "foo bar baz" + use: env["JAVACLASSPATH"] = env.Split("foo bar baz") + There is no change in how JAVACLASSPATH gets turned into the -classpath + argument passed to the JDK tools. IMPROVEMENTS ------------ diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index 0c6df37..f6edf93 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -33,7 +33,7 @@ def _subst_paths(env, paths) -> list: """Return a list of substituted path elements. If *paths* is a string, it is split on the search-path separator - (this makes the interpretation system-specitic - this is warned about + (this makes the interpretation system-specific - this is warned about in the manpage). This helps support behavior like pulling in the external ``CLASSPATH`` and setting it directly into ``JAVACLASSPATH``. Otherwise, substitution is done on string-valued list elements @@ -44,6 +44,7 @@ def _subst_paths(env, paths) -> list: if SCons.Util.is_String(paths): paths = paths.split(os.pathsep) else: + # TODO: may want to revisit splitting list-element strings if requested paths = flatten(paths) paths = [env.subst(path) if is_String(path) else path for path in paths] return paths @@ -73,6 +74,9 @@ def scan(node, env, libpath=()) -> list: result = [] for path in classpath: if is_String(path) and "*" in path: + # This matches more than the Java docs describe: a '*' only + # matches jar files. The filter later should trim this down. + # TODO: should we filter here? use .endswith('*') rather than "in"? libs = env.Glob(path) else: libs = [path] @@ -91,6 +95,10 @@ def scan(node, env, libpath=()) -> list: def JavaScanner(): + """Scanner for .java files. + + .. versionadded:: 4.4 + """ return SCons.Scanner.Base(scan, 'JavaScanner', skeys=['.java']) # Local Variables: diff --git a/SCons/Tool/javac.xml b/SCons/Tool/javac.xml index 922ae2b..014d905 100644 --- a/SCons/Tool/javac.xml +++ b/SCons/Tool/javac.xml @@ -140,10 +140,10 @@ env['ENV']['LANG'] = 'en_GB.UTF-8' search path separator characters (: for POSIX systems or ; for Windows), it will not be modified; + and so is inherently system-specific; to supply the path in a system-independent manner, give &cv-JAVABOOTCLASSPATH; as a list of paths instead. - Can only be used when compiling for releases prior to JDK 9. @@ -239,12 +239,13 @@ env = Environment(JAVACCOMSTR="Compiling class files $TARGETS from $SOURCES") If &cv-JAVACLASSPATH; is a single string containing search path separator characters (: for POSIX systems or - ; for Windows), it will not be modified; + ; for Windows), + it will be split on the separator into a list of individual + paths for dependency scanning purposes. + It will not be modified for JDK command-line usage, + so such a string is inherently system-specific; to supply the path in a system-independent manner, give &cv-JAVACLASSPATH; as a list of paths instead. - Such a string will, however, - be split on the separator into a list of individual paths - for dependency scanning purposes. @@ -289,24 +290,24 @@ env = Environment(JAVACCOMSTR="Compiling class files $TARGETS from $SOURCES") The value will be added to the JDK command lines - via the option. - The JDK option requires a - system-specific search path separator, - which will be supplied by &SCons; as needed when it + via the option, + which requires a system-specific search path separator, + This will be supplied by &SCons; as needed when it constructs the command line if &cv-JAVASOURCEPATH; is provided in list form. If &cv-JAVASOURCEPATH; is a single string containing search path separator characters (: for POSIX systems or - ; for Windows), it will not be modified; + ; for Windows), it will not be modified, + and so is inherently system-specific; to supply the path in a system-independent manner, give &cv-JAVASOURCEPATH; as a list of paths instead. - Note that this currently just adds the specified - directories via the option. + Note that the specified directories are only added to + the command line via the option. &SCons; does not currently search the - &cv-JAVASOURCEPATH; directories for dependency + &cv-JAVASOURCEPATH; directories for dependent .java files. -- cgit v0.12 From c0f80cbbeec176f303d3ef1b7ab94582c018fcd8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 20 Oct 2022 09:10:31 -0600 Subject: Add Java scanner to generated API docs Signed-off-by: Mats Wichmann --- SCons/Scanner/Java.py | 14 ++++++++------ doc/sphinx/SCons.Scanner.rst | 8 ++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index f6edf93..8c31bc1 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -32,12 +32,14 @@ from SCons.Util import flatten, is_String def _subst_paths(env, paths) -> list: """Return a list of substituted path elements. - If *paths* is a string, it is split on the search-path separator - (this makes the interpretation system-specific - this is warned about - in the manpage). This helps support behavior like pulling in the - external ``CLASSPATH`` and setting it directly into ``JAVACLASSPATH``. - Otherwise, substitution is done on string-valued list elements - but not splitting. + If *paths* is a string, it is split on the search-path separator. + Otherwise, substitution is done on string-valued list elements but + they are not split. + + Note helps support behavior like pulling in the external ``CLASSPATH`` + and setting it directly into ``JAVACLASSPATH``, however splitting on + ``os.pathsep`` makes the interpretation system-specific (this is + warned about in the manpage entry for ``JAVACLASSPATH``). """ if is_String(paths): paths = env.subst(paths) diff --git a/doc/sphinx/SCons.Scanner.rst b/doc/sphinx/SCons.Scanner.rst index 181dbde..484bd43 100644 --- a/doc/sphinx/SCons.Scanner.rst +++ b/doc/sphinx/SCons.Scanner.rst @@ -44,6 +44,14 @@ SCons.Scanner.IDL module :undoc-members: :show-inheritance: +SCons.Scanner.Java module +------------------------- + +.. automodule:: SCons.Scanner.Java + :members: + :undoc-members: + :show-inheritance: + SCons.Scanner.LaTeX module -------------------------- -- cgit v0.12