Split the Java parser into a JavaCommon.py module.

5 files changed, 360 insertions, 186 deletions

diff --git a/bin/files b/bin/files
index c22c426..9b24f9a 100644
--- a/bin/files
+++ b/bin/files
@@ -52,6 +52,7 @@
 ./SCons/Tool/ilink.py
 ./SCons/Tool/jar.py
 ./SCons/Tool/javac.py
+./SCons/Tool/JavaCommon.py
 ./SCons/Tool/javah.py
 ./SCons/Tool/latex.py
 ./SCons/Tool/lex.py
diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in
index 03b2c04..8bd3c2f 100644
--- a/src/engine/MANIFEST.in
+++ b/src/engine/MANIFEST.in
@@ -60,6 +60,7 @@ SCons/Tool/hpcc.py
 SCons/Tool/hplink.py
 SCons/Tool/jar.py
 SCons/Tool/javac.py
+SCons/Tool/JavaCommon.py
 SCons/Tool/javah.py
 SCons/Tool/icc.py
 SCons/Tool/ifl.py
diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py
new file mode 100644
index 0000000..3348bc0
--- /dev/null
+++ b/src/engine/SCons/Tool/JavaCommon.py
@@ -0,0 +1,223 @@
+"""SCons.Tool.JavaCommon
+
+Stuff for processing Java.
+
+"""
+
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import os
+import os.path
+import re
+import string
+
+java_parsing = 1
+
+if java_parsing:
+    # Parse Java files for class names.
+    #
+    # This is a really cool parser from Charles Crain
+    # that finds appropriate class names in Java source.
+
+    # A regular expression that will find, in a java file,
+    # any alphanumeric token (keyword, class name, specifier); open or
+    # close brackets; a single-line comment "//"; the multi-line comment
+    # begin and end tokens /* and */; single or double quotes; and
+    # single or double quotes preceeded by a backslash.
+    _reToken = re.compile(r'(//[^\r\n]*|\\[\'"]|[\'"\{\}]|[A-Za-z_][\w\.]*|' +
+                          r'/\*|\*/)')
+
+    class OuterState:
+        """The initial state for parsing a Java file for classes,
+        interfaces, and anonymous inner classes."""
+        def __init__(self):
+            self.listClasses = []
+            self.listOutputs = []
+            self.stackBrackets = []
+            self.brackets = 0
+            self.nextAnon = 1
+            self.package = None
+
+        def __getClassState(self):
+            try:
+                return self.classState
+            except AttributeError:
+                ret = ClassState(self)
+                self.classState = ret
+                return ret
+
+        def __getPackageState(self):
+            try:
+                return self.packageState
+            except AttributeError:
+                ret = PackageState(self)
+                self.packageState = ret
+                return ret
+
+        def __getAnonClassState(self):
+            try:
+                return self.anonState
+            except AttributeError:
+                ret = SkipState(1, AnonClassState(self))
+                self.anonState = ret
+                return ret
+
+        def __getSkipState(self):
+            try:
+                return self.skipState
+            except AttributeError:
+                ret = SkipState(1, self)
+                self.skipState = ret
+                return ret
+
+        def parseToken(self, token):
+            if token[:2] == '//':
+                pass # ignore comment
+            elif token == '/*':
+                return IgnoreState('*/', self)
+            elif token == '{':
+                self.brackets = self.brackets + 1
+            elif token == '}':
+                self.brackets = self.brackets - 1
+                if len(self.stackBrackets) and \
+                   self.brackets == self.stackBrackets[-1]:
+                    self.listOutputs.append(string.join(self.listClasses, '$'))
+                    self.listClasses.pop()
+                    self.stackBrackets.pop()
+            elif token == '"' or token == "'":
+                return IgnoreState(token, self)
+            elif token == "new":
+                # anonymous inner class
+                if len(self.listClasses) > 0:
+                    return self.__getAnonClassState()
+                return self.__getSkipState() # Skip the class name
+            elif token == 'class' or token == 'interface':
+                if len(self.listClasses) == 0:
+                    self.nextAnon = 1
+                self.stackBrackets.append(self.brackets)
+                return self.__getClassState()
+            elif token == 'package':
+                return self.__getPackageState()
+            return self
+
+        def addAnonClass(self):
+            """Add an anonymous inner class"""
+            clazz = self.listClasses[0]
+            self.listOutputs.append('%s$%d' % (clazz, self.nextAnon))
+            self.brackets = self.brackets + 1
+            self.nextAnon = self.nextAnon + 1
+
+        def setPackage(self, package):
+            self.package = package
+
+    class AnonClassState:
+        """A state that looks for anonymous inner classes."""
+        def __init__(self, outer_state):
+            # outer_state is always an instance of OuterState
+            self.outer_state = outer_state
+            self.tokens_to_find = 2
+        def parseToken(self, token):
+            # This is an anonymous class if and only if the next token is a bracket
+            if token == '{':
+                self.outer_state.addAnonClass()
+            return self.outer_state
+
+    class SkipState:
+        """A state that will skip a specified number of tokens before
+        reverting to the previous state."""
+        def __init__(self, tokens_to_skip, old_state):
+            self.tokens_to_skip = tokens_to_skip
+            self.old_state = old_state
+        def parseToken(self, token):
+            self.tokens_to_skip = self.tokens_to_skip - 1
+            if self.tokens_to_skip < 1:
+                return self.old_state
+            return self
+
+    class ClassState:
+        """A state we go into when we hit a class or interface keyword."""
+        def __init__(self, outer_state):
+            # outer_state is always an instance of OuterState
+            self.outer_state = outer_state
+        def parseToken(self, token):
+            # the only token we get should be the name of the class.
+            self.outer_state.listClasses.append(token)
+            return self.outer_state
+
+    class IgnoreState:
+        """A state that will ignore all tokens until it gets to a
+        specified token."""
+        def __init__(self, ignore_until, old_state):
+            self.ignore_until = ignore_until
+            self.old_state = old_state
+        def parseToken(self, token):
+            if self.ignore_until == token:
+                return self.old_state
+            return self
+
+    class PackageState:
+        """The state we enter when we encounter the package keyword.
+        We assume the next token will be the package name."""
+        def __init__(self, outer_state):
+            # outer_state is always an instance of OuterState
+            self.outer_state = outer_state
+        def parseToken(self, token):
+            self.outer_state.setPackage(token)
+            return self.outer_state
+
+    def parse_java_file(fn):
+        return parse_java(open(fn, 'r').read())
+
+    def parse_java(contents):
+        """Parse a .java file and return a double of package directory,
+        plus a list of .class files that compiling that .java file will
+        produce"""
+        package = None
+        initial = OuterState()
+        currstate = initial
+        for token in _reToken.findall(contents):
+            # The regex produces a bunch of groups, but only one will
+            # have anything in it.
+            currstate = currstate.parseToken(token)
+        if initial.package:
+            package = string.replace(initial.package, '.', os.sep)
+        return (package, initial.listOutputs)
+
+else:
+    # Don't actually parse Java files for class names.
+    #
+    # We might make this a configurable option in the future if
+    # Java-file parsing takes too long (although it shouldn't relative
+    # to how long the Java compiler itself seems to take...).
+
+    def parse_java_file(fn):
+        """ "Parse" a .java file.
+
+        This actually just splits the file name, so the assumption here
+        is that the file name matches the public class name, and that
+        the path to the file is the same as the package name.
+        """
+        return os.path.split(file)
diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py
new file mode 100644
index 0000000..96bd31d
--- /dev/null
+++ b/src/engine/SCons/Tool/JavaCommonTests.py
@@ -0,0 +1,133 @@
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import unittest
+
+import SCons.Tool.JavaCommon
+
+class parse_javaTestCase(unittest.TestCase):
+
+    def test_empty(self):
+        """Test a bare-bones class"""
+
+        pkg_dir, classes = SCons.Tool.JavaCommon.parse_java("""\
+package com.sub.bar;
+
+public class Foo
+{
+
+     public static void main(String[] args)
+     {
+
+     }
+
+}
+""")
+        assert pkg_dir == 'com/sub/bar', pkg_dir
+        assert classes == ['Foo'], classes
+
+    def test_inner_classes(self):
+        """Test parsing various forms of inner classes"""
+
+        pkg_dir, classes = SCons.Tool.JavaCommon.parse_java("""\
+class Empty {
+}
+
+interface Listener {
+  public void execute();
+}
+
+public
+class
+Test {
+  class Inner {
+    void go() {
+      use(new Listener() {
+        public void execute() {
+          System.out.println("In Inner");
+        }
+      });
+    }
+    String s1 = "class A";
+    String s2 = "new Listener() { }";
+    /* class B */
+    /* new Listener() { } */
+  }
+
+  public static void main(String[] args) {
+    new Test().run();
+  }
+
+  void run() {
+    use(new Listener() {
+      public void execute() {
+        use(new Listener( ) {
+          public void execute() {
+            System.out.println("Inside execute()");
+          }
+        });
+      }
+    });
+
+    new Inner().go();
+  }
+
+  void use(Listener l) {
+    l.execute();
+  }
+}
+
+class Private {
+  void run() {
+    new Listener() {
+      public void execute() {
+      }
+    };
+  }
+}
+""")
+
+        assert pkg_dir is None, pkg_dir
+        expect = [
+                   'Empty',
+                   'Listener',
+                   'Test$1',
+                   'Test$Inner',
+                   'Test$2',
+                   'Test$3',
+                   'Test',
+                   'Private$1',
+                   'Private',
+                 ]
+        assert classes == expect, classes
+
+if __name__ == "__main__":
+    suite = unittest.TestSuite()
+    tclasses = [ parse_javaTestCase ]
+    for tclass in tclasses:
+        names = unittest.getTestCaseNames(tclass, 'test_')
+        suite.addTests(map(tclass, names))
+    if not unittest.TextTestRunner().run(suite).wasSuccessful():
+        sys.exit(1)
diff --git a/src/engine/SCons/Tool/javac.py b/src/engine/SCons/Tool/javac.py
index be600f1..3d4df10 100644
--- a/src/engine/SCons/Tool/javac.py
+++ b/src/engine/SCons/Tool/javac.py
@@ -40,191 +40,7 @@ import string
 
 import SCons.Builder
 from SCons.Node.FS import _my_normcase
-
-java_parsing = 1
-
-if java_parsing:
-    # Parse Java files for class names.
-    #
-    # This is a really cool parser from Charles Crain
-    # that finds appropriate class names in Java source.
-
-    # A regular expression that will find, in a java file,
-    # any alphanumeric token (keyword, class name, specifier); open or
-    # close brackets; a single-line comment "//"; the multi-line comment
-    # begin and end tokens /* and */; single or double quotes; and
-    # single or double quotes preceeded by a backslash.
-    _reToken = re.compile(r'(//[^\r\n]*|\\[\'"]|[\'"\{\}]|[A-Za-z_][\w\.]*|' +
-                          r'/\*|\*/)')
-
-    class OuterState:
-        """The initial state for parsing a Java file for classes,
-        interfaces, and anonymous inner classes."""
-        def __init__(self):
-            self.listClasses = []
-            self.listOutputs = []
-            self.stackBrackets = []
-            self.brackets = 0
-            self.nextAnon = 1
-            self.package = None
-
-        def __getClassState(self):
-            try:
-                return self.classState
-            except AttributeError:
-                ret = ClassState(self)
-                self.classState = ret
-                return ret
-
-        def __getPackageState(self):
-            try:
-                return self.packageState
-            except AttributeError:
-                ret = PackageState(self)
-                self.packageState = ret
-                return ret
-
-        def __getAnonClassState(self):
-            try:
-                return self.anonState
-            except AttributeError:
-                ret = SkipState(1, AnonClassState(self))
-                self.anonState = ret
-                return ret
-
-        def __getSkipState(self):
-            try:
-                return self.skipState
-            except AttributeError:
-                ret = SkipState(1, self)
-                self.skipState = ret
-                return ret
-
-        def parseToken(self, token):
-            if token[:2] == '//':
-                pass # ignore comment
-            elif token == '/*':
-                return IgnoreState('*/', self)
-            elif token == '{':
-                self.brackets = self.brackets + 1
-            elif token == '}':
-                self.brackets = self.brackets - 1
-                if len(self.stackBrackets) and \
-                   self.brackets == self.stackBrackets[-1]:
-                    self.listOutputs.append(string.join(self.listClasses, '$'))
-                    self.listClasses.pop()
-                    self.stackBrackets.pop()
-            elif token == '"' or token == "'":
-                return IgnoreState(token, self)
-            elif token == "new":
-                # anonymous inner class
-                if len(self.listClasses) > 0:
-                    return self.__getAnonClassState()
-                return self.__getSkipState() # Skip the class name
-            elif token == 'class' or token == 'interface':
-                if len(self.listClasses) == 0:
-                    self.nextAnon = 1
-                self.stackBrackets.append(self.brackets)
-                return self.__getClassState()
-            elif token == 'package':
-                return self.__getPackageState()
-            return self
-
-        def addAnonClass(self):
-            """Add an anonymous inner class"""
-            clazz = self.listClasses[0]
-            self.listOutputs.append('%s$%d' % (clazz, self.nextAnon))
-            self.brackets = self.brackets + 1
-            self.nextAnon = self.nextAnon + 1
-
-        def setPackage(self, package):
-            self.package = package
-
-    class AnonClassState:
-        """A state that looks for anonymous inner classes."""
-        def __init__(self, outer_state):
-            # outer_state is always an instance of OuterState
-            self.outer_state = outer_state
-            self.tokens_to_find = 2
-        def parseToken(self, token):
-            # This is an anonymous class if and only if the next token is a bracket
-            if token == '{':
-                self.outer_state.addAnonClass()
-            return self.outer_state
-
-    class SkipState:
-        """A state that will skip a specified number of tokens before
-        reverting to the previous state."""
-        def __init__(self, tokens_to_skip, old_state):
-            self.tokens_to_skip = tokens_to_skip
-            self.old_state = old_state
-        def parseToken(self, token):
-            self.tokens_to_skip = self.tokens_to_skip - 1
-            if self.tokens_to_skip < 1:
-                return self.old_state
-            return self
-
-    class ClassState:
-        """A state we go into when we hit a class or interface keyword."""
-        def __init__(self, outer_state):
-            # outer_state is always an instance of OuterState
-            self.outer_state = outer_state
-        def parseToken(self, token):
-            # the only token we get should be the name of the class.
-            self.outer_state.listClasses.append(token)
-            return self.outer_state
-
-    class IgnoreState:
-        """A state that will ignore all tokens until it gets to a
-        specified token."""
-        def __init__(self, ignore_until, old_state):
-            self.ignore_until = ignore_until
-            self.old_state = old_state
-        def parseToken(self, token):
-            if self.ignore_until == token:
-                return self.old_state
-            return self
-
-    class PackageState:
-        """The state we enter when we encounter the package keyword.
-        We assume the next token will be the package name."""
-        def __init__(self, outer_state):
-            # outer_state is always an instance of OuterState
-            self.outer_state = outer_state
-        def parseToken(self, token):
-            self.outer_state.setPackage(token)
-            return self.outer_state
-
-    def parse_java(fn):
-        """Parse a .java file and return a double of package directory,
-        plus a list of .class files that compiling that .java file will
-        produce"""
-        package = None
-        initial = OuterState()
-        currstate = initial
-        for token in _reToken.findall(open(fn, 'r').read()):
-            # The regex produces a bunch of groups, but only one will
-            # have anything in it.
-            currstate = currstate.parseToken(token)
-        if initial.package:
-            package = string.replace(initial.package, '.', os.sep)
-        return (package, initial.listOutputs)
-
-else:
-    # Don't actually parse Java files for class names.
-    #
-    # We might make this a configurable option in the future if
-    # Java-file parsing takes too long (although it shouldn't relative
-    # to how long the Java compiler itself seems to take...).
-
-    def parse_java(file):
-        """ "Parse" a .java file.
-
-        This actually just splits the file name, so the assumption here
-        is that the file name matches the public class name, and that
-        the path to the file is the same as the package name.
-        """
-        return os.path.split(file)
+from SCons.Tool.JavaCommon import parse_java_file
 
 def classname(path):
     """Turn a string (path name) into a Java class name."""
@@ -250,7 +66,7 @@ def emit_java_classes(target, source, env):
 
     tlist = []
     for file in slist:
-        pkg_dir, classes = parse_java(file.get_abspath())
+        pkg_dir, classes = parse_java_file(file.get_abspath())
         if pkg_dir:
             for c in classes:
                 t = target[0].Dir(pkg_dir).File(c+class_suffix)