Really parse .java files for inner class names. (Charles Crain)

4 files changed, 206 insertions, 102 deletions

diff --git a/doc/man/scons.1 b/doc/man/scons.1
index 32d4bfb..9f5d7f8 100644
--- a/doc/man/scons.1
+++ b/doc/man/scons.1
@@ -31,7 +31,7 @@
 .RE
 .fi
 ..
-.TH SCONS 1 "February 2003"
+.TH SCONS 1 "April 2003"
 .SH NAME
 scons \- a software construction tool
 .SH SYNOPSIS
@@ -1277,26 +1277,28 @@ Builds one or more Java class files
 from a source tree of .java files.
 The class files will be placed underneath
 the specified target directory.
-SCons assumes that each .java file
-contains a single public class
-with the same name as the basename of the file;
-that is, the file
-.I Foo.java
-contains a single public class named
-.IR Foo .
-SCons will search each Java file
+SCons will parse each source .java file
+to find the classes
+(including inner classes)
+defined within that file,
+and from that figure out the
+target .class files that will be created.
+SCons will also search each Java file
 for the Java package name,
 which it assumes can be found on a line
 beginning with the string
 .B package
-in the first column.
-The resulting .class file
+in the first column;
+the resulting .class files
 will be placed in a directory reflecting
-the specified package name;
-that is,
+the specified package name.
+For example,
 the file
 .I Foo.java
-with a package name of
+defining a single public
+.I Foo
+class and
+containing a package name of
 .I sub.dir
 will generate a corresponding
 .IR sub/dir/Foo.class
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 78a209e..e982d73 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -34,6 +34,11 @@ RELEASE 0.14 - XXX
   - Add an optional sort function argument to the GenerateHelpText()
     Options function.
 
+  From Charles Crain:
+
+  - Parse the source .java files for class names (including inner class
+    names) to figure out the target .class files that will be created.
+
   From Steven Knight:
 
   - Add support for Java (javac and jar).
diff --git a/src/engine/SCons/Tool/javac.py b/src/engine/SCons/Tool/javac.py
index e2d4ecb..60233b3 100644
--- a/src/engine/SCons/Tool/javac.py
+++ b/src/engine/SCons/Tool/javac.py
@@ -34,107 +34,123 @@ selection method.
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os.path
+import re
 import string
 
 import SCons.Builder
 
-# Okay, I don't really know what configurability would be good for
-# parsing Java files for package and/or class names, but it was easy, so
-# here it is.
-#
-# Set java_parsing to the following values to enable three different
-# flavors of parsing:
-#
-#    0  The file isn't actually parsed, so this will be quickest.  The
-#       package + class name is assumed to be the file path name, and we
-#       just split the path name.  This breaks if a package name will
-#       ever be different from the path to the .java file.
-#
-#    1  The file is read to find the package name, after which we stop.
-#       This should be pretty darn quick, and allows flexibility in
-#       package names, but assumes that the public class name in the
-#       file matches the file name.  This seems to be a good assumption
-#       because, for example, if you try to declare a public class
-#       with a different name from the file, javac tells you:
-#
-#           class Foo is public, should be declared in a file named Foo.java
-#
-#    2  Full flexibility of class names.  We parse for the package name
-#       (like level #1) but the output .class file name is assumed to
-#       match the declared public class name--and, as a bonus, this will
-#       actually support multiple public classes in a single file.  My
-#       guess is that's illegal Java, though...  Or is it an option
-#       supported by some compilers?
-#
 java_parsing = 1
 
-if java_parsing == 0:
-    def parse_java(file, suffix):
-        """ "Parse" a .java file.
-
-        This actually just splits the file name, so the assumption here
-        is that the file name matches the public class name, and that
-        the path to the file is the same as the package name.
-        """
-        return os.path.split(file)
-elif java_parsing == 1:
-    def parse_java(file, suffix):
-        """Parse a .java file for a package name.
-
-        This, of course, is not full parsing of Java files, but
-        simple-minded searching for the usual begins-in-column-1
-        "package" string most Java programs use to define their package.
-        """
+if java_parsing:
+    # Parse Java files for class names.
+    #
+    # This is a really simple and cool parser from Charles Crain
+    # that finds appropriate class names in Java source.
+
+    _reToken = re.compile(r'[^\\]([\'"])|([\{\}])|' +
+                          r'(?:^|[\{\}\s;])((?:class|interface)'+
+                          r'\s+[A-Za-z_]\w*)|' +
+                          r'(new\s+[A-Za-z_]\w*\s*\([^\)]*\)\s*\{)|' +
+                          r'(//[^\r\n]*)|(/\*|\*/)')
+
+    class OuterState:
+        def __init__(self):
+            self.listClasses = []
+            self.listOutputs = []
+            self.stackBrackets = []
+            self.brackets = 0
+            self.nextAnon = 1
+
+        def parseToken(self, token):
+            #print token
+            if token[:2] == '//':
+                pass # ignore comment
+            elif token == '/*':
+                return IgnoreState('*/', self)
+            elif token == '{':
+                self.brackets = self.brackets + 1
+            elif token == '}':
+                self.brackets = self.brackets - 1
+                if len(self.stackBrackets) and \
+                   self.brackets == self.stackBrackets[-1]:
+                    self.listOutputs.append(string.join(self.listClasses, '$'))
+                    self.listClasses.pop()
+                    self.stackBrackets.pop()
+            elif token == '"':
+                return IgnoreState('"', self)
+            elif token == "'":
+                return IgnoreState("'", self)
+            elif token[:3] == "new":
+                # anonymous inner class
+                if len(self.listClasses) > 0:
+                    clazz = self.listClasses[0]
+                    self.listOutputs.append('%s$%d' % (clazz, self.nextAnon))
+                    self.brackets = self.brackets + 1
+                    self.nextAnon = self.nextAnon + 1
+            elif token[:5] == 'class':
+                if len(self.listClasses) == 0:
+                    self.nextAnon = 1
+                self.listClasses.append(string.join(string.split(token[6:])))
+                self.stackBrackets.append(self.brackets)
+            elif token[:9] == 'interface':
+                if len(self.listClasses) == 0:
+                    self.nextAnon = 1
+                self.listClasses.append(string.join(string.split(token[10:])))
+                self.stackBrackets.append(self.brackets)
+            return self
+
+    class IgnoreState:
+        def __init__(self, ignore_until, old_state):
+            self.ignore_until = ignore_until
+            self.old_state = old_state
+        def parseToken(self, token):
+            if token == self.ignore_until:
+                return self.old_state
+            return self
+
+    def parse_java(file):
+        contents = open(file, 'r').read()
+
+        # Is there a more efficient way to do this than to split
+        # the contents like this?
         pkg_dir = None
-        classes = []
-        f = open(file, "rb")
-        while 1:
-            line = f.readline()
-            if not line:
-                break
+        for line in string.split(contents, "\n"):
             if line[:7] == 'package':
                 pkg = string.split(line)[1]
                 if pkg[-1] == ';':
                     pkg = pkg[:-1]
                 pkg_dir = apply(os.path.join, string.split(pkg, '.'))
-                classes = [ os.path.split(file[:-len(suffix)])[1] ]
                 break
-        f.close()
-        return pkg_dir, classes
-
-elif java_parsing == 2:
-    import re
-    pub_re = re.compile('^\s*public(\s+abstract)?\s+class\s+(\S+)')
-    def parse_java(file, suffix):
-        """Parse a .java file for package name and classes.
-    
-        This, of course, is not full parsing of Java files, but
-        simple-minded searching for the usual strings most Java programs
-        seem to use for packages and public class names.
+
+        initial = OuterState()
+        currstate = initial
+        for matches in _reToken.findall(contents):
+            # The regex produces a bunch of groups, but only one will
+            # have anything in it.
+            token = filter(lambda x: x, matches)[0]
+            currstate = currstate.parseToken(token)
+
+        return pkg_dir, initial.listOutputs
+
+else:
+    # Don't actually parse Java files for class names.
+    #
+    # We might make this a configurable option in the future if
+    # Java-file parsing takes too long (although it shouldn't relative
+    # to how long the Java compiler itself seems to take...).
+
+    def parse_java(file):
+        """ "Parse" a .java file.
+
+        This actually just splits the file name, so the assumption here
+        is that the file name matches the public class name, and that
+        the path to the file is the same as the package name.
         """
-        pkg_dir = None
-        classes = []
-        f = open(file, "rb")
-        while 1:
-            line = f.readline()
-            if not line:
-                break
-            if line[:7] == 'package':
-                pkg = string.split(line)[1]
-                if pkg[-1] == ';':
-                    pkg = pkg[:-1]
-                pkg_dir = apply(os.path.join, string.split(pkg, '.'))
-            elif line[:6] == 'public':
-                c = pub_re.findall(line)
-                try:
-                    classes.append(c[0][1])
-                except IndexError:
-                    pass
-        f.close()
-        return pkg_dir, classes
+        return os.path.split(file)
 
 def generate(env):
     """Add Builders and construction variables for javac to an Environment."""
+
     def emit_java_files(target, source, env):
         """Create and return lists of source java files
         and their corresponding target class files.
@@ -143,6 +159,7 @@ def generate(env):
         env['_JAVASRCDIR'] = source[0]
         java_suffix = env.get('JAVASUFFIX', '.java')
         class_suffix = env.get('JAVACLASSSUFFIX', '.class')
+
         slist = []
         def visit(arg, dirname, names, js=java_suffix):
             java_files = filter(lambda n, js=js: n[-len(js):] == js, names)
@@ -151,17 +168,24 @@ def generate(env):
                              java_files)
             arg.extend(java_paths)
         os.path.walk(source[0], visit, slist)
+
         tlist = []
         for file in slist:
-            pkg_dir, classes = parse_java(file, java_suffix)
+            pkg_dir, classes = parse_java(file)
             if pkg_dir:
                 for c in classes:
                     tlist.append(os.path.join(target[0],
                                               pkg_dir,
                                               c + class_suffix))
+            elif classes:
+                for c in classes:
+                    tlist.append(os.path.join(target[0], c + class_suffix))
             else:
+                # This is an odd end case:  no package and no classes.
+                # Just do our best based on the source file name.
                 tlist.append(os.path.join(target[0],
-                                          file[:-5] + class_suffix))
+                                          file[:-len(java_suffix)] + class_suffix))
+
         return tlist, slist
 
     JavaBuilder = SCons.Builder.Builder(action = '$JAVACCOM',
diff --git a/test/JAVAC.py b/test/JAVAC.py
index f2910a4..1651be0 100644
--- a/test/JAVAC.py
+++ b/test/JAVAC.py
@@ -111,11 +111,14 @@ javac = foo.Dictionary('JAVAC')
 bar = foo.Copy(JAVAC = r'%s wrapper.py ' + javac)
 foo.Java(target = 'classes', source = 'com/sub/foo')
 bar.Java(target = 'classes', source = 'com/sub/bar')
+foo.Java(target = 'classes', source = 'src')
 """ % python)
 
-test.subdir('com', ['com', 'sub'],
+test.subdir('com',
+            ['com', 'sub'],
             ['com', 'sub', 'foo'],
-            ['com', 'sub', 'bar'])
+            ['com', 'sub', 'bar'],
+            'src')
 
 test.write(['com', 'sub', 'foo', 'Example1.java'], """\
 package com.sub.foo;
@@ -201,6 +204,65 @@ public class Example6
 }
 """)
 
+# Acid-test file for parsing inner Java classes, courtesy Chad Austin.
+test.write(['src', 'Test.java'], """\
+class Empty {
+}
+
+interface Listener {
+  public void execute();
+}
+
+public
+class
+Test {
+  class Inner {
+    void go() {
+      use(new Listener() {
+        public void execute() {
+          System.out.println("In Inner");
+        }
+      });
+    }
+    String s1 = "class A";
+    String s2 = "new Listener() { }";
+    /* class B */
+    /* new Listener() { } */
+  }
+
+  public static void main(String[] args) {
+    new Test().run();
+  }
+
+  void run() {
+    use(new Listener() {
+      public void execute() {
+        use(new Listener( ) {
+          public void execute() {
+            System.out.println("Inside execute()");
+          }
+        });
+      }
+    });
+
+    new Inner().go();
+  }
+
+  void use(Listener l) {
+    l.execute();
+  }
+}
+
+class Private {
+  void run() {
+    new Listener() {
+      public void execute() {
+      }
+    };
+  }
+}
+""")
+
 test.run(arguments = '.')
 
 test.fail_test(test.read('wrapper.out') != "wrapper.py /usr/local/j2sdk1.3.1/bin/javac -d classes -sourcepath com/sub/bar com/sub/bar/Example4.java com/sub/bar/Example5.java com/sub/bar/Example6.java\n")
@@ -208,10 +270,21 @@ test.fail_test(test.read('wrapper.out') != "wrapper.py /usr/local/j2sdk1.3.1/bin
 test.fail_test(not os.path.exists(test.workpath('classes', 'com', 'sub', 'foo', 'Example1.class')))
 test.fail_test(not os.path.exists(test.workpath('classes', 'com', 'other', 'Example2.class')))
 test.fail_test(not os.path.exists(test.workpath('classes', 'com', 'sub', 'foo', 'Example3.class')))
+
 test.fail_test(not os.path.exists(test.workpath('classes', 'com', 'sub', 'bar', 'Example4.class')))
 test.fail_test(not os.path.exists(test.workpath('classes', 'com', 'other', 'Example5.class')))
 test.fail_test(not os.path.exists(test.workpath('classes', 'com', 'sub', 'bar', 'Example6.class')))
 
+test.fail_test(not os.path.exists(test.workpath('classes', 'Empty.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Listener.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Private.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Private$1.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Test.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Test$1.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Test$2.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Test$3.class')))
+test.fail_test(not os.path.exists(test.workpath('classes', 'Test$Inner.class')))
+
 test.up_to_date(arguments = '.')
 
 test.pass_test()