Support Scanner functions returning strings, not just Nodes.

5 files changed, 262 insertions, 19 deletions

diff --git a/doc/man/scons.1 b/doc/man/scons.1
index 779d199..7595724 100644
--- a/doc/man/scons.1
+++ b/doc/man/scons.1
@@ -1097,8 +1097,9 @@ function. Builder accepts the following
 arguments:
 
 .IP name
-The name of the builder. This will be the of the construction environment
-method used to create an instance of the builder.
+The name of the builder. This will be the
+name of the construction environment method
+used to create an instance of the builder.
 
 .IP action
 The command line string used to build the target from the source. 
@@ -1198,10 +1199,13 @@ for each element in the list.
 .PP
 If the action argument is not one of the above,
 None is returned.
+
 .SS Variable Substitution
+
+Before executing a command,
 .B scons
 performs construction variable interpolation on the strings that make up
-the command line of builders before executing the command.
+the command line of builders.
 Variables are introduced by a
 .B $
 prefix.
@@ -1320,7 +1324,64 @@ but the command signature added to any target files would be:
 echo Last build occurred  . > $TARGET
 .EE
 
-.\" XXX document how to add user defined scanners. 
+.SS Scanner Objects
+
+You can use the
+.B Scanner
+function to define
+objects to scan
+new file types for implicit dependencies.
+Scanner accepts the following arguments:
+
+.IP name
+The name of the Scanner.
+This is mainly used
+to identify the Scanner internally.
+
+.IP argument
+An optional argument that, if specified,
+will be passed to the scanner function.
+
+.IP skeys
+An optional list that can be used to
+determine which scanner should be used for
+a given Node.
+In the usual case of scanning for file names,
+this array can be a list of suffixes
+for the different file types that this
+Scanner knows how to scan.
+
+.IP function
+A Python function that will process
+the Node (file)
+and return a list of strings (file names)
+representing the implicit
+dependencies found in the contents.
+The function takes three arguments:
+
+    def scanner_function(node, env, arg):
+
+The
+.B node
+argument is the internal
+SCons node representing the file.
+Use
+.B str(node)
+to fetch the name of the file, and
+.B node.get_contents()
+to fetch contents of the file.
+
+The
+.B env
+argument is the construction environment for the scan.
+Fetch values from it using the
+.B env.Dictionary()
+method.
+
+The
+.B arg
+argument is the argument supplied
+when the scanner was created.
 
 .SH EXAMPLES
 
@@ -1389,8 +1450,8 @@ You
 specify a "name" keyword argument for the builder,
 as that becomes the Environment method name
 you use to call the builder.
-Notice also that you can leave off the suffixes,
-and the builder will add them automatically.
+Notice also that you can leave off the target file suffix,
+and the builder will add it automatically.
 
 .ES
 bld = Builder(name = 'PDFBuilder',
@@ -1400,10 +1461,36 @@ bld = Builder(name = 'PDFBuilder',
 env = Environment(BUILDERS = [bld])
 env.PDFBuilder(target = 'foo.pdf', source = 'foo.tex')
 
-# The following creates "bar.pdf" from "bar.text"
+# The following creates "bar.pdf" from "bar.tex"
 env.PDFBuilder(target = 'bar', source = 'bar')
 .EE
 
+.SS Defining Your Own Scanner Object
+
+.ES
+import re
+
+include_re = re.compile(r'^include\\s+(\\S+)$', re.M)
+
+def kfile_scan(node, env, arg):
+    contents = node.get_contents()
+    includes = include_re.findall(contents)
+    return includes
+
+kscan = Scanner(name = 'kfile',
+                function = kfile_scan,
+                argument = None,
+                skeys = ['.k'])
+scanners = Environment().Dictionary('SCANNERS')
+env = Environment(SCANNERS = scanners + [kscan])
+
+env.Command('foo', 'foo.k', 'kprocess < $SOURCES > $TARGET')
+
+bar_in = File('bar.in')
+env.Command('bar', bar_in, 'kprocess $SOURCES > $TARGET')
+bar_in.scanner_set(kscan)
+.EE
+
 .SS Creating a Hierarchical Build
 
 Notice that the file names specified in a subdirectory
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 7365eb0..8c62366 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -15,11 +15,17 @@ RELEASE 0.05 -
   - Allow a library to specified as a command-line source file, not just
     in the LIBS construction variable.
 
+  - Compensate for a bug in os.path.normpath() that returns '' for './'
+    on WIN32.
+
   From Steven Knight:
 
   - Flush stdout after print so it intermixes correctly with stderr
     when redirected.
 
+  - Allow Scanners to return a list of strings, and document how to
+    write your own Scanners.
+
 
 
 RELEASE 0.04 - Wed, 30 Jan 2002 11:09:42 -0600
diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py
index 99cab94..76df18e 100644
--- a/src/engine/SCons/Scanner/ScannerTests.py
+++ b/src/engine/SCons/Scanner/ScannerTests.py
@@ -41,11 +41,14 @@ class ScannerTestBase:
 
     def test(self, scanner, env, filename, deps, *args):
         self.deps = deps
-        deps = scanner.scan(filename, env)
+        scanned = scanner.scan(filename, env)
+        scanned_strs = map(lambda x: str(x), scanned)
 
         self.failUnless(self.filename == filename, "the filename was passed incorrectly")
         self.failUnless(self.env == env, "the environment was passed incorrectly")
-        self.failUnless(self.deps == deps, "the dependencies were returned incorrectly")
+        self.failUnless(scanned_strs == deps, "the dependencies were returned incorrectly")
+        for d in scanned:
+            self.failUnless(type(d) != type(""), "got a string in the dependencies")
 
         if len(args) > 0:
             self.failUnless(self.arg == args[0], "the argument was passed incorrectly")
diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py
index 25345a5..a00a75b 100644
--- a/src/engine/SCons/Scanner/__init__.py
+++ b/src/engine/SCons/Scanner/__init__.py
@@ -32,7 +32,8 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 __version__ = "__VERSION__"
 
 
-from SCons.Util import scons_str2nodes
+import SCons.Node.FS
+import SCons.Util
 
 
 class _Null:
@@ -48,7 +49,12 @@ class Base:
     straightforward, single-pass scanning of a single file.
     """
 
-    def __init__(self, function, name = "NONE", argument=_null, skeys=[]):
+    def __init__(self,
+                 function,
+                 name = "NONE",
+                 argument = _null,
+                 skeys = [],
+                 node_factory = SCons.Node.FS.default_fs.File):
         """
         Construct a new scanner object given a scanner function.
 
@@ -89,19 +95,26 @@ class Base:
         self.name = name
         self.argument = argument
         self.skeys = skeys
+        self.node_factory = node_factory
 
     def scan(self, node, env):
         """
         This method scans a single object. 'node' is the node
         that will be passed to the scanner function, and 'env' is the
         environment that will be passed to the scanner function. A list of
-        direct dependency nodes for the specified filename will be returned.
+        direct dependency nodes for the specified node will be returned.
         """
 
         if not self.argument is _null:
-            return self.function(node, env, self.argument)
+            list = self.function(node, env, self.argument)
         else:
-            return self.function(node, env)
+            list = self.function(node, env)
+        nodes = []
+        for l in list:
+            if not isinstance(l, SCons.Node.FS.Entry):
+                l = self.node_factory(l)
+            nodes.append(l)
+        return nodes
 
     def instance(self, env):
         """
@@ -141,11 +154,8 @@ class Recursive(Base):
         deps = []
         while nodes:
             n = nodes.pop(0)
-            if not self.argument is _null:
-                d = self.function(n, env, self.argument)
-            else:
-                d = self.function(n, env)
-            d = filter(lambda x, seen=seen: not seen.has_key(x), d)
+            d = filter(lambda x, seen=seen: not seen.has_key(x),
+                       Base.scan(self, n, env))
             if d:
                 deps.extend(d)
                 nodes.extend(d)
diff --git a/test/Scanner.py b/test/Scanner.py
new file mode 100644
index 0000000..b44e96a
--- /dev/null
+++ b/test/Scanner.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2001, 2002 Steven Knight
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import sys
+import TestSCons
+
+python = sys.executable
+
+test = TestSCons.TestSCons()
+
+test.write('build.py', r"""
+import sys
+try:
+    input = open(sys.argv[1], 'r')
+except IndexError:
+    input = sys.stdin
+
+def process(fp):
+    for line in fp.readlines():
+        if line[:8] == 'include ':
+            file = line[8:-1]
+	    process(open(file, 'r'))
+        else:
+            sys.stdout.write(line)
+
+process(input)
+
+sys.exit(0)
+""")
+
+# Execute a subsidiary SConscript just to make sure we can
+# get at the SCanners keyword from there.
+
+test.write('SConstruct', """
+SConscript('SConscript')
+""")
+
+test.write('SConscript', """
+import re
+
+include_re = re.compile(r'^include\s+(\S+)$', re.M)
+
+def kfile_scan(node, env, arg):
+    contents = node.get_contents()
+    includes = include_re.findall(contents)
+    return includes
+
+kscan = Scanner(name = 'kfile',
+                function = kfile_scan,
+                argument = None,
+                skeys = ['.k'])
+scanners = Environment().Dictionary('SCANNERS')
+env = Environment(SCANNERS = scanners + [kscan])
+
+env.Command('foo', 'foo.k', '%s build.py < $SOURCES > $TARGET')
+
+bar_in = File('bar.in')
+env.Command('bar', bar_in, '%s build.py $SOURCES > $TARGET')
+bar_in.scanner_set(kscan)
+""" % (python, python))
+
+test.write('foo.k', 
+"""foo.k 1 line 1
+include xxx
+include yyy
+foo.k 1 line 4
+""")
+
+test.write('bar.in', 
+"""include yyy
+bar.in 1 line 2
+bar.in 1 line 3
+include zzz
+""")
+
+test.write('xxx', "xxx 1\n")
+
+test.write('yyy', "yyy 1\n")
+
+test.write('zzz', "zzz 1\n")
+
+test.run(arguments = '.')
+
+test.fail_test(test.read('foo') != "foo.k 1 line 1\nxxx 1\nyyy 1\nfoo.k 1 line 4\n")
+
+test.fail_test(test.read('bar') != "yyy 1\nbar.in 1 line 2\nbar.in 1 line 3\nzzz 1\n")
+
+test.up_to_date(arguments = '.')
+
+test.write('xxx', "xxx 2\n")
+
+test.run(arguments = '.')
+
+test.fail_test(test.read('foo') != "foo.k 1 line 1\nxxx 2\nyyy 1\nfoo.k 1 line 4\n")
+
+test.fail_test(test.read('bar') != "yyy 1\nbar.in 1 line 2\nbar.in 1 line 3\nzzz 1\n")
+
+test.write('yyy', "yyy 2\n")
+
+test.run(arguments = '.')
+
+test.fail_test(test.read('foo') != "foo.k 1 line 1\nxxx 2\nyyy 2\nfoo.k 1 line 4\n")
+
+test.fail_test(test.read('bar') != "yyy 2\nbar.in 1 line 2\nbar.in 1 line 3\nzzz 1\n")
+
+test.write('zzz', "zzz 2\n")
+
+test.run(arguments = '.')
+
+test.fail_test(test.read('foo') != "foo.k 1 line 1\nxxx 2\nyyy 2\nfoo.k 1 line 4\n")
+
+test.fail_test(test.read('bar') != "yyy 2\nbar.in 1 line 2\nbar.in 1 line 3\nzzz 2\n")
+
+test.pass_test()