Add Textfile and Substfile builders

6 files changed, 350 insertions, 17 deletions

diff --git a/doc/scons.mod b/doc/scons.mod
index 7f1b54a..0b01456 100644
--- a/doc/scons.mod
+++ b/doc/scons.mod
@@ -384,7 +384,9 @@
 <!ENTITY SharedObject "<function>SharedObject</function>">
 <!ENTITY StaticLibrary "<function>StaticLibrary</function>">
 <!ENTITY StaticObject "<function>StaticObject</function>">
+<!ENTITY Substfile "<function>Substfile</function>">
 <!ENTITY Tar "<function>Tar</function>">
+<!ENTITY Textfile "<function>Textfile</function>">
 <!ENTITY VariantDir "<function>VariantDir</function>">
 <!ENTITY Zip "<function>Zip</function>">
 
diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 901931d..94405dd 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -38,7 +38,7 @@ RELEASE X.X.X - XXX
 
   From Greg Noel:
 
-    - Remove leftover __metaclass__ initializations in Environment.py.
+    - Remove redundant __metaclass__ initializations in Environment.py.
 
     - Correct the documentation of text returned by sconf.Result().
 
@@ -47,6 +47,11 @@ RELEASE X.X.X - XXX
 
     - Fix SWIG testing infrastructure to work on Mac OS X.
 
+    - Restructure a test that occasionally hung so that the test would
+      detect when it was stuck and fail instead.
+
+    - Add the Textfile and Substfile builders.
+
   From Gary Oberbrunner:
 
     - When reporting a target that SCons doesn't know how to make,
diff --git a/src/engine/MANIFEST-xml.in b/src/engine/MANIFEST-xml.in
index 5a5c9b6..9cc9923 100644
--- a/src/engine/MANIFEST-xml.in
+++ b/src/engine/MANIFEST-xml.in
@@ -6,19 +6,24 @@ SCons/Platform/posix.xml
 SCons/Platform/sunos.xml
 SCons/Platform/win32.xml
 SCons/Tool/386asm.xml
-SCons/Tool/aixcc.xml
+SCons/Tool/BitKeeper.xml
+SCons/Tool/CVS.xml
+SCons/Tool/Perforce.xml
+SCons/Tool/RCS.xml
+SCons/Tool/SCCS.xml
+SCons/Tool/Subversion.xml
+SCons/Tool/__init__.xml
 SCons/Tool/aixc++.xml
+SCons/Tool/aixcc.xml
 SCons/Tool/aixf77.xml
 SCons/Tool/aixlink.xml
 SCons/Tool/applelink.xml
 SCons/Tool/ar.xml
 SCons/Tool/as.xml
 SCons/Tool/bcc32.xml
-SCons/Tool/BitKeeper.xml
+SCons/Tool/c++.xml
 SCons/Tool/cc.xml
 SCons/Tool/cvf.xml
-SCons/Tool/CVS.xml
-SCons/Tool/c++.xml
 SCons/Tool/default.xml
 SCons/Tool/dmd.xml
 SCons/Tool/dvi.xml
@@ -28,31 +33,30 @@ SCons/Tool/f77.xml
 SCons/Tool/f90.xml
 SCons/Tool/f95.xml
 SCons/Tool/fortran.xml
+SCons/Tool/g++.xml
 SCons/Tool/g77.xml
 SCons/Tool/gas.xml
 SCons/Tool/gcc.xml
 SCons/Tool/gnulink.xml
 SCons/Tool/gs.xml
-SCons/Tool/g++.xml
-SCons/Tool/hpcc.xml
 SCons/Tool/hpc++.xml
+SCons/Tool/hpcc.xml
 SCons/Tool/hplink.xml
 SCons/Tool/icc.xml
 SCons/Tool/icl.xml
 SCons/Tool/ifl.xml
 SCons/Tool/ifort.xml
-SCons/Tool/ilink32.xml
 SCons/Tool/ilink.xml
-SCons/Tool/__init__.xml
-SCons/Tool/intelc.xml
+SCons/Tool/ilink32.xml
 SCons/Tool/install.xml
+SCons/Tool/intelc.xml
 SCons/Tool/jar.xml
 SCons/Tool/javac.xml
 SCons/Tool/javah.xml
 SCons/Tool/latex.xml
 SCons/Tool/lex.xml
-SCons/Tool/linkloc.xml
 SCons/Tool/link.xml
+SCons/Tool/linkloc.xml
 SCons/Tool/m4.xml
 SCons/Tool/masm.xml
 SCons/Tool/midl.xml
@@ -70,24 +74,21 @@ SCons/Tool/packaging/__init__.xml
 SCons/Tool/pdf.xml
 SCons/Tool/pdflatex.xml
 SCons/Tool/pdftex.xml
-SCons/Tool/Perforce.xml
 SCons/Tool/qt.xml
-SCons/Tool/RCS.xml
 SCons/Tool/rmic.xml
 SCons/Tool/rpcgen.xml
-SCons/Tool/SCCS.xml
 SCons/Tool/sgiar.xml
-SCons/Tool/sgicc.xml
 SCons/Tool/sgic++.xml
+SCons/Tool/sgicc.xml
 SCons/Tool/sgilink.xml
-SCons/Tool/Subversion.xml
 SCons/Tool/sunar.xml
-SCons/Tool/suncc.xml
 SCons/Tool/sunc++.xml
+SCons/Tool/suncc.xml
 SCons/Tool/sunlink.xml
 SCons/Tool/swig.xml
 SCons/Tool/tar.xml
 SCons/Tool/tex.xml
+SCons/Tool/textfile.xml
 SCons/Tool/tlib.xml
 SCons/Tool/yacc.xml
 SCons/Tool/zip.xml
diff --git a/src/engine/MANIFEST.in b/src/engine/MANIFEST.in
index ede9888..e7c9bd8 100644
--- a/src/engine/MANIFEST.in
+++ b/src/engine/MANIFEST.in
@@ -169,6 +169,7 @@ SCons/Tool/sunlink.py
 SCons/Tool/swig.py
 SCons/Tool/tar.py
 SCons/Tool/tex.py
+SCons/Tool/textfile.py
 SCons/Tool/tlib.py
 SCons/Tool/wix.py
 SCons/Tool/yacc.py
diff --git a/src/engine/SCons/Tool/textfile.py b/src/engine/SCons/Tool/textfile.py
new file mode 100644
index 0000000..8fa2180
--- /dev/null
+++ b/src/engine/SCons/Tool/textfile.py
@@ -0,0 +1,166 @@
+# -*- python -*-
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+"""
+Textfile/Substfile builder for SCons.
+
+    Create file 'target' which typically is a textfile.  The 'source'
+    may be any combination of strings, Nodes, or lists of same.  A
+    'linesep' will be put between any part written and defaults to
+    os.linesep.
+
+    The only difference between the Textfile builder and the Substfile
+    builder is that strings are converted to Value() nodes for the
+    former and File() nodes for the latter.  To insert files in the
+    former or strings in the latter, wrap them in a File() or Value(),
+    respectively.
+
+    The values of SUBST_DICT first have any construction variables
+    expanded (its keys are not expanded).  If a value of SUBST_DICT is
+    a python callable function, it is called and the result is expanded
+    as the value.  Values are substituted in a "random" order; if any
+    substitution could be further expanded by another subsitition, it
+    is unpredictible whether the expansion will occur.
+"""
+
+import SCons
+
+import os
+import re
+
+from SCons.Node import Node
+from SCons.Node.Python import Value
+from SCons.Util import is_List, is_String
+
+def _do_subst(node, subs):
+    """
+    Fetch the node contents and replace all instances of the keys with
+    their values.  For example, if subs is
+        {'%VERSION%': '1.2345', '%BASE%': 'MyProg', '%prefix%': '/bin'},
+    then all instances of %VERSION% in the file will be replaced with
+    1.2345 and so forth.
+    """
+    contents = node.get_text_contents()
+    if not subs: return contents
+    for (k,v) in subs:
+        contents = re.sub(k, v, contents)
+    return contents
+
+def _action(target, source, env):
+    # prepare the line separator
+    linesep = env['LINESEPARATOR']
+    if linesep is None:
+        linesep = os.linesep
+    elif is_String(linesep):
+        pass
+    elif isinstance(linesep, Value):
+        linesep = linesep.get_text_contents()
+    else:
+        raise SCons.Errors.UserError(
+                           'unexpected type/class for LINESEPARATOR: %s'
+                                         % repr(linesep), None)
+
+    # create a dictionary to use for the substitutions
+    if not env.has_key('SUBST_DICT'):
+        subs = None    # no substitutions
+    else:
+        d = env['SUBST_DICT']
+        if SCons.Util.is_Dict(d):
+            d = d.items()
+        elif SCons.Util.is_Sequence(d):
+            pass
+        else:
+            raise SCons.Errors.UserError('SUBST_DICT must be dict or sequence')
+        subs = []
+        for (k,v) in d:
+            if callable(v):
+                v = v()
+            if SCons.Util.is_String(v):
+                v = env.subst(v)
+            else:
+                v = str(v)
+            subs.append((k,v))
+
+    # write the file
+    try:
+        fd = open(target[0].get_path(), "w")
+    except (OSError,IOError), e:
+        raise SCons.Errors.UserError("Can't write target file %s" % target[0])
+    # separate lines by 'linesep' only if linesep is not empty
+    lsep = None
+    for s in source:
+        if lsep: fd.write(lsep)
+        fd.write(_do_subst(s, subs))
+        lsep = linesep
+    fd.close()
+
+def _strfunc(target, source, env):
+    return "Creating '%s'" % target[0]
+
+def _convert_list(target, source, env):
+    if len(target) != 1:
+        raise SCons.Errors.UserError("Only one target file allowed")
+    def _convert_list_R(newlist, sources):
+        for elem in sources:
+            if is_List(elem):
+                _convert_list_R(newlist, elem)
+            elif isinstance(elem, Node):
+                newlist.append(elem)
+            else:
+                newlist.append(Value(elem))
+    newlist = []
+    _convert_list_R(newlist, source)
+    return target, newlist
+
+_common_varlist = ['SUBST_DICT', 'LINESEPARATOR']
+
+_text_varlist = _common_varlist + ['TEXTFILEPREFIX', 'TEXTFILESUFFIX']
+_text_builder = SCons.Builder.Builder(
+    action = SCons.Action.Action(_action, _strfunc, varlist = _text_varlist),
+    source_factory = Value,
+    emitter = _convert_list,
+    prefix = '$TEXTFILEPREFIX',
+    suffix = '$TEXTFILESUFFIX',
+    )
+
+_subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'TEXTFILESUFFIX']
+_subst_builder = SCons.Builder.Builder(
+    action = SCons.Action.Action(_action, _strfunc, varlist = _subst_varlist),
+    source_factory = SCons.Node.FS.File,
+    emitter = _convert_list,
+    prefix = '$SUBSTFILEPREFIX',
+    suffix = '$SUBSTFILESUFFIX',
+    src_suffix = ['.in'],
+    )
+
+def generate(env):
+    env['LINESEPARATOR'] = os.linesep
+    env['BUILDERS']['Textfile'] = _text_builder
+    env['TEXTFILEPREFIX'] = ''
+    env['TEXTFILESUFFIX'] = '.txt'
+    env['BUILDERS']['Substfile'] = _subst_builder
+    env['SUBSTFILEPREFIX'] = ''
+    env['SUBSTFILESUFFIX'] = ''
+
+def exists(env):
+    return 1
diff --git a/test/textfile.py b/test/textfile.py
new file mode 100644
index 0000000..2c9f5d5
--- /dev/null
+++ b/test/textfile.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+import TestSCons
+
+import os
+import string
+
+test = TestSCons.TestSCons()
+
+foo1  = test.workpath('foo1.txt')
+#foo2  = test.workpath('foo2.txt')
+#foo1a = test.workpath('foo1a.txt')
+#foo2a = test.workpath('foo2a.txt')
+
+test.write('SConstruct', """
+env = Environment(tools=['textfile'])
+data0 = ['Goethe', 'Schiller']
+data  = ['lalala', 42, data0, 'tanteratei']
+
+env.Textfile('foo1', data)
+env.Textfile('foo2', data, LINESEPARATOR='|*')
+env.Textfile('foo1a.txt', data + [''])
+env.Textfile('foo2a.txt', data + [''], LINESEPARATOR='|*')
+
+# recreate the list with the data wrapped in Value()
+data0 = map(Value, data0)
+data = map(Value, data)
+data[2] = data0
+
+env.Substfile('bar1', data)
+env.Substfile('bar2', data, LINESEPARATOR='|*')
+data.append(Value(''))
+env.Substfile('bar1a.txt', data)
+env.Substfile('bar2a.txt', data, LINESEPARATOR='|*')
+""")
+
+test.run(arguments = '.')
+
+textparts = ['lalala', '42',
+             'Goethe', 'Schiller',
+             'tanteratei']
+foo1Text  = string.join(textparts, os.linesep)
+foo2Text  = string.join(textparts, '|*')
+foo1aText = foo1Text + os.linesep
+foo2aText = foo2Text + '|*'
+
+test.up_to_date(arguments = '.')
+
+files = map(test.workpath, (
+            'foo1.txt', 'foo2.txt', 'foo1a.txt', 'foo2a.txt',
+            'bar1',     'bar2',     'bar1a.txt', 'bar2a.txt',
+        ))
+def check_times():
+    # make sure the files didn't get rewritten, because nothing changed:
+    before = map(os.path.getmtime, files)
+    # introduce a small delay, to make the test valid
+    test.sleep()
+    # should still be up-to-date
+    test.up_to_date(arguments = '.')
+    after = map(os.path.getmtime, files)
+    test.fail_test(before != after)
+
+# make sure that the file content is as expected
+test.must_match('foo1.txt',  foo1Text)
+test.must_match('bar1',      foo1Text)
+test.must_match('foo2.txt',  foo2Text)
+test.must_match('bar2',      foo2Text)
+test.must_match('foo1a.txt', foo1aText)
+test.must_match('bar1a.txt', foo1aText)
+test.must_match('foo2a.txt', foo2aText)
+test.must_match('bar2a.txt', foo2aText)
+check_times()
+
+# write the contents and make sure the files
+# didn't get rewritten, because nothing changed:
+test.write('foo1.txt',  foo1Text)
+test.write('bar1',      foo1Text)
+test.write('foo2.txt',  foo2Text)
+test.write('bar2',      foo2Text)
+test.write('foo1a.txt', foo1aText)
+test.write('bar1a.txt', foo1aText)
+test.write('foo2a.txt', foo2aText)
+test.write('bar2a.txt', foo2aText)
+check_times()
+
+test.write('SConstruct', """
+textlist = ['This line has no substitutions',
+            'This line has @subst@ substitutions',
+            'This line has %subst% substitutions',
+           ]
+
+sub1 = { '@subst@' : 'most' }
+sub2 = { '%subst%' : 'many' }
+sub3 = { '@subst@' : 'most' , '%subst%' : 'many' }
+
+env = Environment(tools = ['textfile'])
+
+t = env.Textfile('text', textlist)
+# no substitutions
+s = env.Substfile('sub1', t)
+# one substitution
+s = env.Substfile('sub2', s, SUBST_DICT = sub1)
+# the other substution
+s = env.Substfile('sub3', s, SUBST_DICT = sub2)
+# the reverse direction
+s = env.Substfile('sub4', t, SUBST_DICT = sub2)
+s = env.Substfile('sub5', s, SUBST_DICT = sub1)
+# both
+s = env.Substfile('sub6', t, SUBST_DICT = sub3)
+""")
+
+test.run(arguments = '.')
+
+line1  = 'This line has no substitutions'
+line2a = 'This line has @subst@ substitutions'
+line2b = 'This line has most substitutions'
+line3a = 'This line has %subst% substitutions'
+line3b = 'This line has many substitutions'
+
+def matchem(file, lines):
+    lines = string.join(lines, '\n')
+    test.must_match(file, lines)
+
+matchem('text.txt', [line1, line2a, line3a])
+matchem('sub1', [line1, line2a, line3a])
+matchem('sub2', [line1, line2b, line3a])
+matchem('sub3', [line1, line2b, line3b])
+matchem('sub4', [line1, line2a, line3b])
+matchem('sub5', [line1, line2b, line3b])
+matchem('sub6', [line1, line2b, line3b])
+
+test.up_to_date(arguments = '.')
+
+test.pass_test()