diff options
38 files changed, 2413 insertions, 491 deletions
@@ -76,6 +76,7 @@ dh_builddeb = whereis('dh_builddeb') fakeroot = whereis('fakeroot') gzip = whereis('gzip') rpmbuild = whereis('rpmbuild') or whereis('rpm') +hg = whereis('hg') svn = whereis('svn') unzip = whereis('unzip') zip = whereis('zip') @@ -104,12 +105,45 @@ version = ARGUMENTS.get('VERSION', '') if not version: version = default_version +hg_status_lines = [] +svn_status_lines = [] + +if hg: + cmd = "%s status --all 2> /dev/null" % hg + hg_status_lines = os.popen(cmd, "r").readlines() + +if svn: + cmd = "%s status --verbose 2> /dev/null" % svn + svn_status_lines = os.popen(cmd, "r").readlines() + revision = ARGUMENTS.get('REVISION', '') +def generate_build_id(revision): + return revision + +if not revision and hg: + hg_heads = os.popen("%s heads 2> /dev/null" % hg, "r").read() + cs = re.search('changeset:\s+(\S+)', hg_heads) + if cs: + revision = cs.group(1) + b = re.search('branch:\s+(\S+)', hg_heads) + if b: + revision = b.group(1) + ':' + revision + def generate_build_id(revision): + result = revision + if filter(lambda l: l[0] in 'AMR!', hg_status_lines): + result = result + '[MODIFIED]' + return result + if not revision and svn: svn_info = os.popen("%s info 2> /dev/null" % svn, "r").read() m = re.search('Revision: (\d+)', svn_info) if m: revision = m.group(1) + def generate_build_id(revision): + result = 'r' + revision + if filter(lambda l: l[0] in 'ACDMR', svn_status_lines): + result = result + '[MODIFIED]' + return result checkpoint = ARGUMENTS.get('CHECKPOINT', '') if checkpoint: @@ -120,19 +154,10 @@ if checkpoint: checkpoint = 'r' + revision version = version + '.' + checkpoint -svn_status = None -svn_status_lines = [] - -if svn: - svn_status = os.popen("%s status --verbose 2> /dev/null" % svn, "r").read() - svn_status_lines = svn_status[:-1].split('\n') - build_id = ARGUMENTS.get('BUILD_ID') if build_id is None: if revision: - build_id = 'r' + revision - if filter(lambda l: l[0] in 'ACDMR', svn_status_lines): - build_id = build_id + '[MODIFIED]' + build_id = generate_build_id(revision) else: build_id = '' @@ -1173,17 +1198,24 @@ SConscript('doc/SConscript') # source archive from the project files and files in the change. # -if not svn_status: - "Not building in a Subversion tree; skipping building src package." -else: +sfiles = None +if hg_status_lines: + slines = filter(lambda l: l[0] in 'ACM', hg_status_lines) + sfiles = map(lambda l: l.split()[-1], slines) +elif svn_status_lines: slines = filter(lambda l: l[0] in ' MA', svn_status_lines) sentries = map(lambda l: l.split()[-1], slines) sfiles = filter(os.path.isfile, sentries) +else: + "Not building in a Mercurial or Subversion tree; skipping building src package." +if sfiles: remove_patterns = [ + '.hgt/*', '.svnt/*', '*.aeignore', '*.cvsignore', + '*.hgignore', 'www/*', ] diff --git a/doc/man/scons.1 b/doc/man/scons.1 index 5526c5e..d1f9dcd 100644 --- a/doc/man/scons.1 +++ b/doc/man/scons.1 @@ -1481,11 +1481,15 @@ These warnings are enabled by default. --warn=reserved-variable, --warn=no-reserved-variable Enables or disables warnings about attempts to set the reserved construction variable names +.BR CHANGED_SOURCES , +.BR CHANGED_TARGETS , .BR TARGET , .BR TARGETS , -.BR SOURCE +.BR SOURCE , +.BR SOURCES , +.BR UNCHANGED_SOURCES or -.BR SOURCES . +.BR UNCHANGED_TARGETS . These warnings are disabled by default. .TP @@ -8644,16 +8648,25 @@ a = Action(build_it, varlist=['XXX']) The .BR Action () global function -also takes a +can be passed the following +optional keyword arguments +to modify the Action object's behavior: + +.IP .B chdir -keyword argument -which specifies that +The +.B chdir +keyword argument specifies that scons will execute the action after changing to the specified directory. -If the chdir argument is +If the +.B chdir +argument is a string or a directory Node, scons will change to the specified directory. -If the chdir argument +If the +.B chdir +argument is not a string or Node and is non-zero, then scons will change to the @@ -8688,6 +8701,8 @@ a = Action("build < ${SOURCE.file} > ${TARGET.file}", chdir=1) .EE +.IP +.B exitstatfunc The .BR Action () global function @@ -8703,6 +8718,7 @@ or modified value. This can be used, for example, to specify that an Action object's return value should be ignored +under special conditions and SCons should, therefore, consider that the action always suceeds: @@ -8714,6 +8730,111 @@ a = Action("build < ${SOURCE.file} > ${TARGET.file}", exitstatfunc=always_succeed) .EE +.IP +.B batch_key +The +.B batch_key +keyword argument can be used +to specify that the Action can create multiple target files +by processing multiple independent source files simultaneously. +(The canonical example is "batch compilation" +of multiple object files +by passing multiple source files +to a single invocation of a compiler +such as Microsoft's Visual C / C++ compiler.) +If the +.B batch_key +argument is any non-False, non-callable Python value, +the configured Action object will cause +.B scons +to collect all targets built with the Action object +and configured with the same construction environment +into single invocations of the Action object's +command line or function. +Command lines will typically want to use the +.BR CHANGED_SOURCES +construction variable +(and possibly +.BR CHANGED_TARGETS +as well) +to only pass to the command line those sources that +have actually changed since their targets were built. + +Example: + +.ES +a = Action('build $CHANGED_SOURCES', batch_key=True) +.EE + +The +.B batch_key +argument may also be +a callable function +that returns a key that +will be used to identify different +"batches" of target files to be collected +for batch building. +A +.B batch_key +function must take the following arguments: + +.IP action +The action object. + +.IP env +The construction environment +configured for the target. + +.IP target +The list of targets for a particular configured action. + +.IP source +The list of source for a particular configured action. + +The returned key should typically +be a tuple of values derived from the arguments, +using any appropriate logic to decide +how multiple invocations should be batched. +For example, a +.B batch_key +function may decide to return +the value of a specific construction +variable from the +.B env +argument +which will cause +.B scons +to batch-build targets +with matching values of that variable, +or perhaps return the +.BR id () +of the entire construction environment, +in which case +.B scons +will batch-build +all targets configured with the same construction environment. +Returning +.B None +indicates that +the particular target should +.I not +be part of any batched build, +but instead will be built +by a separate invocation of action's +command or function. +Example: + +.ES +def batch_key(action, env, target, source): + tdir = target[0].dir + if tdir.name == 'special': + # Don't batch-build any target + # in the special/ subdirectory. + return None + return (id(action), id(env), tdir) +a = Action('build $CHANGED_SOURCES', batch_key=batch_key) +.EE + .SS Miscellaneous Action Functions .B scons @@ -8906,20 +9027,42 @@ prefix. Besides construction variables, scons provides the following variables for each command execution: -.IP TARGET -The file name of the target being built, or the file name of the first -target if multiple targets are being built. +.IP CHANGED_SOURCES +The file names of all sources of the build command +that have changed since the target was last built. -.IP TARGETS -The file names of all targets being built. +.IP CHANGED_TARGETS +The file names of all targets that would be built +from sources that have changed since the target was last built. .IP SOURCE -The file name of the source of the build command, or the file name of the -first source if multiple sources are being built. +The file name of the source of the build command, +or the file name of the first source +if multiple sources are being built. .IP SOURCES The file names of the sources of the build command. +.IP TARGET +The file name of the target being built, +or the file name of the first target +if multiple targets are being built. + +.IP TARGETS +The file names of all targets being built. + +.IP UNCHANGED_SOURCES +The file names of all sources of the build command +that have +.I not +changed since the target was last built. + +.IP UNCHANGED_TARGETS +The file names of all targets that would be built +from sources that have +.I not +changed since the target was last built. + (Note that the above variables are reserved and may not be set in a construction environment.) diff --git a/doc/scons.mod b/doc/scons.mod index 767b40c..a2927cf 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -401,6 +401,8 @@ --> <!ENTITY buildfunc "<literal>builder function</literal>"> +<!ENTITY build_action "<literal>build action</literal>"> +<!ENTITY build_actions "<literal>build actions</literal>"> <!ENTITY builder_method "<literal>builder method</literal>"> <!ENTITY Configure_Contexts "<literal>Configure Contexts</literal>"> @@ -444,6 +446,19 @@ <!-- + Python keyword arguments + +--> + +<!ENTITY action "<literal>action=</literal>"> +<!ENTITY batch_key "<literal>batch_key=</literal>"> +<!ENTITY cmdstr "<literal>cmdstr=</literal>"> +<!ENTITY exitstatfunc "<literal>exitstatfunc=</literal>"> +<!ENTITY strfunction "<literal>strfunction=</literal>"> +<!ENTITY varlist "<literal>varlist=</literal>"> + +<!-- + File and program names used in examples. --> diff --git a/doc/user/actions.in b/doc/user/actions.in index 928b7ea..c1e5616 100644 --- a/doc/user/actions.in +++ b/doc/user/actions.in @@ -222,19 +222,183 @@ solutions to the above limitations. --> - <para> + <para> - XXX + &SCons; supports several types of &build_actions; + that can be performed to build one or more target files. + Usually, a &build_action; is a command-line string + that invokes an external command. + A build action can also be an external command + specified as a list of arguments, + or even a Python function. - </para> + </para> - <section> - <title>XXX</title> + <para> - <para> + Build action objects are created by the &Action; function. + This function is, in fact, what &SCons; uses + to interpret the &action; + keyword argument when you call the &Builder; function. + So the following line that creates a simple Builder: - XXX + </para> - </para> + <sconstruct> + b = Builder(action = 'build < $SOURCE > $TARGET') + </sconstruct> - </section> + <para> + + Is equivalent to: + + </para> + + <sconstruct> + b = Builder(action = Action('build < $SOURCE > $TARGET')) + </sconstruct> + + <para> + + The advantage of using the &Action; function directly + is that it can take a number of additional options + to modify the action's behavior in many useful ways. + + </para> + + <section> + <title>Command Strings as Actions</title> + + <section> + <title>Suppressing Command-Line Printing</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Ignoring Exit Status</title> + + <para> + + XXX + + </para> + + </section> + + </section> + + <section> + <title>Argument Lists as Actions</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Python Functions as Actions</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Modifying How an Action is Printed</title> + + <section> + <title>XXX: the &strfunction; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>XXX: the &cmdstr; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + </section> + + <section> + <title>Making an Action Depend on Variable Contents: the &varlist; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>chdir=1</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Batch Building of Multiple Targets from Separate Sources: the &batch_key; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Manipulating the Exit Status of an Action: the &exitstatfunc; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <!-- + + ??? + + <section> + <title>presub=</title> + + <para> + + XXX + + </para> + + </section> + + --> diff --git a/doc/user/actions.xml b/doc/user/actions.xml index 928b7ea..04178b0 100644 --- a/doc/user/actions.xml +++ b/doc/user/actions.xml @@ -222,19 +222,183 @@ solutions to the above limitations. --> - <para> + <para> - XXX + &SCons; supports several types of &build_actions; + that can be performed to build one or more target files. + Usually, a &build_action; is a command-line string + that invokes an external command. + A build action can also be an external command + specified as a list of arguments, + or even a Python function. - </para> + </para> - <section> - <title>XXX</title> + <para> - <para> + Build action objects are created by the &Action; function. + This function is, in fact, what &SCons; uses + to interpret the &action; + keyword argument when you call the &Builder; function. + So the following line that creates a simple Builder: - XXX + </para> - </para> + <programlisting> + b = Builder(action = 'build < $SOURCE > $TARGET') + </programlisting> - </section> + <para> + + Is equivalent to: + + </para> + + <programlisting> + b = Builder(action = Action('build < $SOURCE > $TARGET')) + </programlisting> + + <para> + + The advantage of using the &Action; function directly + is that it can take a number of additional options + to modify the action's behavior in many useful ways. + + </para> + + <section> + <title>Command Strings as Actions</title> + + <section> + <title>Suppressing Command-Line Printing</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Ignoring Exit Status</title> + + <para> + + XXX + + </para> + + </section> + + </section> + + <section> + <title>Argument Lists as Actions</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Python Functions as Actions</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Modifying How an Action is Printed</title> + + <section> + <title>XXX: the &strfunction; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>XXX: the &cmdstr; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + </section> + + <section> + <title>Making an Action Depend on Variable Contents: the &varlist; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>chdir=1</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Batch Building of Multiple Targets from Separate Sources: the &batch_key; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <section> + <title>Manipulating the Exit Status of an Action: the &exitstatfunc; keyword argument</title> + + <para> + + XXX + + </para> + + </section> + + <!-- + + ??? + + <section> + <title>presub=</title> + + <para> + + XXX + + </para> + + </section> + + --> diff --git a/doc/user/builders-writing.in b/doc/user/builders-writing.in index dc6e95b..2460b37 100644 --- a/doc/user/builders-writing.in +++ b/doc/user/builders-writing.in @@ -102,7 +102,7 @@ This functionality could be invoked as in the following example: programs, libraries, documents. you frequently want to be able to build some other type of file - not supported directly by &SCons; + not supported directly by &SCons;. Fortunately, &SCons; makes it very easy to define your own &Builder; objects for any custom file types you want to build. @@ -186,7 +186,8 @@ This functionality could be invoked as in the following example: <para> - With the &Builder; so attached to our &consenv; + With the &Builder; attached to our &consenv; + with the name &Foo;, we can now actually call it like so: </para> @@ -859,164 +860,198 @@ This functionality could be invoked as in the following example: </section> + <!-- + + <section> + <title>target_factor=, source_factory=</title> + + </section> + + <section> + <title>target_scanner=, source_scanner=</title> + + </section> + + <section> + <title>multi=</title> + + </section> + + <section> + <title>single_source=</title> + + </section> + + <section> + <title>src_builder=</title> + + </section> + + <section> + <title>ensure_suffix=</title> + + </section> + + --> + <section> <title>Where To Put Your Custom Builders and Tools</title> - <para> + <para> - The <filename>site_scons</filename> directory gives you a place to - put Python modules you can import into your SConscripts - (site_scons), add-on tools that can integrate into &SCons; - (site_scons/site_tools), and a site_scons/site_init.py file that - gets read before any &SConstruct; or &SConscript;, allowing you to - change &SCons;'s default behavior. + The <filename>site_scons</filename> directory gives you a place to + put Python modules you can import into your SConscripts + (site_scons), add-on tools that can integrate into &SCons; + (site_scons/site_tools), and a site_scons/site_init.py file that + gets read before any &SConstruct; or &SConscript;, allowing you to + change &SCons;'s default behavior. - </para> + </para> - <para> + <para> - If you get a tool from somewhere (the &SCons; wiki or a third party, - for instance) and you'd like to use it in your project, the - <filename>site_scons</filename> dir is the simplest place to put it. - Tools come in two flavors; either a Python function that operates on - an &Environment; or a Python file containing two functions, exists() - and generate(). + If you get a tool from somewhere (the &SCons; wiki or a third party, + for instance) and you'd like to use it in your project, the + <filename>site_scons</filename> dir is the simplest place to put it. + Tools come in two flavors; either a Python function that operates on + an &Environment; or a Python file containing two functions, exists() + and generate(). - </para> + </para> - <para> + <para> - A single-function Tool can just be included in your - <filename>site_scons/site_init.py</filename> file where it will be - parsed and made available for use. For instance, you could have a - <filename>site_scons/site_init.py</filename> file like this: + A single-function Tool can just be included in your + <filename>site_scons/site_init.py</filename> file where it will be + parsed and made available for use. For instance, you could have a + <filename>site_scons/site_init.py</filename> file like this: - </para> + </para> - <scons_example name="site1"> - <file name="site_scons/site_init.py" printme=1> - def TOOL_ADD_HEADER(env): - """A Tool to add a header from $HEADER to the source file""" - add_header = Builder(action=['echo "$HEADER" > $TARGET', - 'cat $SOURCE >> $TARGET']) - env.Append(BUILDERS = {'AddHeader' : add_header}) - env['HEADER'] = '' # set default value - </file> - <file name="SConstruct"> - env=Environment(tools=['default', TOOL_ADD_HEADER], HEADER="=====") - env.AddHeader('tgt', 'src') - </file> - <file name="src"> - hi there - </file> - </scons_example> + <scons_example name="site1"> + <file name="site_scons/site_init.py" printme=1> + def TOOL_ADD_HEADER(env): + """A Tool to add a header from $HEADER to the source file""" + add_header = Builder(action=['echo "$HEADER" > $TARGET', + 'cat $SOURCE >> $TARGET']) + env.Append(BUILDERS = {'AddHeader' : add_header}) + env['HEADER'] = '' # set default value + </file> + <file name="SConstruct"> + env=Environment(tools=['default', TOOL_ADD_HEADER], HEADER="=====") + env.AddHeader('tgt', 'src') + </file> + <file name="src"> + hi there + </file> + </scons_example> - <para> + <para> - and a &SConstruct; like this: + and a &SConstruct; like this: - </para> + </para> - <sconstruct> - # Use TOOL_ADD_HEADER from site_scons/site_init.py - env=Environment(tools=['default', TOOL_ADD_HEADER], HEADER="=====") - env.AddHeader('tgt', 'src') - </sconstruct> + <sconstruct> + # Use TOOL_ADD_HEADER from site_scons/site_init.py + env=Environment(tools=['default', TOOL_ADD_HEADER], HEADER="=====") + env.AddHeader('tgt', 'src') + </sconstruct> - <para> + <para> - The <function>TOOL_ADD_HEADER</function> tool method will be - called to add the <function>AddHeader</function> tool to the - environment. + The <function>TOOL_ADD_HEADER</function> tool method will be + called to add the <function>AddHeader</function> tool to the + environment. - </para> + </para> - <!-- - <scons_output example="site1" os="posix"> - <scons_output_command>scons -Q</scons_output_command> - </scons_output> - --> + <!-- + <scons_output example="site1" os="posix"> + <scons_output_command>scons -Q</scons_output_command> + </scons_output> + --> - <para> - Similarly, a more full-fledged tool with - <function>exists()</function> and <function>generate()</function> - methods can be installed in - <filename>site_scons/site_tools/toolname.py</filename>. Since - <filename>site_scons/site_tools</filename> is automatically added - to the head of the tool search path, any tool found there will be - available to all environments. Furthermore, a tool found there - will override a built-in tool of the same name, so if you need to - change the behavior of a built-in tool, site_scons gives you the - hook you need. - </para> + <para> + Similarly, a more full-fledged tool with + <function>exists()</function> and <function>generate()</function> + methods can be installed in + <filename>site_scons/site_tools/toolname.py</filename>. Since + <filename>site_scons/site_tools</filename> is automatically added + to the head of the tool search path, any tool found there will be + available to all environments. Furthermore, a tool found there + will override a built-in tool of the same name, so if you need to + change the behavior of a built-in tool, site_scons gives you the + hook you need. + </para> - <para> - Many people have a library of utility Python functions they'd like - to include in &SConscript;s; just put that module in - <filename>site_scons/my_utils.py</filename> or any valid Python module name of your - choice. For instance you can do something like this in - <filename>site_scons/my_utils.py</filename> to add build_id and MakeWorkDir functions: - </para> - - <scons_example name="site2"> - <file name="site_scons/my_utils.py" printme=1> - from SCons.Script import * # for Execute and Mkdir - def build_id(): - """Return a build ID (stub version)""" - return "100" - def MakeWorkDir(workdir): - """Create the specified dir immediately""" - Execute(Mkdir(workdir)) - </file> - <file name="SConscript"> - import my_utils - MakeWorkDir('/tmp/work') - print "build_id=" + my_utils.build_id() - </file> - </scons_example> + <para> + Many people have a library of utility Python functions they'd like + to include in &SConscript;s; just put that module in + <filename>site_scons/my_utils.py</filename> or any valid Python module name of your + choice. For instance you can do something like this in + <filename>site_scons/my_utils.py</filename> to add build_id and MakeWorkDir functions: + </para> + + <scons_example name="site2"> + <file name="site_scons/my_utils.py" printme=1> + from SCons.Script import * # for Execute and Mkdir + def build_id(): + """Return a build ID (stub version)""" + return "100" + def MakeWorkDir(workdir): + """Create the specified dir immediately""" + Execute(Mkdir(workdir)) + </file> + <file name="SConscript"> + import my_utils + MakeWorkDir('/tmp/work') + print "build_id=" + my_utils.build_id() + </file> + </scons_example> - <para> + <para> - And then in your &SConscript; or any sub-&SConscript; anywhere in - your build, you can import <filename>my_utils</filename> and use it: + And then in your &SConscript; or any sub-&SConscript; anywhere in + your build, you can import <filename>my_utils</filename> and use it: - </para> + </para> - <sconstruct> - import my_utils - print "build_id=" + my_utils.build_id() - my_utils.MakeWorkDir('/tmp/work') - </sconstruct> + <sconstruct> + import my_utils + print "build_id=" + my_utils.build_id() + my_utils.MakeWorkDir('/tmp/work') + </sconstruct> - <para> - Note that although you can put this library in - <filename>site_scons/site_init.py</filename>, - it is no better there than <filename>site_scons/my_utils.py</filename> - since you still have to import that module into your &SConscript;. - Also note that in order to refer to objects in the SCons namespace - such as &Environment; or &Mkdir; or &Execute; in any file other - than a &SConstruct; or &SConscript; you always need to do - </para> - <sconstruct> - from SCons.Script import * - </sconstruct> + <para> + Note that although you can put this library in + <filename>site_scons/site_init.py</filename>, + it is no better there than <filename>site_scons/my_utils.py</filename> + since you still have to import that module into your &SConscript;. + Also note that in order to refer to objects in the SCons namespace + such as &Environment; or &Mkdir; or &Execute; in any file other + than a &SConstruct; or &SConscript; you always need to do + </para> + <sconstruct> + from SCons.Script import * + </sconstruct> - <para> - This is true in modules in <filename>site_scons</filename> such as - <filename>site_scons/site_init.py</filename> as well. - </para> + <para> + This is true in modules in <filename>site_scons</filename> such as + <filename>site_scons/site_init.py</filename> as well. + </para> - <para> + <para> - If you have a machine-wide site dir you'd like to use instead of - <filename>./site_scons</filename>, use the - <literal>--site-dir</literal> option to point to your dir. - <filename>site_init.py</filename> and - <filename>site_tools</filename> will be located under that dir. - To avoid using a <filename>site_scons</filename> dir at all, even - if it exists, use the <literal>--no-site-dir</literal> option. + If you have a machine-wide site dir you'd like to use instead of + <filename>./site_scons</filename>, use the + <literal>--site-dir</literal> option to point to your dir. + <filename>site_init.py</filename> and + <filename>site_tools</filename> will be located under that dir. + To avoid using a <filename>site_scons</filename> dir at all, even + if it exists, use the <literal>--no-site-dir</literal> option. - </para> + </para> </section> diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 76cca21..3cdc304 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -10,6 +10,11 @@ RELEASE 1.X - XXX + From Stanislav Baranov, Ted Johnson and Steven Knight: + + - Add support for batch compilation of Visual Studio C/C++ source + files, controlled by a new MSVC_BATCH construction variable. + From Steven Knight: - Print the message, "scons: Build interrupted." on error output, @@ -21,6 +26,12 @@ RELEASE 1.X - XXX - Fix use of $SOURCE and $SOURCES attributes when there are no sources specified in the Builder call. + - Add support for new $CHANGED_SOURCES, $CHANGED_TARGETS, + $UNCHANGED_SOURCES and $UNCHANGED_TARGETS variables. + + - Add general support for batch builds through new batch_key= and + targets= keywords to Action object creation. + From Arve Knudsen: - Make linker tools differentiate properly between SharedLibrary diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index bc1d724..e106e74 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -392,7 +392,7 @@ def _do_create_list_action(act, kw): aa = _do_create_action(a, kw) if aa is not None: acts.append(aa) if not acts: - return None + return ListAction([]) elif len(acts) == 1: return acts[0] else: @@ -414,6 +414,11 @@ class ActionBase: def __cmp__(self, other): return cmp(self.__dict__, other) + def no_batch_key(self, env, target, source): + return None + + batch_key = no_batch_key + def genstring(self, target, source, env): return str(self) @@ -446,15 +451,18 @@ class ActionBase: self.presub_env = None # don't need this any more return lines - def get_executor(self, env, overrides, tlist, slist, executor_kw): - """Return the Executor for this Action.""" - return SCons.Executor.Executor(self, env, overrides, - tlist, slist, executor_kw) + def get_targets(self, env, executor): + """ + Returns the type of targets ($TARGETS, $CHANGED_TARGETS) used + by this action. + """ + return self.targets class _ActionAction(ActionBase): """Base class for actions that create output objects.""" def __init__(self, cmdstr=_null, strfunction=_null, varlist=(), presub=_null, chdir=None, exitstatfunc=None, + batch_key=None, targets='$TARGETS', **kw): self.cmdstr = cmdstr if strfunction is not _null: @@ -469,6 +477,19 @@ class _ActionAction(ActionBase): exitstatfunc = default_exitstatfunc self.exitstatfunc = exitstatfunc + self.targets = targets + + if batch_key: + if not callable(batch_key): + # They have set batch_key, but not to their own + # callable. The default behavior here will batch + # *all* targets+sources using this action, separated + # for each construction environment. + def default_batch_key(self, env, target, source): + return (id(self), id(env)) + batch_key = default_batch_key + SCons.Util.AddMethod(self, batch_key, 'batch_key') + def print_cmd_line(self, s, target, source, env): sys.stdout.write(s + "\n") @@ -477,7 +498,8 @@ class _ActionAction(ActionBase): presub=_null, show=_null, execute=_null, - chdir=_null): + chdir=_null, + executor=None): if not is_List(target): target = [target] if not is_List(source): @@ -498,15 +520,27 @@ class _ActionAction(ActionBase): chdir = str(chdir.abspath) except AttributeError: if not is_String(chdir): - chdir = str(target[0].dir) + if executor: + chdir = str(executor.batches[0].targets[0].dir) + else: + chdir = str(target[0].dir) if presub: + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() t = string.join(map(str, target), ' and ') l = string.join(self.presub_lines(env), '\n ') out = "Building %s with action:\n %s\n" % (t, l) sys.stdout.write(out) cmd = None if show and self.strfunction: - cmd = self.strfunction(target, source, env) + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() + try: + cmd = self.strfunction(target, source, env, executor) + except TypeError: + cmd = self.strfunction(target, source, env) if cmd: if chdir: cmd = ('os.chdir(%s)\n' % repr(chdir)) + cmd @@ -524,7 +558,7 @@ class _ActionAction(ActionBase): if chdir: os.chdir(chdir) try: - stat = self.execute(target, source, env) + stat = self.execute(target, source, env, executor=executor) if isinstance(stat, SCons.Errors.BuildError): s = exitstatfunc(stat.status) if s: @@ -657,8 +691,11 @@ class CommandAction(_ActionAction): return string.join(map(str, self.cmd_list), ' ') return str(self.cmd_list) - def process(self, target, source, env): - result = env.subst_list(self.cmd_list, 0, target, source) + def process(self, target, source, env, executor=None): + if executor: + result = env.subst_list(self.cmd_list, 0, executor=executor) + else: + result = env.subst_list(self.cmd_list, 0, target, source) silent = None ignore = None while 1: @@ -675,20 +712,23 @@ class CommandAction(_ActionAction): pass return result, ignore, silent - def strfunction(self, target, source, env): + def strfunction(self, target, source, env, executor=None): if self.cmdstr is None: return None if self.cmdstr is not _null: from SCons.Subst import SUBST_RAW - c = env.subst(self.cmdstr, SUBST_RAW, target, source) + if executor: + c = env.subst(self.cmdstr, SUBST_RAW, executor=executor) + else: + c = env.subst(self.cmdstr, SUBST_RAW, target, source) if c: return c - cmd_list, ignore, silent = self.process(target, source, env) + cmd_list, ignore, silent = self.process(target, source, env, executor) if silent: return '' return _string_from_cmd_list(cmd_list[0]) - def execute(self, target, source, env): + def execute(self, target, source, env, executor=None): """Execute a command action. This will handle lists of commands as well as individual commands, @@ -733,7 +773,10 @@ class CommandAction(_ActionAction): # reasonable for just about everything else: ENV[key] = str(value) - cmd_list, ignore, silent = self.process(target, map(rfile, source), env) + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() + cmd_list, ignore, silent = self.process(target, map(rfile, source), env, executor) # Use len() to filter out any "command" that's zero-length. for cmd_line in filter(len, cmd_list): @@ -748,7 +791,7 @@ class CommandAction(_ActionAction): command=cmd_line) return 0 - def get_presig(self, target, source, env): + def get_presig(self, target, source, env, executor=None): """Return the signature contents of this action's command line. This strips $(-$) and everything in between the string, @@ -760,16 +803,22 @@ class CommandAction(_ActionAction): cmd = string.join(map(str, cmd)) else: cmd = str(cmd) - return env.subst_target_source(cmd, SUBST_SIG, target, source) + if executor: + return env.subst_target_source(cmd, SUBST_SIG, executor=executor) + else: + return env.subst_target_source(cmd, SUBST_SIG, target, source) - def get_implicit_deps(self, target, source, env): + def get_implicit_deps(self, target, source, env, executor=None): icd = env.get('IMPLICIT_COMMAND_DEPENDENCIES', True) if is_String(icd) and icd[:1] == '$': icd = env.subst(icd) if not icd or icd in ('0', 'None'): return [] from SCons.Subst import SUBST_SIG - cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, target, source) + if executor: + cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, executor=executor) + else: + cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, target, source) res = [] for cmd_line in cmd_list: if cmd_line: @@ -785,14 +834,21 @@ class CommandGeneratorAction(ActionBase): self.generator = generator self.gen_kw = kw self.varlist = kw.get('varlist', ()) + self.targets = kw.get('targets', '$TARGETS') - def _generate(self, target, source, env, for_signature): + def _generate(self, target, source, env, for_signature, executor=None): # ensure that target is a list, to make it easier to write # generator functions: if not is_List(target): target = [target] - ret = self.generator(target=target, source=source, env=env, for_signature=for_signature) + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() + ret = self.generator(target=target, + source=source, + env=env, + for_signature=for_signature) #TODO(1.5) gen_cmd = Action(ret, **self.gen_kw) gen_cmd = apply(Action, (ret,), self.gen_kw) if not gen_cmd: @@ -809,25 +865,33 @@ class CommandGeneratorAction(ActionBase): act = self._generate([], [], env, 1) return str(act) - def genstring(self, target, source, env): - return self._generate(target, source, env, 1).genstring(target, source, env) + def batch_key(self, env, target, source): + return self._generate(target, source, env, 1).batch_key(env, target, source) + + def genstring(self, target, source, env, executor=None): + return self._generate(target, source, env, 1, executor).genstring(target, source, env) def __call__(self, target, source, env, exitstatfunc=_null, presub=_null, - show=_null, execute=_null, chdir=_null): - act = self._generate(target, source, env, 0) + show=_null, execute=_null, chdir=_null, executor=None): + act = self._generate(target, source, env, 0, executor) + if act is None: + raise UserError("While building `%s': Cannot deduce file extension from source files: %s" % (repr(map(str, target)), repr(map(str, source)))) return act(target, source, env, exitstatfunc, presub, - show, execute, chdir) + show, execute, chdir, executor) - def get_presig(self, target, source, env): + def get_presig(self, target, source, env, executor=None): """Return the signature contents of this action's command line. This strips $(-$) and everything in between the string, since those parts don't affect signatures. """ - return self._generate(target, source, env, 1).get_presig(target, source, env) + return self._generate(target, source, env, 1, executor).get_presig(target, source, env) - def get_implicit_deps(self, target, source, env): - return self._generate(target, source, env, 1).get_implicit_deps(target, source, env) + def get_implicit_deps(self, target, source, env, executor=None): + return self._generate(target, source, env, 1, executor).get_implicit_deps(target, source, env) + + def get_targets(self, env, executor): + return self._generate(None, None, env, 1, executor).get_targets(env, executor) @@ -864,14 +928,17 @@ class LazyAction(CommandGeneratorAction, CommandAction): return CommandGeneratorAction def _generate_cache(self, env): - c = env.get(self.var, '') + if env: + c = env.get(self.var, '') + else: + c = '' #TODO(1.5) gen_cmd = Action(c, **self.gen_kw) gen_cmd = apply(Action, (c,), self.gen_kw) if not gen_cmd: raise SCons.Errors.UserError("$%s value %s cannot be used to create an Action." % (self.var, repr(c))) return gen_cmd - def _generate(self, target, source, env, for_signature): + def _generate(self, target, source, env, for_signature, executor=None): return self._generate_cache(env) def __call__(self, target, source, env, *args, **kw): @@ -915,12 +982,15 @@ class FunctionAction(_ActionAction): except AttributeError: return "unknown_python_function" - def strfunction(self, target, source, env): + def strfunction(self, target, source, env, executor=None): if self.cmdstr is None: return None if self.cmdstr is not _null: from SCons.Subst import SUBST_RAW - c = env.subst(self.cmdstr, SUBST_RAW, target, source) + if executor: + c = env.subst(self.cmdstr, SUBST_RAW, executor=executor) + else: + c = env.subst(self.cmdstr, SUBST_RAW, target, source) if c: return c def array(a): @@ -953,9 +1023,12 @@ class FunctionAction(_ActionAction): return str(self.execfunction) return "%s(target, source, env)" % name - def execute(self, target, source, env): + def execute(self, target, source, env, executor=None): exc_info = (None,None,None) try: + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() rsources = map(rfile, source) try: result = self.execfunction(target=target, source=rsources, env=env) @@ -971,7 +1044,10 @@ class FunctionAction(_ActionAction): result = SCons.Errors.convert_to_BuildError(result, exc_info) result.node=target result.action=self - result.command=self.strfunction(target, source, env) + try: + result.command=self.strfunction(target, source, env, executor) + except TypeError: + result.command=self.strfunction(target, source, env) # FIXME: This maintains backward compatibility with respect to # which type of exceptions were returned by raising an @@ -1013,6 +1089,7 @@ class ListAction(ActionBase): # our children will have had any varlist # applied; we don't need to do it again self.varlist = () + self.targets = '$TARGETS' def genstring(self, target, source, env): return string.join(map(lambda a, t=target, s=source, e=env: @@ -1038,10 +1115,13 @@ class ListAction(ActionBase): "") def __call__(self, target, source, env, exitstatfunc=_null, presub=_null, - show=_null, execute=_null, chdir=_null): + show=_null, execute=_null, chdir=_null, executor=None): + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() for act in self.list: stat = act(target, source, env, exitstatfunc, presub, - show, execute, chdir) + show, execute, chdir, executor) if stat: return stat return 0 @@ -1111,7 +1191,7 @@ class ActionCaller: kw[key] = self.subst(self.kw[key], target, source, env) return kw - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): args = self.subst_args(target, source, env) kw = self.subst_kw(target, source, env) #TODO(1.5) return self.parent.actfunc(*args, **kw) diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index 643e9fa..ae6a15c 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -405,13 +405,6 @@ class ActionTestCase(unittest.TestCase): a2 = SCons.Action.Action(a1) assert a2 is a1, a2 -class ActionBaseTestCase(unittest.TestCase): - def test_get_executor(self): - """Test the ActionBase.get_executor() method""" - a = SCons.Action.Action('foo') - x = a.get_executor({}, {}, [], [], {}) - assert x is not None, x - class _ActionActionTestCase(unittest.TestCase): def test__init__(self): @@ -1589,13 +1582,14 @@ class FunctionActionTestCase(unittest.TestCase): c = test.read(outfile, 'r') assert c == "class1b\n", c - def build_it(target, source, env, self=self): + def build_it(target, source, env, executor=None, self=self): self.build_it = 1 return 0 - def string_it(target, source, env, self=self): + def string_it(target, source, env, executor=None, self=self): self.string_it = 1 return None - act = SCons.Action.FunctionAction(build_it, { 'strfunction' : string_it }) + act = SCons.Action.FunctionAction(build_it, + { 'strfunction' : string_it }) r = act([], [], Environment()) assert r == 0, r assert self.build_it @@ -1996,8 +1990,7 @@ class ActionCompareTestCase(unittest.TestCase): if __name__ == "__main__": suite = unittest.TestSuite() - tclasses = [ ActionBaseTestCase, - _ActionActionTestCase, + tclasses = [ _ActionActionTestCase, ActionTestCase, CommandActionTestCase, CommandGeneratorActionTestCase, diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index 1d29516..21638a5 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -118,6 +118,15 @@ class _Null: _null = _Null +def match_splitext(path, suffixes = []): + if suffixes: + matchsuf = filter(lambda S,path=path: path[-len(S):] == S, + suffixes) + if matchsuf: + suf = max(map(None, map(len, matchsuf), matchsuf))[1] + return [path[:-len(suf)], path[-len(suf):]] + return SCons.Util.splitext(path) + class DictCmdGenerator(SCons.Util.Selector): """This is a callable class that can be used as a command generator function. It holds on to a dictionary @@ -142,20 +151,22 @@ class DictCmdGenerator(SCons.Util.Selector): return [] if self.source_ext_match: + suffixes = self.src_suffixes() ext = None for src in map(str, source): - my_ext = SCons.Util.splitext(src)[1] + my_ext = match_splitext(src, suffixes)[1] if ext and my_ext != ext: raise UserError("While building `%s' from `%s': Cannot build multiple sources with different extensions: %s, %s" % (repr(map(str, target)), src, ext, my_ext)) ext = my_ext else: - ext = SCons.Util.splitext(str(source[0]))[1] + ext = match_splitext(str(source[0]), self.src_suffixes())[1] if not ext: + #return ext raise UserError("While building `%s': Cannot deduce file extension from source files: %s" % (repr(map(str, target)), repr(map(str, source)))) try: - ret = SCons.Util.Selector.__call__(self, env, source) + ret = SCons.Util.Selector.__call__(self, env, source, ext) except KeyError, e: raise UserError("Ambiguous suffixes after environment substitution: %s == %s == %s" % (e[0], e[1], e[2])) if ret is None: @@ -295,8 +306,9 @@ def _node_errors(builder, env, tlist, slist): if t.builder != builder: msg = "Two different builders (%s and %s) were specified for the same target: %s" % (t.builder.get_name(env), builder.get_name(env), t) raise UserError, msg - if t.get_executor().targets != tlist: - msg = "Two different target lists have a target in common: %s (from %s and from %s)" % (t, map(str, t.get_executor().targets), map(str, tlist)) + # TODO(batch): list constructed each time! + if t.get_executor().get_all_targets() != tlist: + msg = "Two different target lists have a target in common: %s (from %s and from %s)" % (t, map(str, t.get_executor().get_all_targets()), map(str, tlist)) raise UserError, msg elif t.sources != slist: msg = "Multiple ways to build the same target were specified for: %s (from %s and from %s)" % (t, map(str, t.sources), map(str, slist)) @@ -441,30 +453,10 @@ class BuilderBase: if not env: env = self.env if env: - matchsuf = filter(lambda S,path=path: path[-len(S):] == S, - self.src_suffixes(env)) - if matchsuf: - suf = max(map(None, map(len, matchsuf), matchsuf))[1] - return [path[:-len(suf)], path[-len(suf):]] - return SCons.Util.splitext(path) - - def get_single_executor(self, env, tlist, slist, executor_kw): - if not self.action: - raise UserError, "Builder %s must have an action to build %s."%(self.get_name(env or self.env), map(str,tlist)) - return self.action.get_executor(env or self.env, - [], # env already has overrides - tlist, - slist, - executor_kw) - - def get_multi_executor(self, env, tlist, slist, executor_kw): - try: - executor = tlist[0].get_executor(create = 0) - except (AttributeError, IndexError): - return self.get_single_executor(env, tlist, slist, executor_kw) + suffixes = self.src_suffixes(env) else: - executor.add_sources(slist) - return executor + suffixes = [] + return match_splitext(path, suffixes) def _adjustixes(self, files, pre, suf, ensure_suffix=False): if not files: @@ -566,11 +558,37 @@ class BuilderBase: # The targets are fine, so find or make the appropriate Executor to # build this particular list of targets from this particular list of # sources. + + executor = None + key = None + if self.multi: - get_executor = self.get_multi_executor - else: - get_executor = self.get_single_executor - executor = get_executor(env, tlist, slist, executor_kw) + try: + executor = tlist[0].get_executor(create = 0) + except (AttributeError, IndexError): + pass + else: + executor.add_sources(slist) + + if executor is None: + if not self.action: + fmt = "Builder %s must have an action to build %s." + raise UserError, fmt % (self.get_name(env or self.env), + map(str,tlist)) + key = self.action.batch_key(env or self.env, tlist, slist) + if key: + try: + executor = SCons.Executor.GetBatchExecutor(key) + except KeyError: + pass + else: + executor.add_batch(tlist, slist) + + if executor is None: + executor = SCons.Executor.Executor(self.action, env, [], + tlist, slist, executor_kw) + if key: + SCons.Executor.AddBatchExecutor(key, executor) # Now set up the relevant information in the target Nodes themselves. for t in tlist: diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py index eeb3b3f..91dd82a 100644 --- a/src/engine/SCons/BuilderTests.py +++ b/src/engine/SCons/BuilderTests.py @@ -406,25 +406,6 @@ class BuilderTestCase(unittest.TestCase): """Test the get_name() method """ - def test_get_single_executor(self): - """Test the get_single_executor() method - """ - b = SCons.Builder.Builder(action='foo') - x = b.get_single_executor({}, [], [], {}) - assert not x is None, x - - def test_get_multi_executor(self): - """Test the get_multi_executor() method - """ - b = SCons.Builder.Builder(action='foo', multi=1) - t1 = MyNode('t1') - s1 = MyNode('s1') - s2 = MyNode('s2') - x1 = b.get_multi_executor({}, [t1], [s1], {}) - t1.executor = x1 - x2 = b.get_multi_executor({}, [t1], [s2], {}) - assert x1 is x2, "%s is not %s" % (repr(x1), repr(x2)) - def test_cmp(self): """Test simple comparisons of Builder objects """ @@ -1472,8 +1453,13 @@ class BuilderTestCase(unittest.TestCase): assert b5.get_name(None) == 'builder5', b5.get_name(None) assert b6.get_name(None) in b6_names, b6.get_name(None) - tgt = b4(env, target = 'moo', source='cow') - assert tgt[0].builder.get_name(env) == 'bldr4' + # This test worked before adding batch builders, but we must now + # be able to disambiguate a CompositeAction into a more specific + # action based on file suffix at call time. Leave this commented + # out (for now) in case this reflects a real-world use case that + # we must accomodate and we want to resurrect this test. + #tgt = b4(env, target = 'moo', source='cow') + #assert tgt[0].builder.get_name(env) == 'bldr4' class CompositeBuilderTestCase(unittest.TestCase): @@ -1517,12 +1503,11 @@ class CompositeBuilderTestCase(unittest.TestCase): builder = self.builder flag = 0 - tgt = builder(env, target='test3', source=['test2.bar', 'test1.foo'])[0] try: - tgt.build() + builder(env, target='test3', source=['test2.bar', 'test1.foo'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['test3']' from `test1.foo': Cannot build multiple sources with different extensions: .bar, .foo" assert str(e) == expect, e @@ -1558,12 +1543,11 @@ class CompositeBuilderTestCase(unittest.TestCase): env['FOO_SUFFIX'] = '.BAR2' builder.add_action('$NEW_SUFFIX', func_action) flag = 0 - tgt = builder(env, target='test5', source=['test5.BAR2'])[0] try: - tgt.build() + builder(env, target='test5', source=['test5.BAR2'])[0] except SCons.Errors.UserError: flag = 1 - assert flag, "UserError should be thrown when we build targets with ambigous suffixes." + assert flag, "UserError should be thrown when we call a builder with ambigous suffixes." def test_src_builder(self): """Test CompositeBuilder's use of a src_builder""" @@ -1603,52 +1587,47 @@ class CompositeBuilderTestCase(unittest.TestCase): assert isinstance(tgt.builder, SCons.Builder.BuilderBase) flag = 0 - tgt = builder(env, target='t5', source=['test5a.foo', 'test5b.inb'])[0] try: - tgt.build() + builder(env, target='t5', source=['test5a.foo', 'test5b.inb'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t5']' from `test5b.bar': Cannot build multiple sources with different extensions: .foo, .bar" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t6', source=['test6a.bar', 'test6b.ina'])[0] try: - tgt.build() + builder(env, target='t6', source=['test6a.bar', 'test6b.ina'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t6']' from `test6b.foo': Cannot build multiple sources with different extensions: .bar, .foo" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t4', source=['test4a.ina', 'test4b.inb'])[0] try: - tgt.build() + builder(env, target='t4', source=['test4a.ina', 'test4b.inb'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t4']' from `test4b.bar': Cannot build multiple sources with different extensions: .foo, .bar" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t7', source=[env.fs.File('test7')])[0] try: - tgt.build() + builder(env, target='t7', source=[env.fs.File('test7')])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t7']': Cannot deduce file extension from source files: ['test7']" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t8', source=['test8.unknown'])[0] try: - tgt.build() + builder(env, target='t8', source=['test8.unknown'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build a target with an unknown suffix." + assert flag, "UserError should be thrown when we call a builder target with an unknown suffix." expect = "While building `['t8']' from `['test8.unknown']': Don't know how to build from a source file with suffix `.unknown'. Expected a suffix in this list: ['.foo', '.bar']." assert str(e) == expect, e diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index 338ed37..a92a23d 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -109,19 +109,18 @@ def apply_tools(env, tools, toolpath): # set or override them. This warning can optionally be turned off, # but scons will still ignore the illegal variable names even if it's off. reserved_construction_var_names = [ + 'CHANGED_SOURCES', + 'CHANGED_TARGETS', 'SOURCE', 'SOURCES', 'TARGET', 'TARGETS', -] - -future_reserved_construction_var_names = [ - 'CHANGED_SOURCES', - 'CHANGED_TARGETS', 'UNCHANGED_SOURCES', 'UNCHANGED_TARGETS', ] +future_reserved_construction_var_names = [] + def copy_non_reserved_keywords(dict): result = semi_deepcopy(dict) for k in result.keys(): @@ -490,7 +489,7 @@ class SubstitutionEnvironment: def lvars(self): return {} - def subst(self, string, raw=0, target=None, source=None, conv=None): + def subst(self, string, raw=0, target=None, source=None, conv=None, executor=None): """Recursively interpolates construction variables from the Environment into the specified string, returning the expanded result. Construction variables are specified by a $ prefix @@ -503,6 +502,8 @@ class SubstitutionEnvironment: gvars = self.gvars() lvars = self.lvars() lvars['__env__'] = self + if executor: + lvars.update(executor.get_lvars()) return SCons.Subst.scons_subst(string, self, raw, target, source, gvars, lvars, conv) def subst_kw(self, kw, raw=0, target=None, source=None): @@ -514,12 +515,14 @@ class SubstitutionEnvironment: nkw[k] = v return nkw - def subst_list(self, string, raw=0, target=None, source=None, conv=None): + def subst_list(self, string, raw=0, target=None, source=None, conv=None, executor=None): """Calls through to SCons.Subst.scons_subst_list(). See the documentation for that function.""" gvars = self.gvars() lvars = self.lvars() lvars['__env__'] = self + if executor: + lvars.update(executor.get_lvars()) return SCons.Subst.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv) def subst_path(self, path, target=None, source=None): diff --git a/src/engine/SCons/Environment.xml b/src/engine/SCons/Environment.xml index a861105..d63d20f 100644 --- a/src/engine/SCons/Environment.xml +++ b/src/engine/SCons/Environment.xml @@ -115,6 +115,22 @@ below, for more information. </summary> </cvar> +<cvar name="CHANGED_SOURCES"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + +<cvar name="CHANGED_TARGETS"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + <cvar name="SOURCE"> <summary> A reserved variable name @@ -147,6 +163,22 @@ that may not be set or used in a construction environment. </summary> </cvar> +<cvar name="UNCHANGED_SOURCES"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + +<cvar name="UNCHANGED_TARGETS"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + <cvar name="TOOLS"> <summary> A list of the names of the Tool specifications diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 0ea9dda..77d4602 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -172,6 +172,7 @@ class TestEnvironmentFixture: suffix = '.o', single_source = 1) kw['BUILDERS'] = {'Object' : static_obj} + static_obj.add_action('.cpp', 'fake action') env = apply(Environment, args, kw) return env @@ -887,6 +888,17 @@ sys.exit(0) class BaseTestCase(unittest.TestCase,TestEnvironmentFixture): + reserved_variables = [ + 'CHANGED_SOURCES', + 'CHANGED_TARGETS', + 'SOURCE', + 'SOURCES', + 'TARGET', + 'TARGETS', + 'UNCHANGED_SOURCES', + 'UNCHANGED_TARGETS', + ] + def test___init__(self): """Test construction Environment creation @@ -1123,10 +1135,14 @@ env4.builder1.env, env3) """Test warning generation when reserved variable names are set""" reserved_variables = [ + 'CHANGED_SOURCES', + 'CHANGED_TARGETS', 'SOURCE', 'SOURCES', 'TARGET', 'TARGETS', + 'UNCHANGED_SOURCES', + 'UNCHANGED_TARGETS', ] warning = SCons.Warnings.ReservedVariableWarning @@ -1135,7 +1151,7 @@ env4.builder1.env, env3) try: env4 = Environment() - for kw in reserved_variables: + for kw in self.reserved_variables: exc_caught = None try: env4[kw] = 'xyzzy' @@ -1149,12 +1165,7 @@ env4.builder1.env, env3) def test_FutureReservedVariables(self): """Test warning generation when future reserved variable names are set""" - future_reserved_variables = [ - 'CHANGED_SOURCES', - 'CHANGED_TARGETS', - 'UNCHANGED_SOURCES', - 'UNCHANGED_TARGETS', - ] + future_reserved_variables = [] warning = SCons.Warnings.FutureReservedVariableWarning SCons.Warnings.enableWarningClass(warning) @@ -3365,19 +3376,22 @@ def generate(env): f = env.xxx('$FOO') assert f == 'foo', f - def test_bad_keywords(type): + def test_bad_keywords(self): """Test trying to use reserved keywords in an Environment""" - reserved = ['TARGETS','SOURCES', 'SOURCE','TARGET'] added = [] - env = type.TestEnvironment(TARGETS = 'targets', + env = self.TestEnvironment(TARGETS = 'targets', SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', INIT = 'init') bad_msg = '%s is not reserved, but got omitted; see Environment.construction_var_name_ok' added.append('INIT') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3386,9 +3400,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', APPEND = 'append') added.append('APPEND') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3397,9 +3415,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', APPENDUNIQUE = 'appendunique') added.append('APPENDUNIQUE') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3408,9 +3430,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', PREPEND = 'prepend') added.append('PREPEND') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3419,9 +3445,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', PREPENDUNIQUE = 'prependunique') added.append('PREPENDUNIQUE') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3430,9 +3460,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', REPLACE = 'replace') added.append('REPLACE') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3441,8 +3475,12 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', COPY = 'copy') - for x in reserved: + for x in self.reserved_variables: assert not copy.has_key(x), env[x] for x in added + ['COPY']: assert copy.has_key(x), bad_msg % x @@ -3451,8 +3489,12 @@ def generate(env): 'SOURCES' : 'sources', 'SOURCE' : 'source', 'TARGET' : 'target', + 'CHANGED_SOURCES' : 'changed_sources', + 'CHANGED_TARGETS' : 'changed_targets', + 'UNCHANGED_SOURCES' : 'unchanged_sources', + 'UNCHANGED_TARGETS' : 'unchanged_targets', 'OVERRIDE' : 'override'}) - for x in reserved: + for x in self.reserved_variables: assert not over.has_key(x), over[x] for x in added + ['OVERRIDE']: assert over.has_key(x), bad_msg % x diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 25db771..7e831e3 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -31,12 +31,85 @@ Nodes. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import string +import UserList from SCons.Debug import logInstanceCreation import SCons.Errors import SCons.Memoize +class Batch: + """Remembers exact association between targets + and sources of executor.""" + def __init__(self, targets=[], sources=[]): + self.targets = targets + self.sources = sources + + + +class TSList(UserList.UserList): + """A class that implements $TARGETS or $SOURCES expansions by wrapping + an executor Method. This class is used in the Executor.lvars() + to delay creation of NodeList objects until they're needed. + + Note that we subclass UserList.UserList purely so that the + is_Sequence() function will identify an object of this class as + a list during variable expansion. We're not really using any + UserList.UserList methods in practice. + """ + def __init__(self, func): + self.func = func + def __getattr__(self, attr): + nl = self.func() + return getattr(nl, attr) + def __getitem__(self, i): + nl = self.func() + return nl[i] + def __getslice__(self, i, j): + nl = self.func() + i = max(i, 0); j = max(j, 0) + return nl[i:j] + def __str__(self): + nl = self.func() + return str(nl) + def __repr__(self): + nl = self.func() + return repr(nl) + +class TSObject: + """A class that implements $TARGET or $SOURCE expansions by wrapping + an Executor method. + """ + def __init__(self, func): + self.func = func + def __getattr__(self, attr): + n = self.func() + return getattr(n, attr) + def __str__(self): + n = self.func() + if n: + return str(n) + return '' + def __repr__(self): + n = self.func() + if n: + return repr(n) + return '' + +def rfile(node): + """ + A function to return the results of a Node's rfile() method, + if it exists, and the Node itself otherwise (if it's a Value + Node, e.g.). + """ + try: + rfile = node.rfile + except AttributeError: + return node + else: + return rfile() + + class Executor: """A class for controlling instances of executing an action. @@ -58,11 +131,96 @@ class Executor: self.post_actions = [] self.env = env self.overridelist = overridelist - self.targets = targets - self.sources = SCons.Util.UniqueList(sources[:]) + if targets or sources: + self.batches = [Batch(targets[:], sources[:])] + else: + self.batches = [] self.builder_kw = builder_kw self._memo = {} + def get_lvars(self): + try: + return self.lvars + except AttributeError: + self.lvars = { + 'CHANGED_SOURCES' : TSList(self._get_changed_sources), + 'CHANGED_TARGETS' : TSList(self._get_changed_targets), + 'SOURCE' : TSObject(self._get_source), + 'SOURCES' : TSList(self._get_sources), + 'TARGET' : TSObject(self._get_target), + 'TARGETS' : TSList(self._get_targets), + 'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), + 'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), + } + return self.lvars + + def _get_changes(self): + cs = [] + ct = [] + us = [] + ut = [] + for b in self.batches: + if b.targets[0].changed(): + cs.extend(map(rfile, b.sources)) + ct.extend(b.targets) + else: + us.extend(map(rfile, b.sources)) + ut.extend(b.targets) + self._changed_sources_list = SCons.Util.NodeList(cs) + self._changed_targets_list = SCons.Util.NodeList(ct) + self._unchanged_sources_list = SCons.Util.NodeList(us) + self._unchanged_targets_list = SCons.Util.NodeList(ut) + + def _get_changed_sources(self, *args, **kw): + try: + return self._changed_sources_list + except AttributeError: + self._get_changes() + return self._changed_sources_list + + def _get_changed_targets(self, *args, **kw): + try: + return self._changed_targets_list + except AttributeError: + self._get_changes() + return self._changed_targets_list + + def _get_source(self, *args, **kw): + #return SCons.Util.NodeList([rfile(self.batches[0].sources[0]).get_subst_proxy()]) + return rfile(self.batches[0].sources[0]).get_subst_proxy() + + def _get_sources(self, *args, **kw): + return SCons.Util.NodeList(map(lambda n: rfile(n).get_subst_proxy(), self.get_all_sources())) + + def _get_target(self, *args, **kw): + #return SCons.Util.NodeList([self.batches[0].targets[0].get_subst_proxy()]) + return self.batches[0].targets[0].get_subst_proxy() + + def _get_targets(self, *args, **kw): + return SCons.Util.NodeList(map(lambda n: n.get_subst_proxy(), self.get_all_targets())) + + def _get_unchanged_sources(self, *args, **kw): + try: + return self._unchanged_sources_list + except AttributeError: + self._get_changes() + return self._unchanged_sources_list + + def _get_unchanged_targets(self, *args, **kw): + try: + return self._unchanged_targets_list + except AttributeError: + self._get_changes() + return self._unchanged_targets_list + + def get_action_targets(self): + if not self.action_list: + return [] + targets_string = self.action_list[0].get_targets(self.env, self) + if targets_string[0] == '$': + targets_string = targets_string[1:] + return self.get_lvars()[targets_string] + def set_action_list(self, action): import SCons.Util if not SCons.Util.is_List(action): @@ -75,6 +233,58 @@ class Executor: def get_action_list(self): return self.pre_actions + self.action_list + self.post_actions + def get_all_targets(self): + """Returns all targets for all batches of this Executor.""" + result = [] + for batch in self.batches: + # TODO(1.5): remove the list() cast + result.extend(list(batch.targets)) + return result + + def get_all_sources(self): + """Returns all sources for all batches of this Executor.""" + result = [] + for batch in self.batches: + # TODO(1.5): remove the list() cast + result.extend(list(batch.sources)) + return result + + def get_all_children(self): + """Returns all unique children (dependencies) for all batches + of this Executor. + + The Taskmaster can recognize when it's already evaluated a + Node, so we don't have to make this list unique for its intended + canonical use case, but we expect there to be a lot of redundancy + (long lists of batched .cc files #including the same .h files + over and over), so removing the duplicates once up front should + save the Taskmaster a lot of work. + """ + result = SCons.Util.UniqueList([]) + for target in self.get_all_targets(): + result.extend(target.children()) + return result + + def get_all_prerequisites(self): + """Returns all unique (order-only) prerequisites for all batches + of this Executor. + """ + result = SCons.Util.UniqueList([]) + for target in self.get_all_targets(): + # TODO(1.5): remove the list() cast + result.extend(list(target.prerequisites)) + return result + + def get_action_side_effects(self): + + """Returns all side effects for all batches of this + Executor used by the underlying Action. + """ + result = SCons.Util.UniqueList([]) + for target in self.get_action_targets(): + result.extend(target.side_effects) + return result + memoizer_counters.append(SCons.Memoize.CountValue('get_build_env')) def get_build_env(self): @@ -108,14 +318,17 @@ class Executor: """ env = self.get_build_env() try: - cwd = self.targets[0].cwd + cwd = self.batches[0].targets[0].cwd except (IndexError, AttributeError): cwd = None - return scanner.path(env, cwd, self.targets, self.get_sources()) + return scanner.path(env, cwd, + self.get_all_targets(), + self.get_all_sources()) def get_kw(self, kw={}): result = self.builder_kw.copy() result.update(kw) + result['executor'] = self return result def do_nothing(self, target, kw): @@ -127,7 +340,9 @@ class Executor: kw = self.get_kw(kw) status = 0 for act in self.get_action_list(): - status = apply(act, (self.targets, self.get_sources(), env), kw) + #args = (self.get_all_targets(), self.get_all_sources(), env) + args = ([], [], env) + status = apply(act, args, kw) if isinstance(status, SCons.Errors.BuildError): status.executor = self raise status @@ -135,7 +350,7 @@ class Executor: msg = "Error %s" % status raise SCons.Errors.BuildError( errstr=msg, - node=self.targets, + node=self.batches[0].targets, executor=self, action=act) return status @@ -154,20 +369,32 @@ class Executor: """Add source files to this Executor's list. This is necessary for "multi" Builders that can be called repeatedly to build up a source file list for a given target.""" - self.sources.extend(sources) + # TODO(batch): extend to multiple batches + assert (len(self.batches) == 1) + # TODO(batch): remove duplicates? + #slist = filter(lambda x, s=self.batches[0].sources: x not in s, sources) + self.batches[0].sources.extend(sources) def get_sources(self): - return self.sources + return self.batches[0].sources + + def add_batch(self, targets, sources): + """Add pair of associated target and source to this Executor's list. + This is necessary for "batch" Builders that can be called repeatedly + to build up a list of matching target and source files that will be + used in order to update multiple target files at once from multiple + corresponding source files, for tools like MSVC that support it.""" + self.batches.append(Batch(targets, sources)) def prepare(self): """ Preparatory checks for whether this Executor can go ahead and (try to) build its targets. """ - for s in self.get_sources(): + for s in self.get_all_sources(): if s.missing(): msg = "Source `%s' not found, needed by target `%s'." - raise SCons.Errors.StopError, msg % (s, self.targets[0]) + raise SCons.Errors.StopError, msg % (s, self.batches[0].targets[0]) def add_pre_action(self, action): self.pre_actions.append(action) @@ -179,7 +406,7 @@ class Executor: def my_str(self): env = self.get_build_env() - get = lambda action, t=self.targets, s=self.get_sources(), e=env: \ + get = lambda action, t=self.get_all_targets(), s=self.get_all_sources(), e=env: \ action.genstring(t, s, e) return string.join(map(get, self.get_action_list()), "\n") @@ -204,7 +431,7 @@ class Executor: except KeyError: pass env = self.get_build_env() - get = lambda action, t=self.targets, s=self.get_sources(), e=env: \ + get = lambda action, t=self.get_all_targets(), s=self.get_all_sources(), e=env: \ action.get_contents(t, s, e) result = string.join(map(get, self.get_action_list()), "") self._memo['get_contents'] = result @@ -218,11 +445,13 @@ class Executor: return 0 def scan_targets(self, scanner): - self.scan(scanner, self.targets) + # TODO(batch): scan by batches + self.scan(scanner, self.get_all_targets()) def scan_sources(self, scanner): - if self.sources: - self.scan(scanner, self.get_sources()) + # TODO(batch): scan by batches + if self.batches[0].sources: + self.scan(scanner, self.get_all_sources()) def scan(self, scanner, node_list): """Scan a list of this Executor's files (targets or sources) for @@ -232,6 +461,7 @@ class Executor: """ env = self.get_build_env() + # TODO(batch): scan by batches) deps = [] if scanner: for node in node_list: @@ -256,16 +486,16 @@ class Executor: deps.extend(self.get_implicit_deps()) - for tgt in self.targets: + for tgt in self.get_all_targets(): tgt.add_to_implicit(deps) - def _get_unignored_sources_key(self, ignore=()): - return tuple(ignore) + def _get_unignored_sources_key(self, node, ignore=()): + return (node,) + tuple(ignore) memoizer_counters.append(SCons.Memoize.CountDict('get_unignored_sources', _get_unignored_sources_key)) - def get_unignored_sources(self, ignore=()): - ignore = tuple(ignore) + def get_unignored_sources(self, node, ignore=()): + key = (node,) + tuple(ignore) try: memo_dict = self._memo['get_unignored_sources'] except KeyError: @@ -273,56 +503,56 @@ class Executor: self._memo['get_unignored_sources'] = memo_dict else: try: - return memo_dict[ignore] + return memo_dict[key] except KeyError: pass - sourcelist = self.get_sources() + if node: + # TODO: better way to do this (it's a linear search, + # but it may not be critical path)? + sourcelist = [] + for b in self.batches: + if node in b.targets: + sourcelist = b.sources + break + else: + sourcelist = self.get_all_sources() if ignore: idict = {} for i in ignore: idict[i] = 1 sourcelist = filter(lambda s, i=idict: not i.has_key(s), sourcelist) - memo_dict[ignore] = sourcelist + memo_dict[key] = sourcelist return sourcelist - def _process_sources_key(self, func, ignore=()): - return (func, tuple(ignore)) - - memoizer_counters.append(SCons.Memoize.CountDict('process_sources', _process_sources_key)) - - def process_sources(self, func, ignore=()): - memo_key = (func, tuple(ignore)) - try: - memo_dict = self._memo['process_sources'] - except KeyError: - memo_dict = {} - self._memo['process_sources'] = memo_dict - else: - try: - return memo_dict[memo_key] - except KeyError: - pass - - result = map(func, self.get_unignored_sources(ignore)) - - memo_dict[memo_key] = result - - return result - def get_implicit_deps(self): """Return the executor's implicit dependencies, i.e. the nodes of the commands to be executed.""" result = [] build_env = self.get_build_env() for act in self.get_action_list(): - result.extend(act.get_implicit_deps(self.targets, self.get_sources(), build_env)) + deps = act.get_implicit_deps(self.get_all_targets(), + self.get_all_sources(), + build_env) + result.extend(deps) return result + + +_batch_executors = {} + +def GetBatchExecutor(key): + return _batch_executors[key] + +def AddBatchExecutor(key, executor): + assert not _batch_executors.has_key(key) + _batch_executors[key] = executor + nullenv = None + def get_NullEnvironment(): """Use singleton pattern for Null Environments.""" global nullenv @@ -349,7 +579,7 @@ class Null: """ def __init__(self, *args, **kw): if __debug__: logInstanceCreation(self, 'Executor.Null') - self.targets = kw['targets'] + self.batches = [Batch(kw['targets'][:], [])] def get_build_env(self): return get_NullEnvironment() def get_build_scanner_path(self): @@ -360,17 +590,30 @@ class Null: pass def get_unignored_sources(self, *args, **kw): return tuple(()) + def get_action_targets(self): + return [] def get_action_list(self): return [] + def get_all_targets(self): + return self.batches[0].targets + def get_all_sources(self): + return self.batches[0].targets[0].sources + def get_all_children(self): + return self.get_all_sources() + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] def __call__(self, *args, **kw): return 0 def get_contents(self): return '' - def _morph(self): """Morph this Null executor to a real Executor object.""" + batches = self.batches self.__class__ = Executor - self.__init__([], targets=self.targets) + self.__init__([]) + self.batches = batches # The following methods require morphing this Null Executor to a # real Executor object. diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py index 0fd11af..79e9d03 100644 --- a/src/engine/SCons/ExecutorTests.py +++ b/src/engine/SCons/ExecutorTests.py @@ -110,9 +110,11 @@ class ExecutorTestCase(unittest.TestCase): assert x.action_list == ['a'], x.action_list assert x.env == 'e', x.env assert x.overridelist == ['o'], x.overridelist - assert x.targets == 't', x.targets + targets = x.get_all_targets() + assert targets == ['t'], targets source_list.append('s3') - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2'], sources try: x = SCons.Executor.Executor(None, 'e', ['o'], 't', source_list) except SCons.Errors.UserError: @@ -200,11 +202,11 @@ class ExecutorTestCase(unittest.TestCase): ['s1', 's2'], builder_kw={'X':1, 'Y':2}) kw = x.get_kw() - assert kw == {'X':1, 'Y':2}, kw + assert kw == {'X':1, 'Y':2, 'executor':x}, kw kw = x.get_kw({'Z':3}) - assert kw == {'X':1, 'Y':2, 'Z':3}, kw + assert kw == {'X':1, 'Y':2, 'Z':3, 'executor':x}, kw kw = x.get_kw({'X':4}) - assert kw == {'X':4, 'Y':2}, kw + assert kw == {'X':4, 'Y':2, 'executor':x}, kw def test__call__(self): """Test calling an Executor""" @@ -222,7 +224,7 @@ class ExecutorTestCase(unittest.TestCase): a = MyAction([action1, action2]) t = MyNode('t') - x = SCons.Executor.Executor(a, env, [], t, ['s1', 's2']) + x = SCons.Executor.Executor(a, env, [], [t], ['s1', 's2']) x.add_pre_action(pre) x.add_post_action(post) x(t) @@ -233,7 +235,7 @@ class ExecutorTestCase(unittest.TestCase): result.append('pre_err') return 1 - x = SCons.Executor.Executor(a, env, [], t, ['s1', 's2']) + x = SCons.Executor.Executor(a, env, [], [t], ['s1', 's2']) x.add_pre_action(pre_err) x.add_post_action(post) try: @@ -268,22 +270,30 @@ class ExecutorTestCase(unittest.TestCase): def test_add_sources(self): """Test adding sources to an Executor""" x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2']) - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2'], sources + x.add_sources(['s1', 's2']) - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2', 's1', 's2'], sources + x.add_sources(['s3', 's1', 's4']) - assert x.sources == ['s1', 's2', 's3', 's4'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2', 's1', 's2', 's3', 's1', 's4'], sources def test_get_sources(self): """Test getting sources from an Executor""" x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2']) - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_sources() + assert sources == ['s1', 's2'], sources + x.add_sources(['s1', 's2']) - x.get_sources() - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_sources() + assert sources == ['s1', 's2', 's1', 's2'], sources + x.add_sources(['s3', 's1', 's4']) - x.get_sources() - assert x.sources == ['s1', 's2', 's3', 's4'], x.sources + sources = x.get_sources() + assert sources == ['s1', 's2', 's1', 's2', 's3', 's1', 's4'], sources def test_prepare(self): """Test the Executor's prepare() method""" @@ -429,34 +439,15 @@ class ExecutorTestCase(unittest.TestCase): s3 = MyNode('s3') x = SCons.Executor.Executor('b', env, [{}], [], [s1, s2, s3]) - r = x.get_unignored_sources([]) + r = x.get_unignored_sources(None, []) assert r == [s1, s2, s3], map(str, r) - r = x.get_unignored_sources([s2]) + r = x.get_unignored_sources(None, [s2]) assert r == [s1, s3], map(str, r) - r = x.get_unignored_sources([s1, s3]) + r = x.get_unignored_sources(None, [s1, s3]) assert r == [s2], map(str, r) - def test_process_sources(self): - """Test processing the source list through a function""" - env = MyEnvironment() - s1 = MyNode('s1') - s2 = MyNode('s2') - s3 = MyNode('s3') - x = SCons.Executor.Executor('b', env, [{}], [], [s1, s2, s3]) - - r = x.process_sources(str) - assert r == ['s1', 's2', 's3'], r - - r = x.process_sources(str, [s2]) - assert r == ['s1', 's3'], r - - def xxx(x): - return 'xxx-' + str(x) - r = x.process_sources(xxx, [s1, s3]) - assert r == ['xxx-s2'], r - if __name__ == "__main__": diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f8911b0..9da9d8e 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -966,6 +966,9 @@ class Entry(Base): def _glob1(self, pattern, ondisk=True, source=False, strings=False): return self.disambiguate()._glob1(pattern, ondisk, source, strings) + def get_subst_proxy(self): + return self.disambiguate().get_subst_proxy() + # This is for later so we can differentiate between Entry the class and Entry # the method of the FS class. _classEntry = Entry diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 37dc465..e2a1e9e 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -304,7 +304,10 @@ class VariantDirTestCase(unittest.TestCase): class MkdirAction(Action): def __init__(self, dir_made): self.dir_made = dir_made - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() self.dir_made.extend(target) save_Link = SCons.Node.FS.Link @@ -3060,7 +3063,10 @@ class prepareTestCase(unittest.TestCase): class MkdirAction(Action): def __init__(self, dir_made): self.dir_made = dir_made - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() self.dir_made.extend(target) dir_made = [] @@ -3338,7 +3344,8 @@ class SpecialAttrTestCase(unittest.TestCase): assert s == os.path.normpath('baz/sub/file.suffix'), s assert f.srcpath.is_literal(), f.srcpath g = f.srcpath.get() - assert isinstance(g, SCons.Node.FS.Entry), g.__class__ + # Gets disambiguated to SCons.Node.FS.File by get_subst_proxy(). + assert isinstance(g, SCons.Node.FS.File), g.__class__ s = str(f.srcdir) assert s == os.path.normpath('baz/sub'), s @@ -3372,7 +3379,8 @@ class SpecialAttrTestCase(unittest.TestCase): try: fs.Entry('eee').get_subst_proxy().no_such_attr except AttributeError, e: - assert str(e) == "Entry instance 'eee' has no attribute 'no_such_attr'", e + # Gets disambiguated to File instance by get_subst_proxy(). + assert str(e) == "File instance 'eee' has no attribute 'no_such_attr'", e caught = 1 assert caught, "did not catch expected AttributeError" diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 8bceaf6..73e1e05 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -67,8 +67,11 @@ class MyAction(MyActionBase): def __init__(self): self.order = 0 - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): global built_it, built_target, built_source, built_args, built_order + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() built_it = 1 built_target = target built_source = source diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 871efff..c44525f 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -621,7 +621,7 @@ class Node: # essentially short-circuits an N*M scan of the # sources for each individual target, which is a hell # of a lot more efficient. - for tgt in executor.targets: + for tgt in executor.get_all_targets(): tgt.add_to_implicit(implicit) if implicit_deps_unchanged or self.is_up_to_date(): @@ -714,7 +714,7 @@ class Node: if s not in ignore_set: sources.append(s) else: - sources = executor.get_unignored_sources(self.ignore) + sources = executor.get_unignored_sources(self, self.ignore) seen = set() bsources = [] bsourcesigs = [] diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py index 9974485..4fc657e 100644 --- a/src/engine/SCons/SConfTests.py +++ b/src/engine/SCons/SConfTests.py @@ -219,10 +219,11 @@ class SConfTestCase(unittest.TestCase): pass def get_executor(self): class Executor: - pass - e = Executor() - e.targets = [self] - return e + def __init__(self, targets): + self.targets = targets + def get_all_targets(self): + return self.targets + return Executor([self]) return [MyNode('n1'), MyNode('n2')] try: self.scons_env.Append(BUILDERS = {'SConfActionBuilder' : MyBuilder()}) diff --git a/src/engine/SCons/Subst.py b/src/engine/SCons/Subst.py index fc9d7d9..6459eec 100644 --- a/src/engine/SCons/Subst.py +++ b/src/engine/SCons/Subst.py @@ -256,6 +256,9 @@ class Target_or_Source: class NullNodeList(SCons.Util.NullSeq): def __call__(self, *args, **kwargs): return '' def __str__(self): return '' + # TODO(1.5): unneeded after new-style classes introduce iterators + def __getitem__(self, i): + raise IndexError NullNodesList = NullNodeList() @@ -285,6 +288,13 @@ def subst_dict(target, source): tnl = NLWrapper(target, get_tgt_subst_proxy) dict['TARGETS'] = Targets_or_Sources(tnl) dict['TARGET'] = Target_or_Source(tnl) + + # This is a total cheat, but hopefully this dictionary goes + # away soon anyway. We just let these expand to $TARGETS + # because that's "good enough" for the use of ToolSurrogates + # (see test/ToolSurrogate.py) to generate documentation. + dict['CHANGED_TARGETS'] = '$TARGETS' + dict['UNCHANGED_TARGETS'] = '$TARGETS' else: dict['TARGETS'] = NullNodesList dict['TARGET'] = NullNodesList @@ -304,6 +314,13 @@ def subst_dict(target, source): snl = NLWrapper(source, get_src_subst_proxy) dict['SOURCES'] = Targets_or_Sources(snl) dict['SOURCE'] = Target_or_Source(snl) + + # This is a total cheat, but hopefully this dictionary goes + # away soon anyway. We just let these expand to $TARGETS + # because that's "good enough" for the use of ToolSurrogates + # (see test/ToolSurrogate.py) to generate documentation. + dict['CHANGED_SOURCES'] = '$SOURCES' + dict['UNCHANGED_SOURCES'] = '$SOURCES' else: dict['SOURCES'] = NullNodesList dict['SOURCE'] = NullNodesList @@ -392,11 +409,9 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ source with two methods (substitute() and expand()) that handle the expansion. """ - def __init__(self, env, mode, target, source, conv, gvars): + def __init__(self, env, mode, conv, gvars): self.env = env self.mode = mode - self.target = target - self.source = source self.conv = conv self.gvars = gvars @@ -433,14 +448,14 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ except Exception, e: if e.__class__ in AllowableExceptions: return '' - raise_exception(e, self.target, s) + raise_exception(e, lvars['TARGETS'], s) else: if lvars.has_key(key): s = lvars[key] elif self.gvars.has_key(key): s = self.gvars[key] elif not NameError in AllowableExceptions: - raise_exception(NameError(key), self.target, s) + raise_exception(NameError(key), lvars['TARGETS'], s) else: return '' @@ -466,8 +481,8 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ return map(func, s) elif callable(s): try: - s = s(target=self.target, - source=self.source, + s = s(target=lvars['TARGETS'], + source=lvars['SOURCES'], env=self.env, for_signature=(self.mode != SUBST_CMD)) except TypeError: @@ -525,10 +540,11 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ # If we dropped that behavior (or found another way to cover it), # we could get rid of this call completely and just rely on the # Executor setting the variables. - d = subst_dict(target, source) - if d: - lvars = lvars.copy() - lvars.update(d) + if not lvars.has_key('TARGET'): + d = subst_dict(target, source) + if d: + lvars = lvars.copy() + lvars.update(d) # We're (most likely) going to eval() things. If Python doesn't # find a __builtins__ value in the global dictionary used for eval(), @@ -538,7 +554,7 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ # for expansion. gvars['__builtins__'] = __builtins__ - ss = StringSubber(env, mode, target, source, conv, gvars) + ss = StringSubber(env, mode, conv, gvars) result = ss.substitute(strSubst, lvars) try: @@ -595,12 +611,10 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv and the rest of the object takes care of doing the right thing internally. """ - def __init__(self, env, mode, target, source, conv, gvars): + def __init__(self, env, mode, conv, gvars): UserList.UserList.__init__(self, []) self.env = env self.mode = mode - self.target = target - self.source = source self.conv = conv self.gvars = gvars @@ -649,14 +663,14 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv except Exception, e: if e.__class__ in AllowableExceptions: return - raise_exception(e, self.target, s) + raise_exception(e, lvars['TARGETS'], s) else: if lvars.has_key(key): s = lvars[key] elif self.gvars.has_key(key): s = self.gvars[key] elif not NameError in AllowableExceptions: - raise_exception(NameError(), self.target, s) + raise_exception(NameError(), lvars['TARGETS'], s) else: return @@ -676,8 +690,8 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv self.next_word() elif callable(s): try: - s = s(target=self.target, - source=self.source, + s = s(target=lvars['TARGETS'], + source=lvars['SOURCES'], env=self.env, for_signature=(self.mode != SUBST_CMD)) except TypeError: @@ -820,10 +834,11 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv # If we dropped that behavior (or found another way to cover it), # we could get rid of this call completely and just rely on the # Executor setting the variables. - d = subst_dict(target, source) - if d: - lvars = lvars.copy() - lvars.update(d) + if not lvars.has_key('TARGET'): + d = subst_dict(target, source) + if d: + lvars = lvars.copy() + lvars.update(d) # We're (most likely) going to eval() things. If Python doesn't # find a __builtins__ value in the global dictionary used for eval(), @@ -833,7 +848,7 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv # for expansion. gvars['__builtins__'] = __builtins__ - ls = ListSubber(env, mode, target, source, conv, gvars) + ls = ListSubber(env, mode, conv, gvars) ls.substitute(strSubst, lvars, 0) try: diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 934e28b..4245480 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -186,8 +186,9 @@ class Task: # target t.prepare() methods check that each target's explicit # or implicit dependencies exists, and also initialize the # .sconsign info. - self.targets[0].get_executor().prepare() - for t in self.targets: + executor = self.targets[0].get_executor() + executor.prepare() + for t in executor.get_action_targets(): t.prepare() for s in t.side_effects: s.prepare() @@ -763,8 +764,10 @@ class Taskmaster: if T: T.write(self.trace_message(' already handled (executed)')) continue + executor = node.get_executor() + try: - children = node.children() + children = executor.get_all_children() except SystemExit: exc_value = sys.exc_info()[1] e = SCons.Errors.ExplicitExit(node, exc_value.code) @@ -786,7 +789,7 @@ class Taskmaster: children_not_ready = [] children_failed = False - for child in chain(children,node.prerequisites): + for child in chain(children, executor.get_all_prerequisites()): childstate = child.get_state() if T: T.write(self.trace_message(' ' + self.trace_node(child))) @@ -830,7 +833,8 @@ class Taskmaster: # added the other children to the list of candidate nodes # to keep on building (--keep-going). if children_failed: - node.set_state(NODE_FAILED) + for n in executor.get_action_targets(): + n.set_state(NODE_FAILED) if S: S.child_failed = S.child_failed + 1 if T: T.write(self.trace_message('****** %s\n' % self.trace_node(node))) @@ -861,7 +865,7 @@ class Taskmaster: # Skip this node if it has side-effects that are # currently being built: wait_side_effects = False - for se in node.side_effects: + for se in executor.get_action_side_effects(): if se.get_state() == NODE_EXECUTING: se.add_to_waiting_s_e(node) wait_side_effects = True @@ -900,7 +904,7 @@ class Taskmaster: if node is None: return None - tlist = node.get_executor().targets + tlist = node.get_executor().get_all_targets() task = self.tasker(self, tlist, node in self.original_top, node) try: diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index b36e4aa..c8bbdf4 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -166,8 +166,21 @@ class Node: class Executor: def prepare(self): pass + def get_action_targets(self): + return self.targets + def get_all_targets(self): + return self.targets + def get_all_children(self): + result = [] + for node in self.targets: + result.extend(node.children()) + return result + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] self.executor = Executor() - self.executor.targets = self.targets + self.executor.targets = self.targets return self.executor class OtherError(Exception): @@ -752,7 +765,7 @@ class TaskmasterTestCase(unittest.TestCase): # set it up by having something that approximates a real Builder # return this list--but that's more work than is probably # warranted right now. - t.targets = [n1, n2] + n1.get_executor().targets = [n1, n2] t.prepare() assert n1.prepared assert n2.prepared @@ -763,7 +776,7 @@ class TaskmasterTestCase(unittest.TestCase): t = tm.next_task() # More bogus reaching in and setting the targets. n3.set_state(SCons.Node.up_to_date) - t.targets = [n3, n4] + n3.get_executor().targets = [n3, n4] t.prepare() assert n3.prepared assert n4.prepared @@ -803,7 +816,7 @@ class TaskmasterTestCase(unittest.TestCase): tm = SCons.Taskmaster.Taskmaster([n6, n7]) t = tm.next_task() # More bogus reaching in and setting the targets. - t.targets = [n6, n7] + n6.get_executor().targets = [n6, n7] t.prepare() assert n6.prepared assert n7.prepared @@ -815,9 +828,21 @@ class TaskmasterTestCase(unittest.TestCase): class ExceptionExecutor: def prepare(self): raise Exception, "Executor.prepare() exception" + def get_all_targets(self): + return self.nodes + def get_all_children(self): + result = [] + for node in self.nodes: + result.extend(node.children()) + return result + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] n11 = Node("n11") n11.executor = ExceptionExecutor() + n11.executor.nodes = [n11] tm = SCons.Taskmaster.Taskmaster([n11]) t = tm.next_task() try: diff --git a/src/engine/SCons/Tool/javah.py b/src/engine/SCons/Tool/javah.py index 7eb4969..9b3c8c2 100644 --- a/src/engine/SCons/Tool/javah.py +++ b/src/engine/SCons/Tool/javah.py @@ -103,7 +103,7 @@ def emit_java_headers(target, source, env): def JavaHOutFlagGenerator(target, source, env, for_signature): try: t = target[0] - except (AttributeError, TypeError): + except (AttributeError, IndexError, TypeError): t = target try: return '-d ' + str(t.attributes.java_lookupdir) diff --git a/src/engine/SCons/Tool/mslink.py b/src/engine/SCons/Tool/mslink.py index 4268b58..53cab28 100644 --- a/src/engine/SCons/Tool/mslink.py +++ b/src/engine/SCons/Tool/mslink.py @@ -65,7 +65,10 @@ def _dllSources(target, source, env, for_signature, paramtp): deffile = env.FindIxes(source, "WINDOWSDEFPREFIX", "WINDOWSDEFSUFFIX") for src in source: - if src == deffile: + # Check explicitly for a non-None deffile so that the __cmp__ + # method of the base SCons.Util.Proxy class used for some Node + # proxies doesn't try to use a non-existent __dict__ attribute. + if deffile and src == deffile: # Treat this source as a .def file. listCmd.append("/def:%s" % src.get_string(for_signature)) else: diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index b324a32..0898b91 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -672,40 +672,103 @@ res_builder = SCons.Builder.Builder(action=res_action, src_builder=[], source_scanner=res_scanner) +def msvc_batch_key(action, env, target, source): + """ + Returns a key to identify unique batches of sources for compilation. + + If batching is enabled (via the $MSVC_BATCH setting), then all + target+source pairs that use the same action, defined by the same + environment, and have the same target and source directories, will + be batched. + + Returning None specifies that the specified target+source should not + be batched with other compilations. + """ + b = env.subst('$MSVC_BATCH') + if b in (None, '', '0'): + # We're not using batching; return no key. + return None + t = target[0] + s = source[0] + if os.path.splitext(t.name)[0] != os.path.splitext(s.name)[0]: + # The base names are different, so this *must* be compiled + # separately; return no key. + return None + return (id(action), id(env), t.dir, s.dir) + +def msvc_output_flag(target, source, env, for_signature): + """ + Returns the correct /Fo flag for batching. + + If batching is disabled or there's only one source file, then we + return an /Fo string that specifies the target explicitly. Otherwise, + we return an /Fo string that just specifies the first target's + directory (where the Visual C/C++ compiler will put the .obj files). + """ + b = env.subst('$MSVC_BATCH') + if b in (None, '', '0') or len(source) == 1: + return '/Fo$TARGET' + else: + # The Visual C/C++ compiler requires a \ at the end of the /Fo + # option to indicate an output directory. We use os.sep here so + # that the test(s) for this can be run on non-Windows systems + # without having a hard-coded backslash mess up command-line + # argument parsing. + return '/Fo${TARGET.dir}' + os.sep + +CAction = SCons.Action.Action("$CCCOM", "$CCCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') +ShCAction = SCons.Action.Action("$SHCCCOM", "$SHCCCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') +CXXAction = SCons.Action.Action("$CXXCOM", "$CXXCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') +ShCXXAction = SCons.Action.Action("$SHCXXCOM", "$SHCXXCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') def generate(env): """Add Builders and construction variables for MSVC++ to an Environment.""" static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + # TODO(batch): shouldn't reach in to cmdgen this way; necessary + # for now to bypass the checks in Builder.DictCmdGenerator.__call__() + # and allow .cc and .cpp to be compiled in the same command line. + static_obj.cmdgen.source_ext_match = False + shared_obj.cmdgen.source_ext_match = False + for suffix in CSuffixes: - static_obj.add_action(suffix, SCons.Defaults.CAction) - shared_obj.add_action(suffix, SCons.Defaults.ShCAction) + static_obj.add_action(suffix, CAction) + shared_obj.add_action(suffix, ShCAction) static_obj.add_emitter(suffix, static_object_emitter) shared_obj.add_emitter(suffix, shared_object_emitter) for suffix in CXXSuffixes: - static_obj.add_action(suffix, SCons.Defaults.CXXAction) - shared_obj.add_action(suffix, SCons.Defaults.ShCXXAction) + static_obj.add_action(suffix, CXXAction) + shared_obj.add_action(suffix, ShCXXAction) static_obj.add_emitter(suffix, static_object_emitter) shared_obj.add_emitter(suffix, shared_object_emitter) env['CCPDBFLAGS'] = SCons.Util.CLVar(['${(PDB and "/Z7") or ""}']) env['CCPCHFLAGS'] = SCons.Util.CLVar(['${(PCH and "/Yu%s /Fp%s"%(PCHSTOP or "",File(PCH))) or ""}']) + env['_MSVC_OUTPUT_FLAG'] = msvc_output_flag env['_CCCOMCOM'] = '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS' env['CC'] = 'cl' env['CCFLAGS'] = SCons.Util.CLVar('/nologo') env['CFLAGS'] = SCons.Util.CLVar('') - env['CCCOM'] = '$CC /Fo$TARGET /c $SOURCES $CFLAGS $CCFLAGS $_CCCOMCOM' + env['CCCOM'] = '$CC $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $CFLAGS $CCFLAGS $_CCCOMCOM' env['SHCC'] = '$CC' env['SHCCFLAGS'] = SCons.Util.CLVar('$CCFLAGS') env['SHCFLAGS'] = SCons.Util.CLVar('$CFLAGS') - env['SHCCCOM'] = '$SHCC /Fo$TARGET /c $SOURCES $SHCFLAGS $SHCCFLAGS $_CCCOMCOM' + env['SHCCCOM'] = '$SHCC $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $SHCFLAGS $SHCCFLAGS $_CCCOMCOM' env['CXX'] = '$CC' env['CXXFLAGS'] = SCons.Util.CLVar('$CCFLAGS $( /TP $)') - env['CXXCOM'] = '$CXX /Fo$TARGET /c $SOURCES $CXXFLAGS $CCFLAGS $_CCCOMCOM' + env['CXXCOM'] = '$CXX $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $CXXFLAGS $CCFLAGS $_CCCOMCOM' env['SHCXX'] = '$CXX' env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS') - env['SHCXXCOM'] = '$SHCXX /Fo$TARGET /c $SOURCES $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM' + env['SHCXXCOM'] = '$SHCXX $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM' env['CPPDEFPREFIX'] = '/D' env['CPPDEFSUFFIX'] = '' env['INCPREFIX'] = '/I' diff --git a/src/engine/SCons/Tool/msvc.xml b/src/engine/SCons/Tool/msvc.xml index a2fdd7e..31dcdf1 100644 --- a/src/engine/SCons/Tool/msvc.xml +++ b/src/engine/SCons/Tool/msvc.xml @@ -140,6 +140,26 @@ env['CCPDBFLAGS'] = '/Zi /Fd${TARGET}.pdb' </summary> </cvar> +<cvar name="MSVC_BATCH"> +<summary> +When set to any true value, +specifies that &SCons; should batch +compilation of object files +when calling the Microsoft Visual C/C++ compiler. +All compilations of source files from the same source directory +that generate target files in a same output directory +and were configured in &SCons; using the same construction environment +will be built in a single call to the compiler. +Only source files that have changed since their +object files were built will be passed to each compiler invocation +(via the &cv-link-CHANGED_SOURCES; construction variable). +Any compilations where the object (target) file base name +(minus the <filename>.obj</filename>) +does not match the source file base name +will be compiled separately. +</summary> +</cvar> + <cvar name="PCH"> <summary> The Microsoft Visual C++ precompiled header that will be used when compiling diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index cbec5dd..a9f7b70 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -1092,11 +1092,12 @@ class Selector(OrderedDict): """A callable ordered dictionary that maps file suffixes to dictionary values. We preserve the order in which items are added so that get_suffix() calls always return the first suffix added.""" - def __call__(self, env, source): - try: - ext = source[0].suffix - except IndexError: - ext = "" + def __call__(self, env, source, ext=None): + if ext is None: + try: + ext = source[0].suffix + except IndexError: + ext = "" try: return self[ext] except KeyError: @@ -1561,20 +1562,32 @@ class Null: #cls._inst = type.__new__(cls, *args, **kwargs) cls._inst = apply(type.__new__, (cls,) + args, kwargs) return cls._inst - def __init__(self, *args, **kwargs): pass - def __call__(self, *args, **kwargs): return self - def __repr__(self): return "Null(0x%08X)" % id(self) - def __nonzero__(self): return False - def __getattr__(self, mname): return self - def __setattr__(self, name, value): return self - def __delattr__(self, name): return self + def __init__(self, *args, **kwargs): + pass + def __call__(self, *args, **kwargs): + return self + def __repr__(self): + return "Null(0x%08X)" % id(self) + def __nonzero__(self): + return False + def __getattr__(self, name): + return self + def __setattr__(self, name, value): + return self + def __delattr__(self, name): + return self class NullSeq(Null): - def __len__(self): return 0 - def __iter__(self): return iter(()) - def __getitem__(self, i): return self - def __delitem__(self, i): return self - def __setitem__(self, i, v): return self + def __len__(self): + return 0 + def __iter__(self): + return iter(()) + def __getitem__(self, i): + return self + def __delitem__(self, i): + return self + def __setitem__(self, i, v): + return self del __revision__ diff --git a/test/Batch/Boolean.py b/test/Batch/Boolean.py new file mode 100644 index 0000000..e5f8d24 --- /dev/null +++ b/test/Batch/Boolean.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify basic use of batch_key to write a batch builder that handles +arbitrary pairs of target + source files. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """ +def batch_build(target, source, env): + for t, s in zip(target, source): + open(str(t), 'wb').write(open(str(s), 'rb').read()) +env = Environment() +bb = Action(batch_build, batch_key=True) +env['BUILDERS']['Batch'] = Builder(action=bb) +env1 = env.Clone() +env1.Batch('f1a.out', 'f1a.in') +env1.Batch('f1b.out', 'f1b.in') +env2 = env.Clone() +env2.Batch('f2a.out', 'f2a.in') +env3 = env.Clone() +env3.Batch('f3a.out', 'f3a.in') +env3.Batch('f3b.out', 'f3b.in') +""") + +test.write('f1a.in', "f1a.in\n") +test.write('f1b.in', "f1b.in\n") +test.write('f2a.in', "f2a.in\n") +test.write('f3a.in', "f3a.in\n") +test.write('f3b.in', "f3b.in\n") + +expect = test.wrap_stdout("""\ +batch_build(["f1a.out", "f1b.out"], ["f1a.in", "f1b.in"]) +batch_build(["f2a.out"], ["f2a.in"]) +batch_build(["f3a.out", "f3b.out"], ["f3a.in", "f3b.in"]) +""") + +test.run(stdout = expect) + +test.must_match('f1a.out', "f1a.in\n") +test.must_match('f1b.out', "f1b.in\n") +test.must_match('f2a.out', "f2a.in\n") +test.must_match('f3a.out', "f3a.in\n") +test.must_match('f3b.out', "f3b.in\n") + +test.pass_test() diff --git a/test/Batch/CHANGED_SOURCES.py b/test/Batch/CHANGED_SOURCES.py new file mode 100644 index 0000000..9059afd --- /dev/null +++ b/test/Batch/CHANGED_SOURCES.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify use of $CHANGED_SOURCES with batch builders correctly decides +to rebuild if any sources of changed, and specifies only the sources +on the rebuild. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +_python_ = TestSCons._python_ + +test.write('batch_build.py', """\ +import os +import sys +dir = sys.argv[1] +for infile in sys.argv[2:]: + inbase = os.path.splitext(os.path.split(infile)[1])[0] + outfile = os.path.join(dir, inbase+'.out') + open(outfile, 'wb').write(open(infile, 'rb').read()) +sys.exit(0) +""") + +test.write('SConstruct', """ +env = Environment() +env['BATCH_BUILD'] = 'batch_build.py' +env['BATCHCOM'] = r'%(_python_)s $BATCH_BUILD ${TARGET.dir} $CHANGED_SOURCES' +bb = Action('$BATCHCOM', batch_key=True, targets='CHANGED_TARGETS') +env['BUILDERS']['Batch'] = Builder(action=bb) +env1 = env.Clone() +env1.Batch('out1/f1a.out', 'f1a.in') +env1.Batch('out1/f1b.out', 'f1b.in') +env2 = env.Clone() +env2.Batch('out2/f2a.out', 'f2a.in') +env3 = env.Clone() +env3.Batch('out3/f3a.out', 'f3a.in') +env3.Batch('out3/f3b.out', 'f3b.in') +""" % locals()) + +test.write('f1a.in', "f1a.in\n") +test.write('f1b.in', "f1b.in\n") +test.write('f2a.in', "f2a.in\n") +test.write('f3a.in', "f3a.in\n") +test.write('f3b.in', "f3b.in\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out1 f1a.in f1b.in +%(_python_)s batch_build.py out2 f2a.in +%(_python_)s batch_build.py out3 f3a.in f3b.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + + + +test.write('f1b.in', "f1b.in 2\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out1 f1b.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in 2\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + + + +test.write('f3a.in', "f3a.in 2\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out3 f3a.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in 2\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in 2\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + +test.pass_test() diff --git a/test/Batch/SOURCES.py b/test/Batch/SOURCES.py new file mode 100644 index 0000000..86b3b92 --- /dev/null +++ b/test/Batch/SOURCES.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify use of $SOURCES with batch builders correctly decides to +rebuild if any sources of changed, and specifies all the sources +on the rebuild. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +_python_ = TestSCons._python_ + +test.write('batch_build.py', """\ +import os +import sys +dir = sys.argv[1] +for infile in sys.argv[2:]: + inbase = os.path.splitext(os.path.split(infile)[1])[0] + outfile = os.path.join(dir, inbase+'.out') + open(outfile, 'wb').write(open(infile, 'rb').read()) +sys.exit(0) +""") + +test.write('SConstruct', """ +env = Environment() +env['BATCH_BUILD'] = 'batch_build.py' +env['BATCHCOM'] = r'%(_python_)s $BATCH_BUILD ${TARGET.dir} $SOURCES' +bb = Action('$BATCHCOM', batch_key=True) +env['BUILDERS']['Batch'] = Builder(action=bb) +env1 = env.Clone() +env1.Batch('out1/f1a.out', 'f1a.in') +env1.Batch('out1/f1b.out', 'f1b.in') +env2 = env.Clone() +env2.Batch('out2/f2a.out', 'f2a.in') +env3 = env.Clone() +env3.Batch('out3/f3a.out', 'f3a.in') +env3.Batch('out3/f3b.out', 'f3b.in') +""" % locals()) + +test.write('f1a.in', "f1a.in\n") +test.write('f1b.in', "f1b.in\n") +test.write('f2a.in', "f2a.in\n") +test.write('f3a.in', "f3a.in\n") +test.write('f3b.in', "f3b.in\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out1 f1a.in f1b.in +%(_python_)s batch_build.py out2 f2a.in +%(_python_)s batch_build.py out3 f3a.in f3b.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + +test.up_to_date(options = '--debug=explain', arguments = '.') + + + + +test.write('f1b.in', "f1b.in 2\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out1 f1a.in f1b.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in 2\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + + +test.write('f3a.in', "f3a.in 2\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out3 f3a.in f3b.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in 2\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in 2\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + +test.pass_test() diff --git a/test/Batch/action-changed.py b/test/Batch/action-changed.py new file mode 100644 index 0000000..cc0cd41 --- /dev/null +++ b/test/Batch/action-changed.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify that targets in a batch builder are rebuilt when the +build action changes. +""" + +import os + +import TestSCons + +python = TestSCons.python + +test = TestSCons.TestSCons() + +build_py_contents = """\ +#!/usr/bin/env %s +import sys +sep = sys.argv.index('--') +targets = sys.argv[1:sep] +sources = sys.argv[sep+1:] +for t, s in zip(targets, sources): + fp = open(t, 'wb') + fp.write('%s\\n') + fp.write(open(s, 'rb').read()) + fp.close() +sys.exit(0) +""" + +test.write('build.py', build_py_contents % (python, 'one')) +os.chmod(test.workpath('build.py'), 0755) + +test.write('SConstruct', """ +env = Environment() +bb = Action('%s $CHANGED_TARGETS -- $CHANGED_SOURCES', + batch_key=True, + targets='CHANGED_TARGETS') +env['BUILDERS']['Batch'] = Builder(action=bb) +env.Batch('f1.out', 'f1.in') +env.Batch('f2.out', 'f2.in') +env.Batch('f3.out', 'f3.in') +""" % test.workpath('build.py')) + +test.write('f1.in', "f1.in\n") +test.write('f2.in', "f2.in\n") +test.write('f3.in', "f3.in\n") + +test.run(arguments = '.') + +test.must_match('f1.out', "one\nf1.in\n") +test.must_match('f2.out', "one\nf2.in\n") +test.must_match('f3.out', "one\nf3.in\n") + +test.up_to_date(arguments = '.') + +test.write('build.py', build_py_contents % (python, 'two')) +os.chmod(test.workpath('build.py'), 0755) + +#test.not_up_to_date(options = 'CALLER=1 --taskmastertrace=/dev/tty', arguments = '.') +test.not_up_to_date(arguments = '.') + +test.must_match('f1.out', "two\nf1.in\n") +test.must_match('f2.out', "two\nf2.in\n") +test.must_match('f3.out', "two\nf3.in\n") + +test.pass_test() diff --git a/test/Batch/callable.py b/test/Batch/callable.py new file mode 100644 index 0000000..fc96f15 --- /dev/null +++ b/test/Batch/callable.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify passing in a batch_key callable for more control over how +batch builders behave. +""" + +import os + +import TestSCons + +test = TestSCons.TestSCons() + +test.subdir('sub1', 'sub2') + +test.write('SConstruct', """ +def batch_build(target, source, env): + for t, s in zip(target, source): + open(str(t), 'wb').write(open(str(s), 'rb').read()) +if ARGUMENTS.get('BATCH_CALLABLE'): + def batch_key(action, env, target, source): + return (id(action), id(env), target[0].dir) +else: + batch_key=True +env = Environment() +bb = Action(batch_build, batch_key=batch_key) +env['BUILDERS']['Batch'] = Builder(action=bb) +env1 = env.Clone() +env1.Batch('sub1/f1a.out', 'f1a.in') +env1.Batch('sub2/f1b.out', 'f1b.in') +env2 = env.Clone() +env2.Batch('sub1/f2a.out', 'f2a.in') +env2.Batch('sub2/f2b.out', 'f2b.in') +""") + +test.write('f1a.in', "f1a.in\n") +test.write('f1b.in', "f1b.in\n") +test.write('f2a.in', "f2a.in\n") +test.write('f2b.in', "f2b.in\n") + +sub1_f1a_out = os.path.join('sub1', 'f1a.out') +sub2_f1b_out = os.path.join('sub2', 'f1b.out') +sub1_f2a_out = os.path.join('sub1', 'f2a.out') +sub2_f2b_out = os.path.join('sub2', 'f2b.out') + +expect = test.wrap_stdout("""\ +batch_build(["%(sub1_f1a_out)s", "%(sub2_f1b_out)s"], ["f1a.in", "f1b.in"]) +batch_build(["%(sub1_f2a_out)s", "%(sub2_f2b_out)s"], ["f2a.in", "f2b.in"]) +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['sub1', 'f1a.out'], "f1a.in\n") +test.must_match(['sub2', 'f1b.out'], "f1b.in\n") +test.must_match(['sub1', 'f2a.out'], "f2a.in\n") +test.must_match(['sub2', 'f2b.out'], "f2b.in\n") + +test.run(arguments = '-c') + +test.must_not_exist(['sub1', 'f1a.out']) +test.must_not_exist(['sub2', 'f1b.out']) +test.must_not_exist(['sub1', 'f2a.out']) +test.must_not_exist(['sub2', 'f2b.out']) + +expect = test.wrap_stdout("""\ +batch_build(["%(sub1_f1a_out)s"], ["f1a.in"]) +batch_build(["%(sub1_f2a_out)s"], ["f2a.in"]) +batch_build(["%(sub2_f1b_out)s"], ["f1b.in"]) +batch_build(["%(sub2_f2b_out)s"], ["f2b.in"]) +""" % locals()) + +test.run(arguments = 'BATCH_CALLABLE=1', stdout = expect) + +test.must_match(['sub1', 'f1a.out'], "f1a.in\n") +test.must_match(['sub2', 'f1b.out'], "f1b.in\n") +test.must_match(['sub1', 'f2a.out'], "f2a.in\n") +test.must_match(['sub2', 'f2b.out'], "f2b.in\n") + +test.pass_test() diff --git a/test/Batch/generated.py b/test/Batch/generated.py new file mode 100644 index 0000000..4f31a7e --- /dev/null +++ b/test/Batch/generated.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify use of a batch builder when one of the later targets in the +list the list depends on a generated file. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """ +def batch_build(target, source, env): + for t, s in zip(target, source): + fp = open(str(t), 'wb') + if str(t) == 'f3.out': + fp.write(open('f3.include', 'rb').read()) + fp.write(open(str(s), 'rb').read()) +env = Environment() +bb = Action(batch_build, batch_key=True) +env['BUILDERS']['Batch'] = Builder(action=bb) +env1 = env.Clone() +env1.Batch('f1.out', 'f1.in') +env1.Batch('f2.out', 'f2.mid') +f3_out = env1.Batch('f3.out', 'f3.in') + +env2 = env.Clone() +env2.Batch('f2.mid', 'f2.in') + +f3_include = env.Batch('f3.include', 'f3.include.in') +env.Depends(f3_out, f3_include) +""") + +test.write('f1.in', "f1.in\n") +test.write('f2.in', "f2.in\n") +test.write('f3.in', "f3.in\n") +test.write('f3.include.in', "f3.include.in\n") + +expect = test.wrap_stdout("""\ +batch_build(["f2.mid"], ["f2.in"]) +batch_build(["f3.include"], ["f3.include.in"]) +batch_build(["f1.out", "f2.out", "f3.out"], ["f1.in", "f2.mid", "f3.in"]) +""") + +test.run(stdout = expect) + +test.must_match('f1.out', "f1.in\n") +test.must_match('f2.out', "f2.in\n") + +test.up_to_date(arguments = '.') + +test.pass_test() diff --git a/test/Batch/up_to_date.py b/test/Batch/up_to_date.py new file mode 100644 index 0000000..9563c40 --- /dev/null +++ b/test/Batch/up_to_date.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify simple use of $SOURCES with batch builders correctly decide +that files are up to date on a rebuild. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +_python_ = TestSCons._python_ + +test.write('batch_build.py', """\ +import os +import sys +dir = sys.argv[1] +for infile in sys.argv[2:]: + inbase = os.path.splitext(os.path.split(infile)[1])[0] + outfile = os.path.join(dir, inbase+'.out') + open(outfile, 'wb').write(open(infile, 'rb').read()) +sys.exit(0) +""") + +test.write('SConstruct', """ +env = Environment() +env['BATCH_BUILD'] = 'batch_build.py' +env['BATCHCOM'] = r'%(_python_)s $BATCH_BUILD ${TARGET.dir} $SOURCES' +bb = Action('$BATCHCOM', batch_key=True) +env['BUILDERS']['Batch'] = Builder(action=bb) +env1 = env.Clone() +env1.Batch('out1/f1a.out', 'f1a.in') +env1.Batch('out1/f1b.out', 'f1b.in') +env2 = env.Clone() +env2.Batch('out2/f2a.out', 'f2a.in') +env3 = env.Clone() +env3.Batch('out3/f3a.out', 'f3a.in') +env3.Batch('out3/f3b.out', 'f3b.in') +""" % locals()) + +test.write('f1a.in', "f1a.in\n") +test.write('f1b.in', "f1b.in\n") +test.write('f2a.in', "f2a.in\n") +test.write('f3a.in', "f3a.in\n") +test.write('f3b.in', "f3b.in\n") + +expect = test.wrap_stdout("""\ +%(_python_)s batch_build.py out1 f1a.in f1b.in +%(_python_)s batch_build.py out2 f2a.in +%(_python_)s batch_build.py out3 f3a.in f3b.in +""" % locals()) + +test.run(stdout = expect) + +test.must_match(['out1', 'f1a.out'], "f1a.in\n") +test.must_match(['out1', 'f1b.out'], "f1b.in\n") +test.must_match(['out2', 'f2a.out'], "f2a.in\n") +test.must_match(['out3', 'f3a.out'], "f3a.in\n") +test.must_match(['out3', 'f3b.out'], "f3b.in\n") + +test.up_to_date(options = '--debug=explain', arguments = '.') + +test.pass_test() diff --git a/test/MSVC/batch.py b/test/MSVC/batch.py new file mode 100644 index 0000000..3776df7 --- /dev/null +++ b/test/MSVC/batch.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify operation of Visual C/C++ batch builds. + +This uses a fake compiler and linker script, fake command lines, and +explicit suffix settings so that the test should work when run on any +platform. +""" + +import string + +import TestSCons + +test = TestSCons.TestSCons() + +_python_ = TestSCons._python_ + +test.write('fake_cl.py', """\ +import os +import string +import sys +input_files = sys.argv[2:] +if sys.argv[1][-1] in (os.sep, '\\\\'): + # The output (/Fo) argument ends with a backslash, indicating an + # output directory. We accept ending with a slash as well so this + # test runs on non-Windows systems. Strip either character and + # record the directory name. + sys.argv[1] = sys.argv[1][:-1] + dir = sys.argv[1][3:] +else: + dir = None + output = sys.argv[1][3:] +# Delay writing the .log output until here so any trailing slash or +# backslash has been stripped, and the output comparisons later in this +# script don't have to account for the difference. +open('fake_cl.log', 'ab').write(string.join(sys.argv[1:]) + '\\n') +for infile in input_files: + if dir: + outfile = os.path.join(dir, string.replace(infile, '.c', '.obj')) + else: + outfile = output + open(outfile, 'wb').write(open(infile, 'rb').read()) +""") + +test.write('fake_link.py', """\ +import string +import sys +ofp = open(sys.argv[1], 'wb') +for infile in sys.argv[2:]: + ofp.write(open(infile, 'rb').read()) +""") + +test.write('SConstruct', """ +cccom = '%(_python_)s fake_cl.py $_MSVC_OUTPUT_FLAG $CHANGED_SOURCES' +linkcom = '%(_python_)s fake_link.py ${TARGET.windows} $SOURCES' +env = Environment(tools=['msvc', 'mslink'], + CCCOM=cccom, + LINKCOM=linkcom, + PROGSUFFIX='.exe', + OBJSUFFIX='.obj', + MSVC_BATCH=ARGUMENTS.get('MSVC_BATCH')) +p = env.Object('prog.c') +f1 = env.Object('f1.c') +f2 = env.Object('f2.c') +env.Program(p + f1 + f2) +""" % locals()) + +test.write('prog.c', "prog.c\n") +test.write('f1.c', "f1.c\n") +test.write('f2.c', "f2.c\n") + + + +test.run(arguments = 'MSVC_BATCH=1 .') + +test.must_match('prog.exe', "prog.c\nf1.c\nf2.c\n") +test.must_match('fake_cl.log', """\ +/Fo. prog.c f1.c f2.c +""") + +test.up_to_date(options = 'MSVC_BATCH=1', arguments = '.') + + + +test.write('f1.c', "f1.c 2\n") + +test.run(arguments = 'MSVC_BATCH=1 .') + +test.must_match('prog.exe', "prog.c\nf1.c 2\nf2.c\n") +test.must_match('fake_cl.log', """\ +/Fo. prog.c f1.c f2.c +/Fo. f1.c +""") + +test.up_to_date(options = 'MSVC_BATCH=1', arguments = '.') + + + +test.run(arguments = '-c .') + +test.unlink('fake_cl.log') + + + +test.run(arguments = '. MSVC_BATCH=0') + +test.must_match('prog.exe', "prog.c\nf1.c 2\nf2.c\n") +test.must_match('fake_cl.log', """\ +/Fof1.obj f1.c +/Fof2.obj f2.c +/Foprog.obj prog.c +""") + + + +test.write('f1.c', "f1.c 3\n") + +test.run(arguments = '. MSVC_BATCH=0') + +test.must_match('prog.exe', "prog.c\nf1.c 3\nf2.c\n") +test.must_match('fake_cl.log', """\ +/Fof1.obj f1.c +/Fof2.obj f2.c +/Foprog.obj prog.c +/Fof1.obj f1.c +""") + + + +test.pass_test() diff --git a/test/Scanner/generated.py b/test/Scanner/generated.py index eb66fd2..507a3d3 100644 --- a/test/Scanner/generated.py +++ b/test/Scanner/generated.py @@ -411,17 +411,14 @@ int g_3() test.run(stderr=TestSCons.noisy_ar, match=TestSCons.match_re_dotall) -# Note that the generated .h files still get scanned twice, -# but that's really once each as a child of libg_1.o and libg_2.o. - test.must_match("MyCScan.out", """\ libg_1.c: 1 libg_2.c: 1 libg_3.c: 1 -libg_gx.h: 2 +libg_gx.h: 1 libg_gy.h: 1 libg_gz.h: 1 -libg_w.h: 2 +libg_w.h: 1 """) test.pass_test() |