diff options
Diffstat (limited to 'src/engine/SCons')
| -rw-r--r-- | src/engine/SCons/Action.py | 160 | ||||
| -rw-r--r-- | src/engine/SCons/ActionTests.py | 17 | ||||
| -rw-r--r-- | src/engine/SCons/Builder.py | 82 | ||||
| -rw-r--r-- | src/engine/SCons/BuilderTests.py | 63 | ||||
| -rw-r--r-- | src/engine/SCons/Environment.py | 17 | ||||
| -rw-r--r-- | src/engine/SCons/Environment.xml | 32 | ||||
| -rw-r--r-- | src/engine/SCons/EnvironmentTests.py | 78 | ||||
| -rw-r--r-- | src/engine/SCons/Executor.py | 345 | ||||
| -rw-r--r-- | src/engine/SCons/ExecutorTests.py | 65 | ||||
| -rw-r--r-- | src/engine/SCons/Node/FS.py | 3 | ||||
| -rw-r--r-- | src/engine/SCons/Node/FSTests.py | 16 | ||||
| -rw-r--r-- | src/engine/SCons/Node/NodeTests.py | 5 | ||||
| -rw-r--r-- | src/engine/SCons/Node/__init__.py | 4 | ||||
| -rw-r--r-- | src/engine/SCons/SConfTests.py | 9 | ||||
| -rw-r--r-- | src/engine/SCons/Subst.py | 63 | ||||
| -rw-r--r-- | src/engine/SCons/Taskmaster.py | 18 | ||||
| -rw-r--r-- | src/engine/SCons/TaskmasterTests.py | 33 | ||||
| -rw-r--r-- | src/engine/SCons/Tool/javah.py | 2 | ||||
| -rw-r--r-- | src/engine/SCons/Tool/mslink.py | 5 | ||||
| -rw-r--r-- | src/engine/SCons/Tool/msvc.py | 79 | ||||
| -rw-r--r-- | src/engine/SCons/Tool/msvc.xml | 20 | ||||
| -rw-r--r-- | src/engine/SCons/Util.py | 47 |
22 files changed, 851 insertions, 312 deletions
diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index bc1d724..e106e74 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -392,7 +392,7 @@ def _do_create_list_action(act, kw): aa = _do_create_action(a, kw) if aa is not None: acts.append(aa) if not acts: - return None + return ListAction([]) elif len(acts) == 1: return acts[0] else: @@ -414,6 +414,11 @@ class ActionBase: def __cmp__(self, other): return cmp(self.__dict__, other) + def no_batch_key(self, env, target, source): + return None + + batch_key = no_batch_key + def genstring(self, target, source, env): return str(self) @@ -446,15 +451,18 @@ class ActionBase: self.presub_env = None # don't need this any more return lines - def get_executor(self, env, overrides, tlist, slist, executor_kw): - """Return the Executor for this Action.""" - return SCons.Executor.Executor(self, env, overrides, - tlist, slist, executor_kw) + def get_targets(self, env, executor): + """ + Returns the type of targets ($TARGETS, $CHANGED_TARGETS) used + by this action. + """ + return self.targets class _ActionAction(ActionBase): """Base class for actions that create output objects.""" def __init__(self, cmdstr=_null, strfunction=_null, varlist=(), presub=_null, chdir=None, exitstatfunc=None, + batch_key=None, targets='$TARGETS', **kw): self.cmdstr = cmdstr if strfunction is not _null: @@ -469,6 +477,19 @@ class _ActionAction(ActionBase): exitstatfunc = default_exitstatfunc self.exitstatfunc = exitstatfunc + self.targets = targets + + if batch_key: + if not callable(batch_key): + # They have set batch_key, but not to their own + # callable. The default behavior here will batch + # *all* targets+sources using this action, separated + # for each construction environment. + def default_batch_key(self, env, target, source): + return (id(self), id(env)) + batch_key = default_batch_key + SCons.Util.AddMethod(self, batch_key, 'batch_key') + def print_cmd_line(self, s, target, source, env): sys.stdout.write(s + "\n") @@ -477,7 +498,8 @@ class _ActionAction(ActionBase): presub=_null, show=_null, execute=_null, - chdir=_null): + chdir=_null, + executor=None): if not is_List(target): target = [target] if not is_List(source): @@ -498,15 +520,27 @@ class _ActionAction(ActionBase): chdir = str(chdir.abspath) except AttributeError: if not is_String(chdir): - chdir = str(target[0].dir) + if executor: + chdir = str(executor.batches[0].targets[0].dir) + else: + chdir = str(target[0].dir) if presub: + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() t = string.join(map(str, target), ' and ') l = string.join(self.presub_lines(env), '\n ') out = "Building %s with action:\n %s\n" % (t, l) sys.stdout.write(out) cmd = None if show and self.strfunction: - cmd = self.strfunction(target, source, env) + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() + try: + cmd = self.strfunction(target, source, env, executor) + except TypeError: + cmd = self.strfunction(target, source, env) if cmd: if chdir: cmd = ('os.chdir(%s)\n' % repr(chdir)) + cmd @@ -524,7 +558,7 @@ class _ActionAction(ActionBase): if chdir: os.chdir(chdir) try: - stat = self.execute(target, source, env) + stat = self.execute(target, source, env, executor=executor) if isinstance(stat, SCons.Errors.BuildError): s = exitstatfunc(stat.status) if s: @@ -657,8 +691,11 @@ class CommandAction(_ActionAction): return string.join(map(str, self.cmd_list), ' ') return str(self.cmd_list) - def process(self, target, source, env): - result = env.subst_list(self.cmd_list, 0, target, source) + def process(self, target, source, env, executor=None): + if executor: + result = env.subst_list(self.cmd_list, 0, executor=executor) + else: + result = env.subst_list(self.cmd_list, 0, target, source) silent = None ignore = None while 1: @@ -675,20 +712,23 @@ class CommandAction(_ActionAction): pass return result, ignore, silent - def strfunction(self, target, source, env): + def strfunction(self, target, source, env, executor=None): if self.cmdstr is None: return None if self.cmdstr is not _null: from SCons.Subst import SUBST_RAW - c = env.subst(self.cmdstr, SUBST_RAW, target, source) + if executor: + c = env.subst(self.cmdstr, SUBST_RAW, executor=executor) + else: + c = env.subst(self.cmdstr, SUBST_RAW, target, source) if c: return c - cmd_list, ignore, silent = self.process(target, source, env) + cmd_list, ignore, silent = self.process(target, source, env, executor) if silent: return '' return _string_from_cmd_list(cmd_list[0]) - def execute(self, target, source, env): + def execute(self, target, source, env, executor=None): """Execute a command action. This will handle lists of commands as well as individual commands, @@ -733,7 +773,10 @@ class CommandAction(_ActionAction): # reasonable for just about everything else: ENV[key] = str(value) - cmd_list, ignore, silent = self.process(target, map(rfile, source), env) + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() + cmd_list, ignore, silent = self.process(target, map(rfile, source), env, executor) # Use len() to filter out any "command" that's zero-length. for cmd_line in filter(len, cmd_list): @@ -748,7 +791,7 @@ class CommandAction(_ActionAction): command=cmd_line) return 0 - def get_presig(self, target, source, env): + def get_presig(self, target, source, env, executor=None): """Return the signature contents of this action's command line. This strips $(-$) and everything in between the string, @@ -760,16 +803,22 @@ class CommandAction(_ActionAction): cmd = string.join(map(str, cmd)) else: cmd = str(cmd) - return env.subst_target_source(cmd, SUBST_SIG, target, source) + if executor: + return env.subst_target_source(cmd, SUBST_SIG, executor=executor) + else: + return env.subst_target_source(cmd, SUBST_SIG, target, source) - def get_implicit_deps(self, target, source, env): + def get_implicit_deps(self, target, source, env, executor=None): icd = env.get('IMPLICIT_COMMAND_DEPENDENCIES', True) if is_String(icd) and icd[:1] == '$': icd = env.subst(icd) if not icd or icd in ('0', 'None'): return [] from SCons.Subst import SUBST_SIG - cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, target, source) + if executor: + cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, executor=executor) + else: + cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, target, source) res = [] for cmd_line in cmd_list: if cmd_line: @@ -785,14 +834,21 @@ class CommandGeneratorAction(ActionBase): self.generator = generator self.gen_kw = kw self.varlist = kw.get('varlist', ()) + self.targets = kw.get('targets', '$TARGETS') - def _generate(self, target, source, env, for_signature): + def _generate(self, target, source, env, for_signature, executor=None): # ensure that target is a list, to make it easier to write # generator functions: if not is_List(target): target = [target] - ret = self.generator(target=target, source=source, env=env, for_signature=for_signature) + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() + ret = self.generator(target=target, + source=source, + env=env, + for_signature=for_signature) #TODO(1.5) gen_cmd = Action(ret, **self.gen_kw) gen_cmd = apply(Action, (ret,), self.gen_kw) if not gen_cmd: @@ -809,25 +865,33 @@ class CommandGeneratorAction(ActionBase): act = self._generate([], [], env, 1) return str(act) - def genstring(self, target, source, env): - return self._generate(target, source, env, 1).genstring(target, source, env) + def batch_key(self, env, target, source): + return self._generate(target, source, env, 1).batch_key(env, target, source) + + def genstring(self, target, source, env, executor=None): + return self._generate(target, source, env, 1, executor).genstring(target, source, env) def __call__(self, target, source, env, exitstatfunc=_null, presub=_null, - show=_null, execute=_null, chdir=_null): - act = self._generate(target, source, env, 0) + show=_null, execute=_null, chdir=_null, executor=None): + act = self._generate(target, source, env, 0, executor) + if act is None: + raise UserError("While building `%s': Cannot deduce file extension from source files: %s" % (repr(map(str, target)), repr(map(str, source)))) return act(target, source, env, exitstatfunc, presub, - show, execute, chdir) + show, execute, chdir, executor) - def get_presig(self, target, source, env): + def get_presig(self, target, source, env, executor=None): """Return the signature contents of this action's command line. This strips $(-$) and everything in between the string, since those parts don't affect signatures. """ - return self._generate(target, source, env, 1).get_presig(target, source, env) + return self._generate(target, source, env, 1, executor).get_presig(target, source, env) - def get_implicit_deps(self, target, source, env): - return self._generate(target, source, env, 1).get_implicit_deps(target, source, env) + def get_implicit_deps(self, target, source, env, executor=None): + return self._generate(target, source, env, 1, executor).get_implicit_deps(target, source, env) + + def get_targets(self, env, executor): + return self._generate(None, None, env, 1, executor).get_targets(env, executor) @@ -864,14 +928,17 @@ class LazyAction(CommandGeneratorAction, CommandAction): return CommandGeneratorAction def _generate_cache(self, env): - c = env.get(self.var, '') + if env: + c = env.get(self.var, '') + else: + c = '' #TODO(1.5) gen_cmd = Action(c, **self.gen_kw) gen_cmd = apply(Action, (c,), self.gen_kw) if not gen_cmd: raise SCons.Errors.UserError("$%s value %s cannot be used to create an Action." % (self.var, repr(c))) return gen_cmd - def _generate(self, target, source, env, for_signature): + def _generate(self, target, source, env, for_signature, executor=None): return self._generate_cache(env) def __call__(self, target, source, env, *args, **kw): @@ -915,12 +982,15 @@ class FunctionAction(_ActionAction): except AttributeError: return "unknown_python_function" - def strfunction(self, target, source, env): + def strfunction(self, target, source, env, executor=None): if self.cmdstr is None: return None if self.cmdstr is not _null: from SCons.Subst import SUBST_RAW - c = env.subst(self.cmdstr, SUBST_RAW, target, source) + if executor: + c = env.subst(self.cmdstr, SUBST_RAW, executor=executor) + else: + c = env.subst(self.cmdstr, SUBST_RAW, target, source) if c: return c def array(a): @@ -953,9 +1023,12 @@ class FunctionAction(_ActionAction): return str(self.execfunction) return "%s(target, source, env)" % name - def execute(self, target, source, env): + def execute(self, target, source, env, executor=None): exc_info = (None,None,None) try: + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() rsources = map(rfile, source) try: result = self.execfunction(target=target, source=rsources, env=env) @@ -971,7 +1044,10 @@ class FunctionAction(_ActionAction): result = SCons.Errors.convert_to_BuildError(result, exc_info) result.node=target result.action=self - result.command=self.strfunction(target, source, env) + try: + result.command=self.strfunction(target, source, env, executor) + except TypeError: + result.command=self.strfunction(target, source, env) # FIXME: This maintains backward compatibility with respect to # which type of exceptions were returned by raising an @@ -1013,6 +1089,7 @@ class ListAction(ActionBase): # our children will have had any varlist # applied; we don't need to do it again self.varlist = () + self.targets = '$TARGETS' def genstring(self, target, source, env): return string.join(map(lambda a, t=target, s=source, e=env: @@ -1038,10 +1115,13 @@ class ListAction(ActionBase): "") def __call__(self, target, source, env, exitstatfunc=_null, presub=_null, - show=_null, execute=_null, chdir=_null): + show=_null, execute=_null, chdir=_null, executor=None): + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() for act in self.list: stat = act(target, source, env, exitstatfunc, presub, - show, execute, chdir) + show, execute, chdir, executor) if stat: return stat return 0 @@ -1111,7 +1191,7 @@ class ActionCaller: kw[key] = self.subst(self.kw[key], target, source, env) return kw - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): args = self.subst_args(target, source, env) kw = self.subst_kw(target, source, env) #TODO(1.5) return self.parent.actfunc(*args, **kw) diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py index 643e9fa..ae6a15c 100644 --- a/src/engine/SCons/ActionTests.py +++ b/src/engine/SCons/ActionTests.py @@ -405,13 +405,6 @@ class ActionTestCase(unittest.TestCase): a2 = SCons.Action.Action(a1) assert a2 is a1, a2 -class ActionBaseTestCase(unittest.TestCase): - def test_get_executor(self): - """Test the ActionBase.get_executor() method""" - a = SCons.Action.Action('foo') - x = a.get_executor({}, {}, [], [], {}) - assert x is not None, x - class _ActionActionTestCase(unittest.TestCase): def test__init__(self): @@ -1589,13 +1582,14 @@ class FunctionActionTestCase(unittest.TestCase): c = test.read(outfile, 'r') assert c == "class1b\n", c - def build_it(target, source, env, self=self): + def build_it(target, source, env, executor=None, self=self): self.build_it = 1 return 0 - def string_it(target, source, env, self=self): + def string_it(target, source, env, executor=None, self=self): self.string_it = 1 return None - act = SCons.Action.FunctionAction(build_it, { 'strfunction' : string_it }) + act = SCons.Action.FunctionAction(build_it, + { 'strfunction' : string_it }) r = act([], [], Environment()) assert r == 0, r assert self.build_it @@ -1996,8 +1990,7 @@ class ActionCompareTestCase(unittest.TestCase): if __name__ == "__main__": suite = unittest.TestSuite() - tclasses = [ ActionBaseTestCase, - _ActionActionTestCase, + tclasses = [ _ActionActionTestCase, ActionTestCase, CommandActionTestCase, CommandGeneratorActionTestCase, diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index 1d29516..21638a5 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -118,6 +118,15 @@ class _Null: _null = _Null +def match_splitext(path, suffixes = []): + if suffixes: + matchsuf = filter(lambda S,path=path: path[-len(S):] == S, + suffixes) + if matchsuf: + suf = max(map(None, map(len, matchsuf), matchsuf))[1] + return [path[:-len(suf)], path[-len(suf):]] + return SCons.Util.splitext(path) + class DictCmdGenerator(SCons.Util.Selector): """This is a callable class that can be used as a command generator function. It holds on to a dictionary @@ -142,20 +151,22 @@ class DictCmdGenerator(SCons.Util.Selector): return [] if self.source_ext_match: + suffixes = self.src_suffixes() ext = None for src in map(str, source): - my_ext = SCons.Util.splitext(src)[1] + my_ext = match_splitext(src, suffixes)[1] if ext and my_ext != ext: raise UserError("While building `%s' from `%s': Cannot build multiple sources with different extensions: %s, %s" % (repr(map(str, target)), src, ext, my_ext)) ext = my_ext else: - ext = SCons.Util.splitext(str(source[0]))[1] + ext = match_splitext(str(source[0]), self.src_suffixes())[1] if not ext: + #return ext raise UserError("While building `%s': Cannot deduce file extension from source files: %s" % (repr(map(str, target)), repr(map(str, source)))) try: - ret = SCons.Util.Selector.__call__(self, env, source) + ret = SCons.Util.Selector.__call__(self, env, source, ext) except KeyError, e: raise UserError("Ambiguous suffixes after environment substitution: %s == %s == %s" % (e[0], e[1], e[2])) if ret is None: @@ -295,8 +306,9 @@ def _node_errors(builder, env, tlist, slist): if t.builder != builder: msg = "Two different builders (%s and %s) were specified for the same target: %s" % (t.builder.get_name(env), builder.get_name(env), t) raise UserError, msg - if t.get_executor().targets != tlist: - msg = "Two different target lists have a target in common: %s (from %s and from %s)" % (t, map(str, t.get_executor().targets), map(str, tlist)) + # TODO(batch): list constructed each time! + if t.get_executor().get_all_targets() != tlist: + msg = "Two different target lists have a target in common: %s (from %s and from %s)" % (t, map(str, t.get_executor().get_all_targets()), map(str, tlist)) raise UserError, msg elif t.sources != slist: msg = "Multiple ways to build the same target were specified for: %s (from %s and from %s)" % (t, map(str, t.sources), map(str, slist)) @@ -441,30 +453,10 @@ class BuilderBase: if not env: env = self.env if env: - matchsuf = filter(lambda S,path=path: path[-len(S):] == S, - self.src_suffixes(env)) - if matchsuf: - suf = max(map(None, map(len, matchsuf), matchsuf))[1] - return [path[:-len(suf)], path[-len(suf):]] - return SCons.Util.splitext(path) - - def get_single_executor(self, env, tlist, slist, executor_kw): - if not self.action: - raise UserError, "Builder %s must have an action to build %s."%(self.get_name(env or self.env), map(str,tlist)) - return self.action.get_executor(env or self.env, - [], # env already has overrides - tlist, - slist, - executor_kw) - - def get_multi_executor(self, env, tlist, slist, executor_kw): - try: - executor = tlist[0].get_executor(create = 0) - except (AttributeError, IndexError): - return self.get_single_executor(env, tlist, slist, executor_kw) + suffixes = self.src_suffixes(env) else: - executor.add_sources(slist) - return executor + suffixes = [] + return match_splitext(path, suffixes) def _adjustixes(self, files, pre, suf, ensure_suffix=False): if not files: @@ -566,11 +558,37 @@ class BuilderBase: # The targets are fine, so find or make the appropriate Executor to # build this particular list of targets from this particular list of # sources. + + executor = None + key = None + if self.multi: - get_executor = self.get_multi_executor - else: - get_executor = self.get_single_executor - executor = get_executor(env, tlist, slist, executor_kw) + try: + executor = tlist[0].get_executor(create = 0) + except (AttributeError, IndexError): + pass + else: + executor.add_sources(slist) + + if executor is None: + if not self.action: + fmt = "Builder %s must have an action to build %s." + raise UserError, fmt % (self.get_name(env or self.env), + map(str,tlist)) + key = self.action.batch_key(env or self.env, tlist, slist) + if key: + try: + executor = SCons.Executor.GetBatchExecutor(key) + except KeyError: + pass + else: + executor.add_batch(tlist, slist) + + if executor is None: + executor = SCons.Executor.Executor(self.action, env, [], + tlist, slist, executor_kw) + if key: + SCons.Executor.AddBatchExecutor(key, executor) # Now set up the relevant information in the target Nodes themselves. for t in tlist: diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py index eeb3b3f..91dd82a 100644 --- a/src/engine/SCons/BuilderTests.py +++ b/src/engine/SCons/BuilderTests.py @@ -406,25 +406,6 @@ class BuilderTestCase(unittest.TestCase): """Test the get_name() method """ - def test_get_single_executor(self): - """Test the get_single_executor() method - """ - b = SCons.Builder.Builder(action='foo') - x = b.get_single_executor({}, [], [], {}) - assert not x is None, x - - def test_get_multi_executor(self): - """Test the get_multi_executor() method - """ - b = SCons.Builder.Builder(action='foo', multi=1) - t1 = MyNode('t1') - s1 = MyNode('s1') - s2 = MyNode('s2') - x1 = b.get_multi_executor({}, [t1], [s1], {}) - t1.executor = x1 - x2 = b.get_multi_executor({}, [t1], [s2], {}) - assert x1 is x2, "%s is not %s" % (repr(x1), repr(x2)) - def test_cmp(self): """Test simple comparisons of Builder objects """ @@ -1472,8 +1453,13 @@ class BuilderTestCase(unittest.TestCase): assert b5.get_name(None) == 'builder5', b5.get_name(None) assert b6.get_name(None) in b6_names, b6.get_name(None) - tgt = b4(env, target = 'moo', source='cow') - assert tgt[0].builder.get_name(env) == 'bldr4' + # This test worked before adding batch builders, but we must now + # be able to disambiguate a CompositeAction into a more specific + # action based on file suffix at call time. Leave this commented + # out (for now) in case this reflects a real-world use case that + # we must accomodate and we want to resurrect this test. + #tgt = b4(env, target = 'moo', source='cow') + #assert tgt[0].builder.get_name(env) == 'bldr4' class CompositeBuilderTestCase(unittest.TestCase): @@ -1517,12 +1503,11 @@ class CompositeBuilderTestCase(unittest.TestCase): builder = self.builder flag = 0 - tgt = builder(env, target='test3', source=['test2.bar', 'test1.foo'])[0] try: - tgt.build() + builder(env, target='test3', source=['test2.bar', 'test1.foo'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['test3']' from `test1.foo': Cannot build multiple sources with different extensions: .bar, .foo" assert str(e) == expect, e @@ -1558,12 +1543,11 @@ class CompositeBuilderTestCase(unittest.TestCase): env['FOO_SUFFIX'] = '.BAR2' builder.add_action('$NEW_SUFFIX', func_action) flag = 0 - tgt = builder(env, target='test5', source=['test5.BAR2'])[0] try: - tgt.build() + builder(env, target='test5', source=['test5.BAR2'])[0] except SCons.Errors.UserError: flag = 1 - assert flag, "UserError should be thrown when we build targets with ambigous suffixes." + assert flag, "UserError should be thrown when we call a builder with ambigous suffixes." def test_src_builder(self): """Test CompositeBuilder's use of a src_builder""" @@ -1603,52 +1587,47 @@ class CompositeBuilderTestCase(unittest.TestCase): assert isinstance(tgt.builder, SCons.Builder.BuilderBase) flag = 0 - tgt = builder(env, target='t5', source=['test5a.foo', 'test5b.inb'])[0] try: - tgt.build() + builder(env, target='t5', source=['test5a.foo', 'test5b.inb'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t5']' from `test5b.bar': Cannot build multiple sources with different extensions: .foo, .bar" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t6', source=['test6a.bar', 'test6b.ina'])[0] try: - tgt.build() + builder(env, target='t6', source=['test6a.bar', 'test6b.ina'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t6']' from `test6b.foo': Cannot build multiple sources with different extensions: .bar, .foo" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t4', source=['test4a.ina', 'test4b.inb'])[0] try: - tgt.build() + builder(env, target='t4', source=['test4a.ina', 'test4b.inb'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t4']' from `test4b.bar': Cannot build multiple sources with different extensions: .foo, .bar" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t7', source=[env.fs.File('test7')])[0] try: - tgt.build() + builder(env, target='t7', source=[env.fs.File('test7')])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build targets with files of different suffixes." + assert flag, "UserError should be thrown when we call a builder with files of different suffixes." expect = "While building `['t7']': Cannot deduce file extension from source files: ['test7']" assert str(e) == expect, e flag = 0 - tgt = builder(env, target='t8', source=['test8.unknown'])[0] try: - tgt.build() + builder(env, target='t8', source=['test8.unknown'])[0] except SCons.Errors.UserError, e: flag = 1 - assert flag, "UserError should be thrown when we build a target with an unknown suffix." + assert flag, "UserError should be thrown when we call a builder target with an unknown suffix." expect = "While building `['t8']' from `['test8.unknown']': Don't know how to build from a source file with suffix `.unknown'. Expected a suffix in this list: ['.foo', '.bar']." assert str(e) == expect, e diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py index 338ed37..a92a23d 100644 --- a/src/engine/SCons/Environment.py +++ b/src/engine/SCons/Environment.py @@ -109,19 +109,18 @@ def apply_tools(env, tools, toolpath): # set or override them. This warning can optionally be turned off, # but scons will still ignore the illegal variable names even if it's off. reserved_construction_var_names = [ + 'CHANGED_SOURCES', + 'CHANGED_TARGETS', 'SOURCE', 'SOURCES', 'TARGET', 'TARGETS', -] - -future_reserved_construction_var_names = [ - 'CHANGED_SOURCES', - 'CHANGED_TARGETS', 'UNCHANGED_SOURCES', 'UNCHANGED_TARGETS', ] +future_reserved_construction_var_names = [] + def copy_non_reserved_keywords(dict): result = semi_deepcopy(dict) for k in result.keys(): @@ -490,7 +489,7 @@ class SubstitutionEnvironment: def lvars(self): return {} - def subst(self, string, raw=0, target=None, source=None, conv=None): + def subst(self, string, raw=0, target=None, source=None, conv=None, executor=None): """Recursively interpolates construction variables from the Environment into the specified string, returning the expanded result. Construction variables are specified by a $ prefix @@ -503,6 +502,8 @@ class SubstitutionEnvironment: gvars = self.gvars() lvars = self.lvars() lvars['__env__'] = self + if executor: + lvars.update(executor.get_lvars()) return SCons.Subst.scons_subst(string, self, raw, target, source, gvars, lvars, conv) def subst_kw(self, kw, raw=0, target=None, source=None): @@ -514,12 +515,14 @@ class SubstitutionEnvironment: nkw[k] = v return nkw - def subst_list(self, string, raw=0, target=None, source=None, conv=None): + def subst_list(self, string, raw=0, target=None, source=None, conv=None, executor=None): """Calls through to SCons.Subst.scons_subst_list(). See the documentation for that function.""" gvars = self.gvars() lvars = self.lvars() lvars['__env__'] = self + if executor: + lvars.update(executor.get_lvars()) return SCons.Subst.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv) def subst_path(self, path, target=None, source=None): diff --git a/src/engine/SCons/Environment.xml b/src/engine/SCons/Environment.xml index a861105..d63d20f 100644 --- a/src/engine/SCons/Environment.xml +++ b/src/engine/SCons/Environment.xml @@ -115,6 +115,22 @@ below, for more information. </summary> </cvar> +<cvar name="CHANGED_SOURCES"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + +<cvar name="CHANGED_TARGETS"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + <cvar name="SOURCE"> <summary> A reserved variable name @@ -147,6 +163,22 @@ that may not be set or used in a construction environment. </summary> </cvar> +<cvar name="UNCHANGED_SOURCES"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + +<cvar name="UNCHANGED_TARGETS"> +<summary> +A reserved variable name +that may not be set or used in a construction environment. +(See "Variable Substitution," below.) +</summary> +</cvar> + <cvar name="TOOLS"> <summary> A list of the names of the Tool specifications diff --git a/src/engine/SCons/EnvironmentTests.py b/src/engine/SCons/EnvironmentTests.py index 0ea9dda..77d4602 100644 --- a/src/engine/SCons/EnvironmentTests.py +++ b/src/engine/SCons/EnvironmentTests.py @@ -172,6 +172,7 @@ class TestEnvironmentFixture: suffix = '.o', single_source = 1) kw['BUILDERS'] = {'Object' : static_obj} + static_obj.add_action('.cpp', 'fake action') env = apply(Environment, args, kw) return env @@ -887,6 +888,17 @@ sys.exit(0) class BaseTestCase(unittest.TestCase,TestEnvironmentFixture): + reserved_variables = [ + 'CHANGED_SOURCES', + 'CHANGED_TARGETS', + 'SOURCE', + 'SOURCES', + 'TARGET', + 'TARGETS', + 'UNCHANGED_SOURCES', + 'UNCHANGED_TARGETS', + ] + def test___init__(self): """Test construction Environment creation @@ -1123,10 +1135,14 @@ env4.builder1.env, env3) """Test warning generation when reserved variable names are set""" reserved_variables = [ + 'CHANGED_SOURCES', + 'CHANGED_TARGETS', 'SOURCE', 'SOURCES', 'TARGET', 'TARGETS', + 'UNCHANGED_SOURCES', + 'UNCHANGED_TARGETS', ] warning = SCons.Warnings.ReservedVariableWarning @@ -1135,7 +1151,7 @@ env4.builder1.env, env3) try: env4 = Environment() - for kw in reserved_variables: + for kw in self.reserved_variables: exc_caught = None try: env4[kw] = 'xyzzy' @@ -1149,12 +1165,7 @@ env4.builder1.env, env3) def test_FutureReservedVariables(self): """Test warning generation when future reserved variable names are set""" - future_reserved_variables = [ - 'CHANGED_SOURCES', - 'CHANGED_TARGETS', - 'UNCHANGED_SOURCES', - 'UNCHANGED_TARGETS', - ] + future_reserved_variables = [] warning = SCons.Warnings.FutureReservedVariableWarning SCons.Warnings.enableWarningClass(warning) @@ -3365,19 +3376,22 @@ def generate(env): f = env.xxx('$FOO') assert f == 'foo', f - def test_bad_keywords(type): + def test_bad_keywords(self): """Test trying to use reserved keywords in an Environment""" - reserved = ['TARGETS','SOURCES', 'SOURCE','TARGET'] added = [] - env = type.TestEnvironment(TARGETS = 'targets', + env = self.TestEnvironment(TARGETS = 'targets', SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', INIT = 'init') bad_msg = '%s is not reserved, but got omitted; see Environment.construction_var_name_ok' added.append('INIT') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3386,9 +3400,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', APPEND = 'append') added.append('APPEND') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3397,9 +3415,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', APPENDUNIQUE = 'appendunique') added.append('APPENDUNIQUE') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3408,9 +3430,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', PREPEND = 'prepend') added.append('PREPEND') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3419,9 +3445,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', PREPENDUNIQUE = 'prependunique') added.append('PREPENDUNIQUE') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3430,9 +3460,13 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', REPLACE = 'replace') added.append('REPLACE') - for x in reserved: + for x in self.reserved_variables: assert not env.has_key(x), env[x] for x in added: assert env.has_key(x), bad_msg % x @@ -3441,8 +3475,12 @@ def generate(env): SOURCES = 'sources', SOURCE = 'source', TARGET = 'target', + CHANGED_SOURCES = 'changed_sources', + CHANGED_TARGETS = 'changed_targets', + UNCHANGED_SOURCES = 'unchanged_sources', + UNCHANGED_TARGETS = 'unchanged_targets', COPY = 'copy') - for x in reserved: + for x in self.reserved_variables: assert not copy.has_key(x), env[x] for x in added + ['COPY']: assert copy.has_key(x), bad_msg % x @@ -3451,8 +3489,12 @@ def generate(env): 'SOURCES' : 'sources', 'SOURCE' : 'source', 'TARGET' : 'target', + 'CHANGED_SOURCES' : 'changed_sources', + 'CHANGED_TARGETS' : 'changed_targets', + 'UNCHANGED_SOURCES' : 'unchanged_sources', + 'UNCHANGED_TARGETS' : 'unchanged_targets', 'OVERRIDE' : 'override'}) - for x in reserved: + for x in self.reserved_variables: assert not over.has_key(x), over[x] for x in added + ['OVERRIDE']: assert over.has_key(x), bad_msg % x diff --git a/src/engine/SCons/Executor.py b/src/engine/SCons/Executor.py index 25db771..7e831e3 100644 --- a/src/engine/SCons/Executor.py +++ b/src/engine/SCons/Executor.py @@ -31,12 +31,85 @@ Nodes. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import string +import UserList from SCons.Debug import logInstanceCreation import SCons.Errors import SCons.Memoize +class Batch: + """Remembers exact association between targets + and sources of executor.""" + def __init__(self, targets=[], sources=[]): + self.targets = targets + self.sources = sources + + + +class TSList(UserList.UserList): + """A class that implements $TARGETS or $SOURCES expansions by wrapping + an executor Method. This class is used in the Executor.lvars() + to delay creation of NodeList objects until they're needed. + + Note that we subclass UserList.UserList purely so that the + is_Sequence() function will identify an object of this class as + a list during variable expansion. We're not really using any + UserList.UserList methods in practice. + """ + def __init__(self, func): + self.func = func + def __getattr__(self, attr): + nl = self.func() + return getattr(nl, attr) + def __getitem__(self, i): + nl = self.func() + return nl[i] + def __getslice__(self, i, j): + nl = self.func() + i = max(i, 0); j = max(j, 0) + return nl[i:j] + def __str__(self): + nl = self.func() + return str(nl) + def __repr__(self): + nl = self.func() + return repr(nl) + +class TSObject: + """A class that implements $TARGET or $SOURCE expansions by wrapping + an Executor method. + """ + def __init__(self, func): + self.func = func + def __getattr__(self, attr): + n = self.func() + return getattr(n, attr) + def __str__(self): + n = self.func() + if n: + return str(n) + return '' + def __repr__(self): + n = self.func() + if n: + return repr(n) + return '' + +def rfile(node): + """ + A function to return the results of a Node's rfile() method, + if it exists, and the Node itself otherwise (if it's a Value + Node, e.g.). + """ + try: + rfile = node.rfile + except AttributeError: + return node + else: + return rfile() + + class Executor: """A class for controlling instances of executing an action. @@ -58,11 +131,96 @@ class Executor: self.post_actions = [] self.env = env self.overridelist = overridelist - self.targets = targets - self.sources = SCons.Util.UniqueList(sources[:]) + if targets or sources: + self.batches = [Batch(targets[:], sources[:])] + else: + self.batches = [] self.builder_kw = builder_kw self._memo = {} + def get_lvars(self): + try: + return self.lvars + except AttributeError: + self.lvars = { + 'CHANGED_SOURCES' : TSList(self._get_changed_sources), + 'CHANGED_TARGETS' : TSList(self._get_changed_targets), + 'SOURCE' : TSObject(self._get_source), + 'SOURCES' : TSList(self._get_sources), + 'TARGET' : TSObject(self._get_target), + 'TARGETS' : TSList(self._get_targets), + 'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), + 'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), + } + return self.lvars + + def _get_changes(self): + cs = [] + ct = [] + us = [] + ut = [] + for b in self.batches: + if b.targets[0].changed(): + cs.extend(map(rfile, b.sources)) + ct.extend(b.targets) + else: + us.extend(map(rfile, b.sources)) + ut.extend(b.targets) + self._changed_sources_list = SCons.Util.NodeList(cs) + self._changed_targets_list = SCons.Util.NodeList(ct) + self._unchanged_sources_list = SCons.Util.NodeList(us) + self._unchanged_targets_list = SCons.Util.NodeList(ut) + + def _get_changed_sources(self, *args, **kw): + try: + return self._changed_sources_list + except AttributeError: + self._get_changes() + return self._changed_sources_list + + def _get_changed_targets(self, *args, **kw): + try: + return self._changed_targets_list + except AttributeError: + self._get_changes() + return self._changed_targets_list + + def _get_source(self, *args, **kw): + #return SCons.Util.NodeList([rfile(self.batches[0].sources[0]).get_subst_proxy()]) + return rfile(self.batches[0].sources[0]).get_subst_proxy() + + def _get_sources(self, *args, **kw): + return SCons.Util.NodeList(map(lambda n: rfile(n).get_subst_proxy(), self.get_all_sources())) + + def _get_target(self, *args, **kw): + #return SCons.Util.NodeList([self.batches[0].targets[0].get_subst_proxy()]) + return self.batches[0].targets[0].get_subst_proxy() + + def _get_targets(self, *args, **kw): + return SCons.Util.NodeList(map(lambda n: n.get_subst_proxy(), self.get_all_targets())) + + def _get_unchanged_sources(self, *args, **kw): + try: + return self._unchanged_sources_list + except AttributeError: + self._get_changes() + return self._unchanged_sources_list + + def _get_unchanged_targets(self, *args, **kw): + try: + return self._unchanged_targets_list + except AttributeError: + self._get_changes() + return self._unchanged_targets_list + + def get_action_targets(self): + if not self.action_list: + return [] + targets_string = self.action_list[0].get_targets(self.env, self) + if targets_string[0] == '$': + targets_string = targets_string[1:] + return self.get_lvars()[targets_string] + def set_action_list(self, action): import SCons.Util if not SCons.Util.is_List(action): @@ -75,6 +233,58 @@ class Executor: def get_action_list(self): return self.pre_actions + self.action_list + self.post_actions + def get_all_targets(self): + """Returns all targets for all batches of this Executor.""" + result = [] + for batch in self.batches: + # TODO(1.5): remove the list() cast + result.extend(list(batch.targets)) + return result + + def get_all_sources(self): + """Returns all sources for all batches of this Executor.""" + result = [] + for batch in self.batches: + # TODO(1.5): remove the list() cast + result.extend(list(batch.sources)) + return result + + def get_all_children(self): + """Returns all unique children (dependencies) for all batches + of this Executor. + + The Taskmaster can recognize when it's already evaluated a + Node, so we don't have to make this list unique for its intended + canonical use case, but we expect there to be a lot of redundancy + (long lists of batched .cc files #including the same .h files + over and over), so removing the duplicates once up front should + save the Taskmaster a lot of work. + """ + result = SCons.Util.UniqueList([]) + for target in self.get_all_targets(): + result.extend(target.children()) + return result + + def get_all_prerequisites(self): + """Returns all unique (order-only) prerequisites for all batches + of this Executor. + """ + result = SCons.Util.UniqueList([]) + for target in self.get_all_targets(): + # TODO(1.5): remove the list() cast + result.extend(list(target.prerequisites)) + return result + + def get_action_side_effects(self): + + """Returns all side effects for all batches of this + Executor used by the underlying Action. + """ + result = SCons.Util.UniqueList([]) + for target in self.get_action_targets(): + result.extend(target.side_effects) + return result + memoizer_counters.append(SCons.Memoize.CountValue('get_build_env')) def get_build_env(self): @@ -108,14 +318,17 @@ class Executor: """ env = self.get_build_env() try: - cwd = self.targets[0].cwd + cwd = self.batches[0].targets[0].cwd except (IndexError, AttributeError): cwd = None - return scanner.path(env, cwd, self.targets, self.get_sources()) + return scanner.path(env, cwd, + self.get_all_targets(), + self.get_all_sources()) def get_kw(self, kw={}): result = self.builder_kw.copy() result.update(kw) + result['executor'] = self return result def do_nothing(self, target, kw): @@ -127,7 +340,9 @@ class Executor: kw = self.get_kw(kw) status = 0 for act in self.get_action_list(): - status = apply(act, (self.targets, self.get_sources(), env), kw) + #args = (self.get_all_targets(), self.get_all_sources(), env) + args = ([], [], env) + status = apply(act, args, kw) if isinstance(status, SCons.Errors.BuildError): status.executor = self raise status @@ -135,7 +350,7 @@ class Executor: msg = "Error %s" % status raise SCons.Errors.BuildError( errstr=msg, - node=self.targets, + node=self.batches[0].targets, executor=self, action=act) return status @@ -154,20 +369,32 @@ class Executor: """Add source files to this Executor's list. This is necessary for "multi" Builders that can be called repeatedly to build up a source file list for a given target.""" - self.sources.extend(sources) + # TODO(batch): extend to multiple batches + assert (len(self.batches) == 1) + # TODO(batch): remove duplicates? + #slist = filter(lambda x, s=self.batches[0].sources: x not in s, sources) + self.batches[0].sources.extend(sources) def get_sources(self): - return self.sources + return self.batches[0].sources + + def add_batch(self, targets, sources): + """Add pair of associated target and source to this Executor's list. + This is necessary for "batch" Builders that can be called repeatedly + to build up a list of matching target and source files that will be + used in order to update multiple target files at once from multiple + corresponding source files, for tools like MSVC that support it.""" + self.batches.append(Batch(targets, sources)) def prepare(self): """ Preparatory checks for whether this Executor can go ahead and (try to) build its targets. """ - for s in self.get_sources(): + for s in self.get_all_sources(): if s.missing(): msg = "Source `%s' not found, needed by target `%s'." - raise SCons.Errors.StopError, msg % (s, self.targets[0]) + raise SCons.Errors.StopError, msg % (s, self.batches[0].targets[0]) def add_pre_action(self, action): self.pre_actions.append(action) @@ -179,7 +406,7 @@ class Executor: def my_str(self): env = self.get_build_env() - get = lambda action, t=self.targets, s=self.get_sources(), e=env: \ + get = lambda action, t=self.get_all_targets(), s=self.get_all_sources(), e=env: \ action.genstring(t, s, e) return string.join(map(get, self.get_action_list()), "\n") @@ -204,7 +431,7 @@ class Executor: except KeyError: pass env = self.get_build_env() - get = lambda action, t=self.targets, s=self.get_sources(), e=env: \ + get = lambda action, t=self.get_all_targets(), s=self.get_all_sources(), e=env: \ action.get_contents(t, s, e) result = string.join(map(get, self.get_action_list()), "") self._memo['get_contents'] = result @@ -218,11 +445,13 @@ class Executor: return 0 def scan_targets(self, scanner): - self.scan(scanner, self.targets) + # TODO(batch): scan by batches + self.scan(scanner, self.get_all_targets()) def scan_sources(self, scanner): - if self.sources: - self.scan(scanner, self.get_sources()) + # TODO(batch): scan by batches + if self.batches[0].sources: + self.scan(scanner, self.get_all_sources()) def scan(self, scanner, node_list): """Scan a list of this Executor's files (targets or sources) for @@ -232,6 +461,7 @@ class Executor: """ env = self.get_build_env() + # TODO(batch): scan by batches) deps = [] if scanner: for node in node_list: @@ -256,16 +486,16 @@ class Executor: deps.extend(self.get_implicit_deps()) - for tgt in self.targets: + for tgt in self.get_all_targets(): tgt.add_to_implicit(deps) - def _get_unignored_sources_key(self, ignore=()): - return tuple(ignore) + def _get_unignored_sources_key(self, node, ignore=()): + return (node,) + tuple(ignore) memoizer_counters.append(SCons.Memoize.CountDict('get_unignored_sources', _get_unignored_sources_key)) - def get_unignored_sources(self, ignore=()): - ignore = tuple(ignore) + def get_unignored_sources(self, node, ignore=()): + key = (node,) + tuple(ignore) try: memo_dict = self._memo['get_unignored_sources'] except KeyError: @@ -273,56 +503,56 @@ class Executor: self._memo['get_unignored_sources'] = memo_dict else: try: - return memo_dict[ignore] + return memo_dict[key] except KeyError: pass - sourcelist = self.get_sources() + if node: + # TODO: better way to do this (it's a linear search, + # but it may not be critical path)? + sourcelist = [] + for b in self.batches: + if node in b.targets: + sourcelist = b.sources + break + else: + sourcelist = self.get_all_sources() if ignore: idict = {} for i in ignore: idict[i] = 1 sourcelist = filter(lambda s, i=idict: not i.has_key(s), sourcelist) - memo_dict[ignore] = sourcelist + memo_dict[key] = sourcelist return sourcelist - def _process_sources_key(self, func, ignore=()): - return (func, tuple(ignore)) - - memoizer_counters.append(SCons.Memoize.CountDict('process_sources', _process_sources_key)) - - def process_sources(self, func, ignore=()): - memo_key = (func, tuple(ignore)) - try: - memo_dict = self._memo['process_sources'] - except KeyError: - memo_dict = {} - self._memo['process_sources'] = memo_dict - else: - try: - return memo_dict[memo_key] - except KeyError: - pass - - result = map(func, self.get_unignored_sources(ignore)) - - memo_dict[memo_key] = result - - return result - def get_implicit_deps(self): """Return the executor's implicit dependencies, i.e. the nodes of the commands to be executed.""" result = [] build_env = self.get_build_env() for act in self.get_action_list(): - result.extend(act.get_implicit_deps(self.targets, self.get_sources(), build_env)) + deps = act.get_implicit_deps(self.get_all_targets(), + self.get_all_sources(), + build_env) + result.extend(deps) return result + + +_batch_executors = {} + +def GetBatchExecutor(key): + return _batch_executors[key] + +def AddBatchExecutor(key, executor): + assert not _batch_executors.has_key(key) + _batch_executors[key] = executor + nullenv = None + def get_NullEnvironment(): """Use singleton pattern for Null Environments.""" global nullenv @@ -349,7 +579,7 @@ class Null: """ def __init__(self, *args, **kw): if __debug__: logInstanceCreation(self, 'Executor.Null') - self.targets = kw['targets'] + self.batches = [Batch(kw['targets'][:], [])] def get_build_env(self): return get_NullEnvironment() def get_build_scanner_path(self): @@ -360,17 +590,30 @@ class Null: pass def get_unignored_sources(self, *args, **kw): return tuple(()) + def get_action_targets(self): + return [] def get_action_list(self): return [] + def get_all_targets(self): + return self.batches[0].targets + def get_all_sources(self): + return self.batches[0].targets[0].sources + def get_all_children(self): + return self.get_all_sources() + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] def __call__(self, *args, **kw): return 0 def get_contents(self): return '' - def _morph(self): """Morph this Null executor to a real Executor object.""" + batches = self.batches self.__class__ = Executor - self.__init__([], targets=self.targets) + self.__init__([]) + self.batches = batches # The following methods require morphing this Null Executor to a # real Executor object. diff --git a/src/engine/SCons/ExecutorTests.py b/src/engine/SCons/ExecutorTests.py index 0fd11af..79e9d03 100644 --- a/src/engine/SCons/ExecutorTests.py +++ b/src/engine/SCons/ExecutorTests.py @@ -110,9 +110,11 @@ class ExecutorTestCase(unittest.TestCase): assert x.action_list == ['a'], x.action_list assert x.env == 'e', x.env assert x.overridelist == ['o'], x.overridelist - assert x.targets == 't', x.targets + targets = x.get_all_targets() + assert targets == ['t'], targets source_list.append('s3') - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2'], sources try: x = SCons.Executor.Executor(None, 'e', ['o'], 't', source_list) except SCons.Errors.UserError: @@ -200,11 +202,11 @@ class ExecutorTestCase(unittest.TestCase): ['s1', 's2'], builder_kw={'X':1, 'Y':2}) kw = x.get_kw() - assert kw == {'X':1, 'Y':2}, kw + assert kw == {'X':1, 'Y':2, 'executor':x}, kw kw = x.get_kw({'Z':3}) - assert kw == {'X':1, 'Y':2, 'Z':3}, kw + assert kw == {'X':1, 'Y':2, 'Z':3, 'executor':x}, kw kw = x.get_kw({'X':4}) - assert kw == {'X':4, 'Y':2}, kw + assert kw == {'X':4, 'Y':2, 'executor':x}, kw def test__call__(self): """Test calling an Executor""" @@ -222,7 +224,7 @@ class ExecutorTestCase(unittest.TestCase): a = MyAction([action1, action2]) t = MyNode('t') - x = SCons.Executor.Executor(a, env, [], t, ['s1', 's2']) + x = SCons.Executor.Executor(a, env, [], [t], ['s1', 's2']) x.add_pre_action(pre) x.add_post_action(post) x(t) @@ -233,7 +235,7 @@ class ExecutorTestCase(unittest.TestCase): result.append('pre_err') return 1 - x = SCons.Executor.Executor(a, env, [], t, ['s1', 's2']) + x = SCons.Executor.Executor(a, env, [], [t], ['s1', 's2']) x.add_pre_action(pre_err) x.add_post_action(post) try: @@ -268,22 +270,30 @@ class ExecutorTestCase(unittest.TestCase): def test_add_sources(self): """Test adding sources to an Executor""" x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2']) - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2'], sources + x.add_sources(['s1', 's2']) - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2', 's1', 's2'], sources + x.add_sources(['s3', 's1', 's4']) - assert x.sources == ['s1', 's2', 's3', 's4'], x.sources + sources = x.get_all_sources() + assert sources == ['s1', 's2', 's1', 's2', 's3', 's1', 's4'], sources def test_get_sources(self): """Test getting sources from an Executor""" x = SCons.Executor.Executor('b', 'e', 'o', 't', ['s1', 's2']) - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_sources() + assert sources == ['s1', 's2'], sources + x.add_sources(['s1', 's2']) - x.get_sources() - assert x.sources == ['s1', 's2'], x.sources + sources = x.get_sources() + assert sources == ['s1', 's2', 's1', 's2'], sources + x.add_sources(['s3', 's1', 's4']) - x.get_sources() - assert x.sources == ['s1', 's2', 's3', 's4'], x.sources + sources = x.get_sources() + assert sources == ['s1', 's2', 's1', 's2', 's3', 's1', 's4'], sources def test_prepare(self): """Test the Executor's prepare() method""" @@ -429,34 +439,15 @@ class ExecutorTestCase(unittest.TestCase): s3 = MyNode('s3') x = SCons.Executor.Executor('b', env, [{}], [], [s1, s2, s3]) - r = x.get_unignored_sources([]) + r = x.get_unignored_sources(None, []) assert r == [s1, s2, s3], map(str, r) - r = x.get_unignored_sources([s2]) + r = x.get_unignored_sources(None, [s2]) assert r == [s1, s3], map(str, r) - r = x.get_unignored_sources([s1, s3]) + r = x.get_unignored_sources(None, [s1, s3]) assert r == [s2], map(str, r) - def test_process_sources(self): - """Test processing the source list through a function""" - env = MyEnvironment() - s1 = MyNode('s1') - s2 = MyNode('s2') - s3 = MyNode('s3') - x = SCons.Executor.Executor('b', env, [{}], [], [s1, s2, s3]) - - r = x.process_sources(str) - assert r == ['s1', 's2', 's3'], r - - r = x.process_sources(str, [s2]) - assert r == ['s1', 's3'], r - - def xxx(x): - return 'xxx-' + str(x) - r = x.process_sources(xxx, [s1, s3]) - assert r == ['xxx-s2'], r - if __name__ == "__main__": diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f8911b0..9da9d8e 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -966,6 +966,9 @@ class Entry(Base): def _glob1(self, pattern, ondisk=True, source=False, strings=False): return self.disambiguate()._glob1(pattern, ondisk, source, strings) + def get_subst_proxy(self): + return self.disambiguate().get_subst_proxy() + # This is for later so we can differentiate between Entry the class and Entry # the method of the FS class. _classEntry = Entry diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 37dc465..e2a1e9e 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -304,7 +304,10 @@ class VariantDirTestCase(unittest.TestCase): class MkdirAction(Action): def __init__(self, dir_made): self.dir_made = dir_made - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() self.dir_made.extend(target) save_Link = SCons.Node.FS.Link @@ -3060,7 +3063,10 @@ class prepareTestCase(unittest.TestCase): class MkdirAction(Action): def __init__(self, dir_made): self.dir_made = dir_made - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() self.dir_made.extend(target) dir_made = [] @@ -3338,7 +3344,8 @@ class SpecialAttrTestCase(unittest.TestCase): assert s == os.path.normpath('baz/sub/file.suffix'), s assert f.srcpath.is_literal(), f.srcpath g = f.srcpath.get() - assert isinstance(g, SCons.Node.FS.Entry), g.__class__ + # Gets disambiguated to SCons.Node.FS.File by get_subst_proxy(). + assert isinstance(g, SCons.Node.FS.File), g.__class__ s = str(f.srcdir) assert s == os.path.normpath('baz/sub'), s @@ -3372,7 +3379,8 @@ class SpecialAttrTestCase(unittest.TestCase): try: fs.Entry('eee').get_subst_proxy().no_such_attr except AttributeError, e: - assert str(e) == "Entry instance 'eee' has no attribute 'no_such_attr'", e + # Gets disambiguated to File instance by get_subst_proxy(). + assert str(e) == "File instance 'eee' has no attribute 'no_such_attr'", e caught = 1 assert caught, "did not catch expected AttributeError" diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index 8bceaf6..73e1e05 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -67,8 +67,11 @@ class MyAction(MyActionBase): def __init__(self): self.order = 0 - def __call__(self, target, source, env): + def __call__(self, target, source, env, executor=None): global built_it, built_target, built_source, built_args, built_order + if executor: + target = executor.get_all_targets() + source = executor.get_all_sources() built_it = 1 built_target = target built_source = source diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 871efff..c44525f 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -621,7 +621,7 @@ class Node: # essentially short-circuits an N*M scan of the # sources for each individual target, which is a hell # of a lot more efficient. - for tgt in executor.targets: + for tgt in executor.get_all_targets(): tgt.add_to_implicit(implicit) if implicit_deps_unchanged or self.is_up_to_date(): @@ -714,7 +714,7 @@ class Node: if s not in ignore_set: sources.append(s) else: - sources = executor.get_unignored_sources(self.ignore) + sources = executor.get_unignored_sources(self, self.ignore) seen = set() bsources = [] bsourcesigs = [] diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py index 9974485..4fc657e 100644 --- a/src/engine/SCons/SConfTests.py +++ b/src/engine/SCons/SConfTests.py @@ -219,10 +219,11 @@ class SConfTestCase(unittest.TestCase): pass def get_executor(self): class Executor: - pass - e = Executor() - e.targets = [self] - return e + def __init__(self, targets): + self.targets = targets + def get_all_targets(self): + return self.targets + return Executor([self]) return [MyNode('n1'), MyNode('n2')] try: self.scons_env.Append(BUILDERS = {'SConfActionBuilder' : MyBuilder()}) diff --git a/src/engine/SCons/Subst.py b/src/engine/SCons/Subst.py index fc9d7d9..6459eec 100644 --- a/src/engine/SCons/Subst.py +++ b/src/engine/SCons/Subst.py @@ -256,6 +256,9 @@ class Target_or_Source: class NullNodeList(SCons.Util.NullSeq): def __call__(self, *args, **kwargs): return '' def __str__(self): return '' + # TODO(1.5): unneeded after new-style classes introduce iterators + def __getitem__(self, i): + raise IndexError NullNodesList = NullNodeList() @@ -285,6 +288,13 @@ def subst_dict(target, source): tnl = NLWrapper(target, get_tgt_subst_proxy) dict['TARGETS'] = Targets_or_Sources(tnl) dict['TARGET'] = Target_or_Source(tnl) + + # This is a total cheat, but hopefully this dictionary goes + # away soon anyway. We just let these expand to $TARGETS + # because that's "good enough" for the use of ToolSurrogates + # (see test/ToolSurrogate.py) to generate documentation. + dict['CHANGED_TARGETS'] = '$TARGETS' + dict['UNCHANGED_TARGETS'] = '$TARGETS' else: dict['TARGETS'] = NullNodesList dict['TARGET'] = NullNodesList @@ -304,6 +314,13 @@ def subst_dict(target, source): snl = NLWrapper(source, get_src_subst_proxy) dict['SOURCES'] = Targets_or_Sources(snl) dict['SOURCE'] = Target_or_Source(snl) + + # This is a total cheat, but hopefully this dictionary goes + # away soon anyway. We just let these expand to $TARGETS + # because that's "good enough" for the use of ToolSurrogates + # (see test/ToolSurrogate.py) to generate documentation. + dict['CHANGED_SOURCES'] = '$SOURCES' + dict['UNCHANGED_SOURCES'] = '$SOURCES' else: dict['SOURCES'] = NullNodesList dict['SOURCE'] = NullNodesList @@ -392,11 +409,9 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ source with two methods (substitute() and expand()) that handle the expansion. """ - def __init__(self, env, mode, target, source, conv, gvars): + def __init__(self, env, mode, conv, gvars): self.env = env self.mode = mode - self.target = target - self.source = source self.conv = conv self.gvars = gvars @@ -433,14 +448,14 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ except Exception, e: if e.__class__ in AllowableExceptions: return '' - raise_exception(e, self.target, s) + raise_exception(e, lvars['TARGETS'], s) else: if lvars.has_key(key): s = lvars[key] elif self.gvars.has_key(key): s = self.gvars[key] elif not NameError in AllowableExceptions: - raise_exception(NameError(key), self.target, s) + raise_exception(NameError(key), lvars['TARGETS'], s) else: return '' @@ -466,8 +481,8 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ return map(func, s) elif callable(s): try: - s = s(target=self.target, - source=self.source, + s = s(target=lvars['TARGETS'], + source=lvars['SOURCES'], env=self.env, for_signature=(self.mode != SUBST_CMD)) except TypeError: @@ -525,10 +540,11 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ # If we dropped that behavior (or found another way to cover it), # we could get rid of this call completely and just rely on the # Executor setting the variables. - d = subst_dict(target, source) - if d: - lvars = lvars.copy() - lvars.update(d) + if not lvars.has_key('TARGET'): + d = subst_dict(target, source) + if d: + lvars = lvars.copy() + lvars.update(d) # We're (most likely) going to eval() things. If Python doesn't # find a __builtins__ value in the global dictionary used for eval(), @@ -538,7 +554,7 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ # for expansion. gvars['__builtins__'] = __builtins__ - ss = StringSubber(env, mode, target, source, conv, gvars) + ss = StringSubber(env, mode, conv, gvars) result = ss.substitute(strSubst, lvars) try: @@ -595,12 +611,10 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv and the rest of the object takes care of doing the right thing internally. """ - def __init__(self, env, mode, target, source, conv, gvars): + def __init__(self, env, mode, conv, gvars): UserList.UserList.__init__(self, []) self.env = env self.mode = mode - self.target = target - self.source = source self.conv = conv self.gvars = gvars @@ -649,14 +663,14 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv except Exception, e: if e.__class__ in AllowableExceptions: return - raise_exception(e, self.target, s) + raise_exception(e, lvars['TARGETS'], s) else: if lvars.has_key(key): s = lvars[key] elif self.gvars.has_key(key): s = self.gvars[key] elif not NameError in AllowableExceptions: - raise_exception(NameError(), self.target, s) + raise_exception(NameError(), lvars['TARGETS'], s) else: return @@ -676,8 +690,8 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv self.next_word() elif callable(s): try: - s = s(target=self.target, - source=self.source, + s = s(target=lvars['TARGETS'], + source=lvars['SOURCES'], env=self.env, for_signature=(self.mode != SUBST_CMD)) except TypeError: @@ -820,10 +834,11 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv # If we dropped that behavior (or found another way to cover it), # we could get rid of this call completely and just rely on the # Executor setting the variables. - d = subst_dict(target, source) - if d: - lvars = lvars.copy() - lvars.update(d) + if not lvars.has_key('TARGET'): + d = subst_dict(target, source) + if d: + lvars = lvars.copy() + lvars.update(d) # We're (most likely) going to eval() things. If Python doesn't # find a __builtins__ value in the global dictionary used for eval(), @@ -833,7 +848,7 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv # for expansion. gvars['__builtins__'] = __builtins__ - ls = ListSubber(env, mode, target, source, conv, gvars) + ls = ListSubber(env, mode, conv, gvars) ls.substitute(strSubst, lvars, 0) try: diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 934e28b..4245480 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -186,8 +186,9 @@ class Task: # target t.prepare() methods check that each target's explicit # or implicit dependencies exists, and also initialize the # .sconsign info. - self.targets[0].get_executor().prepare() - for t in self.targets: + executor = self.targets[0].get_executor() + executor.prepare() + for t in executor.get_action_targets(): t.prepare() for s in t.side_effects: s.prepare() @@ -763,8 +764,10 @@ class Taskmaster: if T: T.write(self.trace_message(' already handled (executed)')) continue + executor = node.get_executor() + try: - children = node.children() + children = executor.get_all_children() except SystemExit: exc_value = sys.exc_info()[1] e = SCons.Errors.ExplicitExit(node, exc_value.code) @@ -786,7 +789,7 @@ class Taskmaster: children_not_ready = [] children_failed = False - for child in chain(children,node.prerequisites): + for child in chain(children, executor.get_all_prerequisites()): childstate = child.get_state() if T: T.write(self.trace_message(' ' + self.trace_node(child))) @@ -830,7 +833,8 @@ class Taskmaster: # added the other children to the list of candidate nodes # to keep on building (--keep-going). if children_failed: - node.set_state(NODE_FAILED) + for n in executor.get_action_targets(): + n.set_state(NODE_FAILED) if S: S.child_failed = S.child_failed + 1 if T: T.write(self.trace_message('****** %s\n' % self.trace_node(node))) @@ -861,7 +865,7 @@ class Taskmaster: # Skip this node if it has side-effects that are # currently being built: wait_side_effects = False - for se in node.side_effects: + for se in executor.get_action_side_effects(): if se.get_state() == NODE_EXECUTING: se.add_to_waiting_s_e(node) wait_side_effects = True @@ -900,7 +904,7 @@ class Taskmaster: if node is None: return None - tlist = node.get_executor().targets + tlist = node.get_executor().get_all_targets() task = self.tasker(self, tlist, node in self.original_top, node) try: diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index b36e4aa..c8bbdf4 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -166,8 +166,21 @@ class Node: class Executor: def prepare(self): pass + def get_action_targets(self): + return self.targets + def get_all_targets(self): + return self.targets + def get_all_children(self): + result = [] + for node in self.targets: + result.extend(node.children()) + return result + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] self.executor = Executor() - self.executor.targets = self.targets + self.executor.targets = self.targets return self.executor class OtherError(Exception): @@ -752,7 +765,7 @@ class TaskmasterTestCase(unittest.TestCase): # set it up by having something that approximates a real Builder # return this list--but that's more work than is probably # warranted right now. - t.targets = [n1, n2] + n1.get_executor().targets = [n1, n2] t.prepare() assert n1.prepared assert n2.prepared @@ -763,7 +776,7 @@ class TaskmasterTestCase(unittest.TestCase): t = tm.next_task() # More bogus reaching in and setting the targets. n3.set_state(SCons.Node.up_to_date) - t.targets = [n3, n4] + n3.get_executor().targets = [n3, n4] t.prepare() assert n3.prepared assert n4.prepared @@ -803,7 +816,7 @@ class TaskmasterTestCase(unittest.TestCase): tm = SCons.Taskmaster.Taskmaster([n6, n7]) t = tm.next_task() # More bogus reaching in and setting the targets. - t.targets = [n6, n7] + n6.get_executor().targets = [n6, n7] t.prepare() assert n6.prepared assert n7.prepared @@ -815,9 +828,21 @@ class TaskmasterTestCase(unittest.TestCase): class ExceptionExecutor: def prepare(self): raise Exception, "Executor.prepare() exception" + def get_all_targets(self): + return self.nodes + def get_all_children(self): + result = [] + for node in self.nodes: + result.extend(node.children()) + return result + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] n11 = Node("n11") n11.executor = ExceptionExecutor() + n11.executor.nodes = [n11] tm = SCons.Taskmaster.Taskmaster([n11]) t = tm.next_task() try: diff --git a/src/engine/SCons/Tool/javah.py b/src/engine/SCons/Tool/javah.py index 7eb4969..9b3c8c2 100644 --- a/src/engine/SCons/Tool/javah.py +++ b/src/engine/SCons/Tool/javah.py @@ -103,7 +103,7 @@ def emit_java_headers(target, source, env): def JavaHOutFlagGenerator(target, source, env, for_signature): try: t = target[0] - except (AttributeError, TypeError): + except (AttributeError, IndexError, TypeError): t = target try: return '-d ' + str(t.attributes.java_lookupdir) diff --git a/src/engine/SCons/Tool/mslink.py b/src/engine/SCons/Tool/mslink.py index 4268b58..53cab28 100644 --- a/src/engine/SCons/Tool/mslink.py +++ b/src/engine/SCons/Tool/mslink.py @@ -65,7 +65,10 @@ def _dllSources(target, source, env, for_signature, paramtp): deffile = env.FindIxes(source, "WINDOWSDEFPREFIX", "WINDOWSDEFSUFFIX") for src in source: - if src == deffile: + # Check explicitly for a non-None deffile so that the __cmp__ + # method of the base SCons.Util.Proxy class used for some Node + # proxies doesn't try to use a non-existent __dict__ attribute. + if deffile and src == deffile: # Treat this source as a .def file. listCmd.append("/def:%s" % src.get_string(for_signature)) else: diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index b324a32..0898b91 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -672,40 +672,103 @@ res_builder = SCons.Builder.Builder(action=res_action, src_builder=[], source_scanner=res_scanner) +def msvc_batch_key(action, env, target, source): + """ + Returns a key to identify unique batches of sources for compilation. + + If batching is enabled (via the $MSVC_BATCH setting), then all + target+source pairs that use the same action, defined by the same + environment, and have the same target and source directories, will + be batched. + + Returning None specifies that the specified target+source should not + be batched with other compilations. + """ + b = env.subst('$MSVC_BATCH') + if b in (None, '', '0'): + # We're not using batching; return no key. + return None + t = target[0] + s = source[0] + if os.path.splitext(t.name)[0] != os.path.splitext(s.name)[0]: + # The base names are different, so this *must* be compiled + # separately; return no key. + return None + return (id(action), id(env), t.dir, s.dir) + +def msvc_output_flag(target, source, env, for_signature): + """ + Returns the correct /Fo flag for batching. + + If batching is disabled or there's only one source file, then we + return an /Fo string that specifies the target explicitly. Otherwise, + we return an /Fo string that just specifies the first target's + directory (where the Visual C/C++ compiler will put the .obj files). + """ + b = env.subst('$MSVC_BATCH') + if b in (None, '', '0') or len(source) == 1: + return '/Fo$TARGET' + else: + # The Visual C/C++ compiler requires a \ at the end of the /Fo + # option to indicate an output directory. We use os.sep here so + # that the test(s) for this can be run on non-Windows systems + # without having a hard-coded backslash mess up command-line + # argument parsing. + return '/Fo${TARGET.dir}' + os.sep + +CAction = SCons.Action.Action("$CCCOM", "$CCCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') +ShCAction = SCons.Action.Action("$SHCCCOM", "$SHCCCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') +CXXAction = SCons.Action.Action("$CXXCOM", "$CXXCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') +ShCXXAction = SCons.Action.Action("$SHCXXCOM", "$SHCXXCOMSTR", + batch_key=msvc_batch_key, + targets='$CHANGED_TARGETS') def generate(env): """Add Builders and construction variables for MSVC++ to an Environment.""" static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + # TODO(batch): shouldn't reach in to cmdgen this way; necessary + # for now to bypass the checks in Builder.DictCmdGenerator.__call__() + # and allow .cc and .cpp to be compiled in the same command line. + static_obj.cmdgen.source_ext_match = False + shared_obj.cmdgen.source_ext_match = False + for suffix in CSuffixes: - static_obj.add_action(suffix, SCons.Defaults.CAction) - shared_obj.add_action(suffix, SCons.Defaults.ShCAction) + static_obj.add_action(suffix, CAction) + shared_obj.add_action(suffix, ShCAction) static_obj.add_emitter(suffix, static_object_emitter) shared_obj.add_emitter(suffix, shared_object_emitter) for suffix in CXXSuffixes: - static_obj.add_action(suffix, SCons.Defaults.CXXAction) - shared_obj.add_action(suffix, SCons.Defaults.ShCXXAction) + static_obj.add_action(suffix, CXXAction) + shared_obj.add_action(suffix, ShCXXAction) static_obj.add_emitter(suffix, static_object_emitter) shared_obj.add_emitter(suffix, shared_object_emitter) env['CCPDBFLAGS'] = SCons.Util.CLVar(['${(PDB and "/Z7") or ""}']) env['CCPCHFLAGS'] = SCons.Util.CLVar(['${(PCH and "/Yu%s /Fp%s"%(PCHSTOP or "",File(PCH))) or ""}']) + env['_MSVC_OUTPUT_FLAG'] = msvc_output_flag env['_CCCOMCOM'] = '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS' env['CC'] = 'cl' env['CCFLAGS'] = SCons.Util.CLVar('/nologo') env['CFLAGS'] = SCons.Util.CLVar('') - env['CCCOM'] = '$CC /Fo$TARGET /c $SOURCES $CFLAGS $CCFLAGS $_CCCOMCOM' + env['CCCOM'] = '$CC $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $CFLAGS $CCFLAGS $_CCCOMCOM' env['SHCC'] = '$CC' env['SHCCFLAGS'] = SCons.Util.CLVar('$CCFLAGS') env['SHCFLAGS'] = SCons.Util.CLVar('$CFLAGS') - env['SHCCCOM'] = '$SHCC /Fo$TARGET /c $SOURCES $SHCFLAGS $SHCCFLAGS $_CCCOMCOM' + env['SHCCCOM'] = '$SHCC $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $SHCFLAGS $SHCCFLAGS $_CCCOMCOM' env['CXX'] = '$CC' env['CXXFLAGS'] = SCons.Util.CLVar('$CCFLAGS $( /TP $)') - env['CXXCOM'] = '$CXX /Fo$TARGET /c $SOURCES $CXXFLAGS $CCFLAGS $_CCCOMCOM' + env['CXXCOM'] = '$CXX $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $CXXFLAGS $CCFLAGS $_CCCOMCOM' env['SHCXX'] = '$CXX' env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS') - env['SHCXXCOM'] = '$SHCXX /Fo$TARGET /c $SOURCES $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM' + env['SHCXXCOM'] = '$SHCXX $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM' env['CPPDEFPREFIX'] = '/D' env['CPPDEFSUFFIX'] = '' env['INCPREFIX'] = '/I' diff --git a/src/engine/SCons/Tool/msvc.xml b/src/engine/SCons/Tool/msvc.xml index a2fdd7e..31dcdf1 100644 --- a/src/engine/SCons/Tool/msvc.xml +++ b/src/engine/SCons/Tool/msvc.xml @@ -140,6 +140,26 @@ env['CCPDBFLAGS'] = '/Zi /Fd${TARGET}.pdb' </summary> </cvar> +<cvar name="MSVC_BATCH"> +<summary> +When set to any true value, +specifies that &SCons; should batch +compilation of object files +when calling the Microsoft Visual C/C++ compiler. +All compilations of source files from the same source directory +that generate target files in a same output directory +and were configured in &SCons; using the same construction environment +will be built in a single call to the compiler. +Only source files that have changed since their +object files were built will be passed to each compiler invocation +(via the &cv-link-CHANGED_SOURCES; construction variable). +Any compilations where the object (target) file base name +(minus the <filename>.obj</filename>) +does not match the source file base name +will be compiled separately. +</summary> +</cvar> + <cvar name="PCH"> <summary> The Microsoft Visual C++ precompiled header that will be used when compiling diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index cbec5dd..a9f7b70 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -1092,11 +1092,12 @@ class Selector(OrderedDict): """A callable ordered dictionary that maps file suffixes to dictionary values. We preserve the order in which items are added so that get_suffix() calls always return the first suffix added.""" - def __call__(self, env, source): - try: - ext = source[0].suffix - except IndexError: - ext = "" + def __call__(self, env, source, ext=None): + if ext is None: + try: + ext = source[0].suffix + except IndexError: + ext = "" try: return self[ext] except KeyError: @@ -1561,20 +1562,32 @@ class Null: #cls._inst = type.__new__(cls, *args, **kwargs) cls._inst = apply(type.__new__, (cls,) + args, kwargs) return cls._inst - def __init__(self, *args, **kwargs): pass - def __call__(self, *args, **kwargs): return self - def __repr__(self): return "Null(0x%08X)" % id(self) - def __nonzero__(self): return False - def __getattr__(self, mname): return self - def __setattr__(self, name, value): return self - def __delattr__(self, name): return self + def __init__(self, *args, **kwargs): + pass + def __call__(self, *args, **kwargs): + return self + def __repr__(self): + return "Null(0x%08X)" % id(self) + def __nonzero__(self): + return False + def __getattr__(self, name): + return self + def __setattr__(self, name, value): + return self + def __delattr__(self, name): + return self class NullSeq(Null): - def __len__(self): return 0 - def __iter__(self): return iter(()) - def __getitem__(self, i): return self - def __delitem__(self, i): return self - def __setitem__(self, i, v): return self + def __len__(self): + return 0 + def __iter__(self): + return iter(()) + def __getitem__(self, i): + return self + def __delitem__(self, i): + return self + def __setitem__(self, i, v): + return self del __revision__ |