From 123be6df3fe113bad181c0c93aab4ee58b26e91f Mon Sep 17 00:00:00 2001 From: Steven Knight Date: Thu, 25 Oct 2001 13:11:10 +0000 Subject: Support building (sub)directories. --- src/engine/SCons/Job.py | 8 +- src/engine/SCons/JobTests.py | 48 +++++---- src/engine/SCons/Node/FS.py | 79 +++++++++++--- src/engine/SCons/Node/FSTests.py | 88 ++++++++++++++-- src/engine/SCons/Node/NodeTests.py | 6 ++ src/engine/SCons/Node/__init__.py | 16 ++- src/engine/SCons/Sig/SigTests.py | 84 ++++++++------- src/engine/SCons/Sig/__init__.py | 68 +++--------- src/engine/SCons/Taskmaster.py | 172 ++++++++++++++++++------------ src/engine/SCons/TaskmasterTests.py | 205 ++++++++++++++++++++---------------- src/script/scons.py | 62 +++++------ test/Command.py | 3 +- test/ENV.py | 3 +- test/Program.py | 24 ++--- test/option-c.py | 5 +- test/subdir.py | 65 ++++++++++++ 16 files changed, 582 insertions(+), 354 deletions(-) create mode 100644 test/subdir.py diff --git a/src/engine/SCons/Job.py b/src/engine/SCons/Job.py index 17315fc..a3707be 100644 --- a/src/engine/SCons/Job.py +++ b/src/engine/SCons/Job.py @@ -111,9 +111,9 @@ class Serial: except: # Let the failed() callback function arrange for the # build to stop if that's appropriate. - self.taskmaster.failed(task) + task.failed() else: - self.taskmaster.executed(task) + task.executed() def stop(self): """Serial jobs are always finished when start() returns, so there @@ -246,9 +246,9 @@ class Parallel: # Let the failed() callback function arrange for # calling self.jobs.stop() to to stop the build # if that's appropriate. - self.taskmaster.failed(task) + task.failed() else: - self.taskmaster.executed(task) + task.executed() # signal the cv whether the task failed or not, # or otherwise the other Jobs might diff --git a/src/engine/SCons/JobTests.py b/src/engine/SCons/JobTests.py index 7026726..e5168a4 100644 --- a/src/engine/SCons/JobTests.py +++ b/src/engine/SCons/JobTests.py @@ -75,15 +75,39 @@ class Task: self.taskmaster.end_list.append(self.i) self.taskmaster.guard.release() + def executed(self): + self.taskmaster.num_executed = self.taskmaster.num_executed + 1 + + self.taskmaster.test_case.failUnless(self.was_executed, + "the task wasn't really executed") + self.taskmaster.test_case.failUnless(self.__class__ is Task, + "the task wasn't really a Task instance") + + def failed(self): + self.taskmaster.num_failed = self.taskmaster.num_failed + 1 + self.taskmaster.stop = 1 + class ExceptionTask: """A dummy task class for testing purposes.""" def __init__(self, i, taskmaster): - pass + self.taskmaster = taskmaster def execute(self): raise "exception" + def executed(self): + self.taskmaster.num_executed = self.taskmaster.num_executed + 1 + + self.taskmaster.test_case.failUnless(self.was_executed, + "the task wasn't really executed") + self.taskmaster.test_case.failUnless(self.__class__ is Task, + "the task wasn't really a Task instance") + + def failed(self): + self.taskmaster.num_failed = self.taskmaster.num_failed + 1 + self.taskmaster.stop = 1 + class Taskmaster: """A dummy taskmaster class for testing the job classes.""" @@ -123,18 +147,6 @@ class Taskmaster: def all_tasks_are_iterated(self): return self.num_iterated == self.num_tasks - - def executed(self, task): - self.num_executed = self.num_executed + 1 - - self.test_case.failUnless(task.was_executed, - "the task wasn't really executed") - self.test_case.failUnless(task.__class__ is Task, - "the task wasn't really a Task instance") - - def failed(self, task): - self.num_failed = self.num_failed + 1 - self.stop = 1 def is_blocked(self): # simulate blocking tasks @@ -241,13 +253,3 @@ if __name__ == "__main__": sys.exit(2) elif not result.wasSuccessful(): sys.exit(1) - - - - - - - - - - diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index e44da22..cfb4142 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -35,7 +35,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import os.path -from SCons.Node import Node +import SCons.Node from UserDict import UserDict import sys @@ -119,6 +119,7 @@ class FS: self.Root = PathDict() self.Top = self.__doLookup(Dir, path) self.Top.path = '.' + self.Top.path_ = './' def __doLookup(self, fsclass, name, directory=None): """This method differs from the File and Dir factory methods in @@ -228,7 +229,7 @@ class FS: -class Entry(Node): +class Entry(SCons.Node.Node): """A generic class for file system entries. This class if for when we don't know yet whether the entry being looked up is a file or a directory. Instances of this class can morph into either @@ -241,8 +242,9 @@ class Entry(Node): our relative and absolute paths, identify our parent directory, and indicate that this node should use signatures.""" - Node.__init__(self) + SCons.Node.Node.__init__(self) + self.name = name if directory: self.abspath = os.path.join(directory.abspath, name) if str(directory.path) == '.': @@ -251,13 +253,18 @@ class Entry(Node): self.path = os.path.join(directory.path, name) else: self.abspath = self.path = name - self.parent = directory + self.path_ = self.path + self.abspath_ = self.abspath + self.dir = directory self.use_signature = 1 def __str__(self): """A FS node's string representation is its path name.""" return self.path + def set_signature(self, sig): + SCons.Node.Node.set_signature(self, sig) + def exists(self): return os.path.exists(self.path) @@ -299,17 +306,18 @@ class Dir(Entry): into the file system tree. Specify that directories (this node) don't use signatures for currency calculation.""" - self.path = os.path.join(self.path, '') - self.abspath = os.path.join(self.abspath, '') + self.path_ = os.path.join(self.path, '') + self.abspath_ = os.path.join(self.abspath, '') self.entries = PathDict() self.entries['.'] = self - if hasattr(self, 'parent'): - self.entries['..'] = self.parent - delattr(self, 'parent') + if hasattr(self, 'dir'): + self.entries['..'] = self.dir else: self.entries['..'] = None self.use_signature = None + self.builder = 1 + self._sconsign = None def up(self): return self.entries['..'] @@ -321,15 +329,42 @@ class Dir(Entry): return self.entries['..'].root() def children(self): - return map(lambda x, s=self: s.entries[x], + #XXX --random: randomize "dependencies?" + kids = map(lambda x, s=self: s.entries[x], filter(lambda k: k != '.' and k != '..', self.entries.keys())) + kids.sort() + return kids + + def build(self): + """A null "builder" for directories.""" + pass + + def set_signature(self, sig): + """A directory has no signature.""" + pass def current(self): - """Always return that a directory node is out-of-date so - that it will always be "built" by trying to build all of - its directory entries.""" - return 0 + """If all of our children were up-to-date, then this + directory was up-to-date, too.""" + state = 0 + for kid in self.children(): + s = kid.get_state() + if s and (not state or s > state): + state = s + import SCons.Node + if state == SCons.Node.up_to_date: + return 1 + else: + return 0 + + def sconsign(self): + if not self._sconsign: + #XXX Rework this to get rid of the hard-coding + import SCons.Sig + import SCons.Sig.MD5 + self._sconsign = SCons.Sig.SConsignFile(self.path, SCons.Sig.MD5) + return self._sconsign # XXX TODO? @@ -362,13 +397,25 @@ class File(Entry): pass def root(self): - return self.parent.root() + return self.dir.root() def get_contents(self): return open(self.path, "r").read() def get_timestamp(self): - return os.path.getmtime(self.path) + if self.exists(): + return os.path.getmtime(self.path) + else: + return 0 + + def set_signature(self, sig): + Entry.set_signature(self, sig) + #XXX Rework this to get rid of the hard-coding + import SCons.Sig.MD5 + self.dir.sconsign().set(self.name, self.get_timestamp(), sig, SCons.Sig.MD5) + + def get_oldentry(self): + return self.dir.sconsign().get(self.name) default_fs = FS() diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 044f83f..8b1ee5a 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -84,36 +84,60 @@ class FSTestCase(unittest.TestCase): for sep in seps: - def Dir_test(lpath, path, abspath, up_path, fileSys=fs, s=sep): + def Dir_test(lpath, path_, abspath_, up_path_, fileSys=fs, s=sep): dir = fileSys.Dir(string.replace(lpath, '/', s)) + def strip_slash(p): + if p[-1] == '/' and len(p) > 1: + p = p[:-1] + return p + path = strip_slash(path_) + abspath = strip_slash(abspath_) + up_path = strip_slash(up_path_) + name = string.split(abspath, '/')[-1] + if os.sep != '/': path = string.replace(path, '/', os.sep) + path_ = string.replace(path_, '/', os.sep) abspath = string.replace(abspath, '/', os.sep) + abspath_ = string.replace(abspath_, '/', os.sep) up_path = string.replace(up_path, '/', os.sep) + up_path_ = string.replace(up_path_, '/', os.sep) + assert dir.name == name, \ + "dir.name %s != expected name %s" % \ + (dir.name, name) assert dir.path == path, \ "dir.path %s != expected path %s" % \ (dir.path, path) assert str(dir) == path, \ "str(dir) %s != expected path %s" % \ (str(dir), path) + assert dir.path_ == path_, \ + "dir.path_ %s != expected path_ %s" % \ + (dir.path_, path_) assert dir.abspath == abspath, \ "dir.abspath %s != expected absolute path %s" % \ (dir.abspath, abspath) + assert dir.abspath_ == abspath_, \ + "dir.abspath_ %s != expected absolute path_ %s" % \ + (dir.abspath_, abspath_) assert dir.up().path == up_path, \ "dir.up().path %s != expected parent path %s" % \ (dir.up().path, up_path) + assert dir.up().path_ == up_path_, \ + "dir.up().path_ %s != expected parent path_ %s" % \ + (dir.up().path_, up_path_) - Dir_test('foo', 'foo/', sub_dir_foo, '.') + Dir_test('foo', 'foo/', sub_dir_foo, './') Dir_test('foo/bar', 'foo/bar/', sub_dir_foo_bar, 'foo/') Dir_test('/foo', '/foo/', '/foo/', '/') Dir_test('/foo/bar', '/foo/bar/', '/foo/bar/', '/foo/') Dir_test('..', sub, sub, wp) - Dir_test('foo/..', '.', sub_dir, sub) + Dir_test('foo/..', './', sub_dir, sub) Dir_test('../foo', sub_foo, sub_foo, sub) - Dir_test('.', '.', sub_dir, sub) - Dir_test('./.', '.', sub_dir, sub) + Dir_test('.', './', sub_dir, sub) + Dir_test('./.', './', sub_dir, sub) Dir_test('foo/./bar', 'foo/bar/', sub_dir_foo_bar, 'foo/') try: @@ -136,7 +160,7 @@ class FSTestCase(unittest.TestCase): f2 = fs.File('d1') except TypeError, x: assert str(x) == ("Tried to lookup Dir '%s' as a File." % - os.path.join('d1', '')), x + 'd1'), x except: raise @@ -150,10 +174,16 @@ class FSTestCase(unittest.TestCase): fs.Dir(string.join(['ddd', 'd1', 'f5'], sep)) kids = map(lambda x: x.path, dir.children()) kids.sort() - assert kids == [os.path.join('ddd', 'd1', ''), + assert kids == [os.path.join('ddd', 'd1'), os.path.join('ddd', 'f1'), os.path.join('ddd', 'f2'), os.path.join('ddd', 'f3')] + kids = map(lambda x: x.path_, dir.children()) + kids.sort() + assert kids == [os.path.join('ddd', 'd1', ''), + os.path.join('ddd', 'f1'), + os.path.join('ddd', 'f2'), + os.path.join('ddd', 'f3')] # Test for sub-classing of node building. global built_it @@ -164,7 +194,7 @@ class FSTestCase(unittest.TestCase): d1.builder_set(Builder()) d1.env_set(Environment()) d1.build() - assert built_it + assert not built_it assert d1.get_parents() == [] @@ -178,35 +208,71 @@ class FSTestCase(unittest.TestCase): e1 = fs.Entry("d1") assert e1.__class__.__name__ == 'Dir' - assert e1.path == "d1/", e1.path + assert e1.path == "d1", e1.path + assert e1.path_ == "d1/", e1.path_ + assert e1.dir.path == ".", e1.dir.path e2 = fs.Entry("d1/f1") assert e2.__class__.__name__ == 'File' assert e2.path == "d1/f1", e2.path + assert e2.path_ == "d1/f1", e2.path_ + assert e2.dir.path == "d1", e2.dir.path e3 = fs.Entry("e3") assert e3.__class__.__name__ == 'Entry' assert e3.path == "e3", e3.path + assert e3.path_ == "e3", e3.path_ + assert e3.dir.path == ".", e3.dir.path e4 = fs.Entry("d1/e4") assert e4.__class__.__name__ == 'Entry' assert e4.path == "d1/e4", e4.path + assert e4.path_ == "d1/e4", e4.path_ + assert e4.dir.path == "d1", e4.dir.path e5 = fs.Entry("e3/e5") assert e3.__class__.__name__ == 'Dir' - assert e3.path == "e3/", e3.path + assert e3.path == "e3", e3.path + assert e3.path_ == "e3/", e3.path_ + assert e3.dir.path == ".", e3.dir.path assert e5.__class__.__name__ == 'Entry' assert e5.path == "e3/e5", e5.path + assert e5.path_ == "e3/e5", e5.path_ + assert e5.dir.path == "e3", e5.dir.path e6 = fs.Dir("d1/e4") assert e6 is e4 assert e4.__class__.__name__ == 'Dir' - assert e4.path == "d1/e4/", e4.path + assert e4.path == "d1/e4", e4.path + assert e4.path_ == "d1/e4/", e4.path_ + assert e4.dir.path == "d1", e4.dir.path e7 = fs.File("e3/e5") assert e7 is e5 assert e5.__class__.__name__ == 'File' assert e5.path == "e3/e5", e5.path + assert e5.path_ == "e3/e5", e5.path_ + assert e5.dir.path == "e3", e5.dir.path + + #XXX test set_signature() + + #XXX test exists() + + #XXX test current() for directories + + #XXX test sconsign() for directories + + #XXX test set_signature() for directories + + #XXX test build() for directories + + #XXX test root() + + #XXX test get_contents() + + #XXX test get_timestamp() + + #XXX test get_oldentry() diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index b8015c2..cf7e59b 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -90,6 +90,8 @@ class NodeTestCase(unittest.TestCase): assert node.builder == b def test_current(self): + """Test the default current() method + """ node = SCons.Node.Node() assert node.current() is None @@ -226,6 +228,10 @@ class NodeTestCase(unittest.TestCase): assert node.get_state() == None node.set_state(SCons.Node.executing) assert node.get_state() == SCons.Node.executing + assert SCons.Node.pending < SCons.Node.executing + assert SCons.Node.executing < SCons.Node.up_to_date + assert SCons.Node.up_to_date < SCons.Node.executed + assert SCons.Node.executed < SCons.Node.failed def test_walker(self): """Test walking a Node tree. diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index b7bdecf..2995576 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -36,12 +36,18 @@ import string import types import copy -# Node states: -executing = 1 -executed = 2 +# Node states +# +# These are in "priority" order, so that the maximum value for any +# child/dependency of a node represents the state of that node if +# it has no builder of its own. The canonical example is a file +# system directory, which is only up to date if all of its children +# were up to date. +pending = 1 +executing = 2 up_to_date = 3 -failed = 4 -pending = 5 +executed = 4 +failed = 5 class Node: """The base Node class, for entities that we know how to diff --git a/src/engine/SCons/Sig/SigTests.py b/src/engine/SCons/Sig/SigTests.py index b42e464..2122fe7 100644 --- a/src/engine/SCons/Sig/SigTests.py +++ b/src/engine/SCons/Sig/SigTests.py @@ -52,6 +52,8 @@ class DummyNode: self.builder = file.builder self.depends = [] self.use_signature = 1 + self.oldtime = 0 + self.oldsig = 0 def get_contents(self): # a file that doesn't exist has no contents: @@ -85,6 +87,9 @@ class DummyNode: def get_signature(self): return self.sig + def get_oldentry(self): + return (self.oldtime, self.oldsig) + def create_files(test): args = [(test.workpath('f1.c'), 'blah blah', 111, 0), #0 @@ -119,6 +124,15 @@ def create_nodes(files): nodes[10].sources = [nodes[9]] return nodes + +def current(calc, node): + s = calc.get_signature(node) + return calc.current(node, s) + +def write(calc, nodes): + for node in nodes: + node.oldtime = node.file.timestamp + node.oldsig = calc.get_signature(node) class SigTestBase: @@ -140,79 +154,77 @@ class SigTestBase: calc = SCons.Sig.Calculator(self.module) for node in nodes: - self.failUnless(not calc.current(node), "none of the nodes should be current") + self.failUnless(not current(calc, node), "none of the nodes should be current") # simulate a build: self.files[1].modify('built', 222) self.files[7].modify('built', 222) self.files[9].modify('built', 222) self.files[10].modify('built', 222) - - calc.write(nodes) def test_built(self): nodes = create_nodes(self.files) calc = SCons.Sig.Calculator(self.module) + + write(calc, nodes) for node in nodes: - self.failUnless(calc.current(node), "all of the nodes should be current") - - calc.write(nodes) + self.failUnless(current(calc, node), "all of the nodes should be current") def test_modify(self): nodes = create_nodes(self.files) + calc = SCons.Sig.Calculator(self.module) + + write(calc, nodes) + #simulate a modification of some files self.files[0].modify('blah blah blah', 333) self.files[3].modify('blah blah blah', 333) self.files[6].modify('blah blah blah', 333) self.files[8].modify('blah blah blah', 333) - calc = SCons.Sig.Calculator(self.module) - - self.failUnless(not calc.current(nodes[0]), "modified directly") - self.failUnless(not calc.current(nodes[1]), "direct source modified") - self.failUnless(calc.current(nodes[2])) - self.failUnless(not calc.current(nodes[3]), "modified directly") - self.failUnless(calc.current(nodes[4])) - self.failUnless(calc.current(nodes[5])) - self.failUnless(not calc.current(nodes[6]), "modified directly") - self.failUnless(not calc.current(nodes[7]), "indirect source modified") - self.failUnless(not calc.current(nodes[8]), "modified directory") - self.failUnless(not calc.current(nodes[9]), "direct source modified") - self.failUnless(not calc.current(nodes[10]), "indirect source modified") - - calc.write(nodes) + self.failUnless(not current(calc, nodes[0]), "modified directly") + self.failUnless(not current(calc, nodes[1]), "direct source modified") + self.failUnless(current(calc, nodes[2])) + self.failUnless(not current(calc, nodes[3]), "modified directly") + self.failUnless(current(calc, nodes[4])) + self.failUnless(current(calc, nodes[5])) + self.failUnless(not current(calc, nodes[6]), "modified directly") + self.failUnless(not current(calc, nodes[7]), "indirect source modified") + self.failUnless(not current(calc, nodes[8]), "modified directory") + self.failUnless(not current(calc, nodes[9]), "direct source modified") + self.failUnless(not current(calc, nodes[10]), "indirect source modified") def test_delete(self): nodes = create_nodes(self.files) + + calc = SCons.Sig.Calculator(self.module) + + write(calc, nodes) #simulate the deletion of some files self.files[1].modify(None, 0) self.files[7].modify(None, 0) self.files[9].modify(None, 0) - - calc = SCons.Sig.Calculator(self.module) - self.failUnless(calc.current(nodes[0])) - self.failUnless(not calc.current(nodes[1]), "deleted") - self.failUnless(calc.current(nodes[2])) - self.failUnless(calc.current(nodes[3])) - self.failUnless(calc.current(nodes[4])) - self.failUnless(calc.current(nodes[5])) - self.failUnless(calc.current(nodes[6])) - self.failUnless(not calc.current(nodes[7]), "deleted") - self.failUnless(calc.current(nodes[8])) - self.failUnless(not calc.current(nodes[9]), "deleted") - self.failUnless(calc.current(nodes[10]), + self.failUnless(current(calc, nodes[0])) + self.failUnless(not current(calc, nodes[1]), "deleted") + self.failUnless(current(calc, nodes[2])) + self.failUnless(current(calc, nodes[3])) + self.failUnless(current(calc, nodes[4])) + self.failUnless(current(calc, nodes[5])) + self.failUnless(current(calc, nodes[6])) + self.failUnless(not current(calc, nodes[7]), "deleted") + self.failUnless(current(calc, nodes[8])) + self.failUnless(not current(calc, nodes[9]), "deleted") + self.failUnless(current(calc, nodes[10]), "current even though it's source was deleted") - calc.write(nodes) - class MD5TestCase(unittest.TestCase, SigTestBase): """Test MD5 signatures""" diff --git a/src/engine/SCons/Sig/__init__.py b/src/engine/SCons/Sig/__init__.py index 8e4ed56..36bceba 100644 --- a/src/engine/SCons/Sig/__init__.py +++ b/src/engine/SCons/Sig/__init__.py @@ -32,6 +32,15 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path import string + +#XXX Get rid of the global array so this becomes re-entrant. +sig_files = [] + +def write(): + global sig_files + for sig_file in sig_files: + sig_file.write() + class SConsignFile: """ Encapsulates reading and writing a .sconsign file. @@ -56,6 +65,9 @@ class SConsignFile: time, signature = map(string.strip, string.split(rest, " ")) self.entries[filename] = (int(time), module.from_string(signature)) + global sig_files + sig_files.append(self) + def get(self, filename): """ Get the signature for a file @@ -103,7 +115,6 @@ class Calculator: module - the signature module to use for signature calculations """ self.module = module - self.sig_files = {} def collect(self, node, signatures): @@ -116,24 +127,11 @@ class Calculator: """ for source_node in node.children(): if not signatures.has_key(source_node): - signature = self.signature(source_node) + signature = self.get_signature(source_node) signatures[source_node] = signature self.collect(source_node, signatures) - def get_sig_file(self, dir): - """ - Get a sconsign file from the cache, or add it to the cache. - - dir - the dir for the sconsign file - returns - the sconsign file - """ - if self.sig_files.has_key(dir): - return self.sig_files[dir] - else: - self.sig_files[dir] = SConsignFile(dir, self.module) - return self.sig_files[dir] - - def signature(self, node): + def get_signature(self, node): """ Get the signature for a node. @@ -141,7 +139,7 @@ class Calculator: returns - the signature or None if the signature could not be computed. - This method also stores the signature in the node and + This method does not store the signature in the node and in the .sconsign file. """ @@ -163,22 +161,9 @@ class Calculator: # XXX handle nodes that are not under the source root sig = self.module.signature(node) - node.set_signature(sig) - - dir, filename = os.path.split(node.path) - if node.exists(): - timestamp = node.get_timestamp() - else: - timestamp = 0 - - self.get_sig_file(dir).set(filename, - timestamp, - sig, - self.module) - return sig - def current(self, node): + def current(self, node, newsig): """ Check if a node is up to date. @@ -196,30 +181,11 @@ class Calculator: # that doesn't exist, or a directory. return c - dir, filename = os.path.split(node.path) - oldtime, oldsig = self.get_sig_file(dir).get(filename) + oldtime, oldsig = node.get_oldentry() newtime = node.get_timestamp() if not node.builder and newtime == oldtime: newsig = oldsig - else: - newsig = self.signature(node) return self.module.current(newsig, oldsig) - - def write(self, nodes): - """ - Write out all of the signature files. - - nodes - the nodes whose signatures may have changed durring - the build - """ - - # make sure all the signatures have been calculated: - for node in nodes: - self.signature(node) - - for sig_file in self.sig_files.values(): - sig_file.write() - diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 3b4ee85..9e8e105 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -38,25 +38,68 @@ import SCons.Node class Task: """Default SCons build engine task.""" - def __init__(self,target): + def __init__(self, tm, target, top): + self.tm = tm self.target = target + self.sig = None + self.top = top def execute(self): self.target.build() - def set_state(self, state): - return self.target.set_state(state) - def get_target(self): return self.target + + def set_sig(self, sig): + self.sig = sig + + def set_state(self, state): + self.target.set_state(state) + + def up_to_date(self): + self.set_state(SCons.Node.up_to_date) + + def executed(self): + self.set_state(SCons.Node.executed) + self.tm.add_pending(self.target) + self.target.set_signature(self.sig) + + def failed(self): + self.fail_stop() + + def fail_stop(self): + self.set_state(SCons.Node.failed) + self.tm.stop() + + def fail_continue(self): + def get_parents(node): return node.get_parents() + walker = SCons.Node.Walker(self.target, get_parents) + while 1: + node = walker.next() + if node == None: break + self.tm.remove_pending(node) + node.set_state(SCons.Node.failed) -def current(node): - """Default SCons build engine is-it-current function. - This returns "always out of date," so every node is always - built/visited. - """ - return None + +class Calc: + def get_signature(self, node): + """ + """ + return None + + def set_signature(self, node): + """ + """ + pass + + def current(self, node, sig): + """Default SCons build engine is-it-current function. + + This returns "always out of date," so every node is always + built/visited. + """ + return 0 @@ -64,92 +107,85 @@ class Taskmaster: """A generic Taskmaster for handling a bunch of targets. Classes that override methods of this class should call - the base class method, so this class can do it's thing. + the base class method, so this class can do its thing. """ - def __init__(self, - targets=[], - tasker=Task, - current=current, - ignore_errors=0, - keep_going_on_error=0): + def __init__(self, targets=[], tasker=Task, calc=Calc()): self.walkers = map(SCons.Node.Walker, targets) self.tasker = tasker - self.current = current + self.calc = calc self.targets = targets self.ready = [] self.pending = 0 - self.ignore_errors = ignore_errors - self.keep_going_on_error = keep_going_on_error self._find_next_ready_node() - + def next_task(self): if self.ready: - n = self.ready.pop() - n.set_state(SCons.Node.executing) + task = self.ready.pop() + task.set_state(SCons.Node.executing) if not self.ready: self._find_next_ready_node() - - return self.tasker(n) + return task else: return None - + def _find_next_ready_node(self): """Find the next node that is ready to be built""" while self.walkers: n = self.walkers[0].next() if n == None: self.walkers.pop(0) - elif n.get_state() == SCons.Node.up_to_date: - self.up_to_date(n, self.walkers[0].is_done()) - elif n.get_state() == None: - if not n.children_are_executed(): - n.set_state(SCons.Node.pending) - self.pending = self.pending + 1 - elif self.current(n): - n.set_state(SCons.Node.up_to_date) - self.up_to_date(n, self.walkers[0].is_done()) - else: - self.ready.append(n) - return - + continue + if n.get_state(): + # The state is set, so someone has already been here + # (finished or currently executing). Find another one. + continue + if not n.builder: + # It's a source file, we don't need to build it, + # but mark it as "up to date" so targets won't + # wait for it. + n.set_state(SCons.Node.up_to_date) + continue + task = self.tasker(self, n, self.walkers[0].is_done()) + if not n.children_are_executed(): + n.set_state(SCons.Node.pending) + n.task = task + self.pending = self.pending + 1 + continue + sig = self.calc.get_signature(n) + task.set_sig(sig) + if self.calc.current(n, sig): + task.up_to_date() + else: + self.ready.append(task) + return None + def is_blocked(self): return not self.ready and self.pending - def up_to_date(self, node): - pass - - def executed(self, task): - task.set_state(SCons.Node.executed) + def stop(self): + self.walkers = [] + self.pending = 0 + self.ready = [] + def add_pending(self, node): # add all the pending parents that are now executable to the 'ready' # queue: - n = task.get_target() ready = filter(lambda x: (x.get_state() == SCons.Node.pending and x.children_are_executed()), - n.get_parents()) - self.ready.extend(ready) - self.pending = self.pending - len(ready) - - def failed(self, task): - if self.ignore_errors: - self.executed(task) - else: - if self.keep_going_on_error: - # mark all the depants of this node as failed: - def get_parents(node): return node.get_parents() - walker = SCons.Node.Walker(task.get_target(), get_parents) - while 1: - node = walker.next() - if node == None: break - if node.get_state() == SCons.Node.pending: - self.pending = self.pending - 1 - node.set_state(SCons.Node.failed) + node.get_parents()) + for n in ready: + task = n.task + delattr(n, "task") + sig = self.calc.get_signature(n) + task.set_sig(sig) + if self.calc.current(n, sig): + task.up_to_date() else: - # terminate the build: - self.walkers = [] - self.pending = 0 - self.ready = [] + self.ready.append(task) + self.pending = self.pending - len(ready) - task.set_state(SCons.Node.failed) + def remove_pending(self, node): + if node.get_state() == SCons.Node.pending: + self.pending = self.pending - 1 diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index 809df6a..a15a673 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -31,41 +31,94 @@ import SCons.Taskmaster built = None +executed = None class Node: def __init__(self, name, kids = []): self.name = name - self.kids = kids + self.kids = kids + self.builder = Node.build + self.signature = None self.state = None self.parents = [] - + for kid in kids: kid.parents.append(self) - + def build(self): global built built = self.name + " built" def children(self): return self.kids - + def get_parents(self): return self.parents - + def get_state(self): return self.state def set_state(self, state): self.state = state + def set_signature(self, sig): + self.signature = sig + def children_are_executed(self): return reduce(lambda x,y: ((y.get_state() == SCons.Node.executed or y.get_state() == SCons.Node.up_to_date) and x), self.children(), 1) + + + +#class Task(unittest.TestCase): +# def test_execute(self): +# pass +# +# def test_get_target(self): +# pass +# +# def test_set_sig(self): +# pass +# +# def test_set_state(self): +# pass +# +# def test_up_to_date(self): +# pass +# +# def test_executed(self): +# pass +# +# def test_failed(self): +# pass +# +# def test_fail_stop(self): +# pass +# +# def test_fail_continue(self): +# pass + +class Task: + def __init__(self, target): + self.target = target + + def get_target(self): + return self.target + + def up_to_date(self): + pass + + def executed(self): + pass + + def failed(self): + pass + class TaskmasterTestCase(unittest.TestCase): @@ -73,17 +126,16 @@ class TaskmasterTestCase(unittest.TestCase): """Test fetching the next task """ global built - - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster([n1,n1]) + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1, n1]) t = tm.next_task() - tm.executed(t) + t.executed() t = tm.next_task() assert t == None - - n1 = Node("n1") - n2 = Node("n2") + n1 = Node("n1") + n2 = Node("n2") n3 = Node("n3", [n1, n2]) tm = SCons.Taskmaster.Taskmaster([n3]) @@ -91,41 +143,42 @@ class TaskmasterTestCase(unittest.TestCase): t = tm.next_task() t.execute() assert built == "n1 built" - tm.executed(t) + t.executed() t = tm.next_task() t.execute() assert built == "n2 built" - tm.executed(t) + t.executed() t = tm.next_task() t.execute() assert built == "n3 built" - tm.executed(t) + t.executed() assert tm.next_task() == None - def current(node): - return 1 - built = "up to date: " global top_node top_node = n3 - class MyTM(SCons.Taskmaster.Taskmaster): - def up_to_date(self, node, top): - if node == top_node: - assert top + class MyTask(SCons.Taskmaster.Task): + def up_to_date(self): + if self.target == top_node: + assert self.top global built - built = built + " " + node.name + built = built + " " + self.target.name + SCons.Taskmaster.Task.up_to_date(self) + class MyCalc(SCons.Taskmaster.Calc): + def current(self, node, sig): + return 1 n1.set_state(None) n2.set_state(None) n3.set_state(None) - tm = MyTM(targets = [n3], current = current) + tm = SCons.Taskmaster.Taskmaster(targets = [n3], + tasker = MyTask, calc = MyCalc()) assert tm.next_task() == None - print built assert built == "up to date: n1 n2 n3" @@ -149,18 +202,18 @@ class TaskmasterTestCase(unittest.TestCase): t4 = tm.next_task() assert t4.get_target() == n4 assert tm.is_blocked() - tm.executed(t4) + t4.executed() assert tm.is_blocked() - tm.executed(t1) + t1.executed() assert tm.is_blocked() - tm.executed(t2) + t2.executed() assert not tm.is_blocked() t3 = tm.next_task() assert t3.get_target() == n3 assert tm.is_blocked() - tm.executed(t3) + t3.executed() assert not tm.is_blocked() t5 = tm.next_task() assert t5.get_target() == n5 @@ -173,7 +226,7 @@ class TaskmasterTestCase(unittest.TestCase): n4.set_state(SCons.Node.executed) tm = SCons.Taskmaster.Taskmaster([n4]) assert tm.next_task() == None - + def test_is_blocked(self): """Test whether a task is blocked @@ -188,78 +241,50 @@ class TaskmasterTestCase(unittest.TestCase): tm = MyTM() assert tm.is_blocked() == 1 - def test_executed(self): - """Test the executed() method + def test_stop(self): + """Test the stop() method - Both default and overridden in a subclass. - """ - tm = SCons.Taskmaster.Taskmaster() - foo = Node('foo') - tm.executed(SCons.Taskmaster.Task(foo)) - - class MyTM(SCons.Taskmaster.Taskmaster): - def executed(self, task): - return 'x' + task - tm = MyTM() - assert tm.executed('foo') == 'xfoo' + Both default and overridden in a subclass. + """ + global built - def test_ignore_errors(self): n1 = Node("n1") n2 = Node("n2") - n3 = Node("n3", [n1]) + n3 = Node("n3", [n1, n2]) - tm = SCons.Taskmaster.Taskmaster([n3, n2], - SCons.Taskmaster.Task, - SCons.Taskmaster.current, - 1) - - t = tm.next_task() - assert t.get_target() == n1 - tm.failed(t) - t = tm.next_task() - assert t.get_target() == n3 - tm.failed(t) + tm = SCons.Taskmaster.Taskmaster([n3]) t = tm.next_task() - assert t.get_target() == n2 - + t.execute() + assert built == "n1 built" + t.executed() - def test_keep_going(self): - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1]) - - tm = SCons.Taskmaster.Taskmaster([n3, n2], - SCons.Taskmaster.Task, - SCons.Taskmaster.current, - 0, - 1) + tm.stop() + assert tm.next_task() is None - tm.failed(tm.next_task()) - t = tm.next_task() - assert t.get_target() == n2 - tm.executed(t) - assert not tm.is_blocked() - t = tm.next_task() - assert t == None + class MyTM(SCons.Taskmaster.Taskmaster): + def stop(self): + global built + built = "MyTM.stop()" + SCons.Taskmaster.Taskmaster.stop(self) + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) - def test_failed(self): - """Test the failed() method + built = None + tm = MyTM([n3]) + tm.next_task().execute() + assert built == "n1 built" - Both default and overridden in a subclass. - """ - foo = Node('foo') - bar = Node('bar') - tm = SCons.Taskmaster.Taskmaster([foo,bar]) - tm.failed(tm.next_task()) - assert tm.next_task() == None - - class MyTM(SCons.Taskmaster.Taskmaster): - def failed(self, task): - return 'y' + task - tm = MyTM() - assert tm.failed('foo') == 'yfoo' + tm.stop() + assert built == "MyTM.stop()" + assert tm.next_task() is None + #def test_add_pending(self): + # passs + # + #def test_remove_pending(self): + # passs diff --git a/src/script/scons.py b/src/script/scons.py index 84d7f78..5f5d45d 100644 --- a/src/script/scons.py +++ b/src/script/scons.py @@ -66,9 +66,20 @@ class BuildTask(SCons.Taskmaster.Task): except BuildError, e: sys.stderr.write("scons: *** [%s] Error %d\n" % (e.node, e.stat)) raise + + def up_to_date(self): + if self.top: + print 'scons: "%s" is up to date.' % str(self.target) + SCons.Taskmaster.Task.up_to_date(self) - def set_state(self, state): - return self.target.set_state(state) + def failed(self): + global ignore_errors + if ignore_errors: + SCons.Taskmaster.Task.executed(self) + elif keep_going_on_error: + SCons.Taskmaster.Task.fail_continue(self) + else: + SCons.Taskmaster.Task.fail_stop(self) class CleanTask(SCons.Taskmaster.Task): """An SCons clean task.""" @@ -77,19 +88,6 @@ class CleanTask(SCons.Taskmaster.Task): os.unlink(self.target.path) print "Removed " + self.target.path -class ScriptTaskmaster(SCons.Taskmaster.Taskmaster): - """Controlling logic for tasks. - - This is the stock Taskmaster from the build engine, except - that we override the up_to_date() method to provide our - script-specific up-to-date message for command-line targets, - and failed to provide the ignore-errors feature. - """ - def up_to_date(self, node, top): - if top: - print 'scons: "%s" is up to date.' % node - SCons.Taskmaster.Taskmaster.up_to_date(self, node) - # Global variables @@ -100,6 +98,7 @@ num_jobs = 1 scripts = [] task_class = BuildTask # default action is to build targets current_func = None +calc = None ignore_errors = 0 keep_going_on_error = 0 @@ -310,9 +309,18 @@ def options_init(): help = "Ignored for compatibility.") def opt_c(opt, arg): - global task_class, current_func - task_class = CleanTask - current_func = SCons.Taskmaster.current + global task_class, calc + task_class = CleanTask + class CleanCalculator: + def get_signature(self, node): + return None + def set_signature(self, node, sig): + pass + def current(self, node, sig): + return 0 + def write(self): + pass + calc = CleanCalculator() Option(func = opt_c, short = 'c', long = ['clean', 'remove'], @@ -540,7 +548,7 @@ def UsageString(): def main(): - global scripts, help_option, num_jobs, task_class, current_func + global scripts, help_option, num_jobs, task_class, calc targets = [] @@ -629,24 +637,18 @@ def main(): if not targets: targets = default_targets - nodes = map(lambda x: SCons.Node.FS.default_fs.File(x), targets) - - calc = SCons.Sig.Calculator(SCons.Sig.MD5) + nodes = map(lambda x: SCons.Node.FS.default_fs.Entry(x), targets) - if not current_func: - current_func = calc.current + if not calc: + calc = SCons.Sig.Calculator(SCons.Sig.MD5) - taskmaster = ScriptTaskmaster(nodes, - task_class, - current_func, - ignore_errors, - keep_going_on_error) + taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, calc) jobs = SCons.Job.Jobs(num_jobs, taskmaster) jobs.start() jobs.wait() - calc.write(nodes) + SCons.Sig.write() if __name__ == "__main__": try: diff --git a/test/Command.py b/test/Command.py index c3080cb..e9d7452 100644 --- a/test/Command.py +++ b/test/Command.py @@ -57,8 +57,7 @@ test.write('f2.in', "f2.in\n") test.write('f3.in', "f3.in\n") -#XXXtest.run(arguments = '.') -test.run(arguments = 'f1.out f2.out f3.out') +test.run(arguments = '.') test.fail_test(test.read('f1.out') != "f1.in\n") test.fail_test(test.read('f2.out') != "f2.in\n") diff --git a/test/ENV.py b/test/ENV.py index 221e7f9..1ff699d 100644 --- a/test/ENV.py +++ b/test/ENV.py @@ -71,8 +71,7 @@ os.chmod(bin2_build_py, 0755) test.write('input', "input file\n") -#test.run(arguments = '.') -test.run(arguments = 'bin1.out bin2.out') +test.run(arguments = '.') test.fail_test(test.read('bin1.out') != "bin1/build.py\ninput file\n") test.fail_test(test.read('bin2.out') != "bin2/build.py\ninput file\n") diff --git a/test/Program.py b/test/Program.py index 2612d1c..6a0db70 100644 --- a/test/Program.py +++ b/test/Program.py @@ -36,7 +36,7 @@ test.write('SConstruct', """ env = Environment() env.Program(target = 'foo1', source = 'f1.c') env.Program(target = 'foo2', source = 'f2a.c f2b.c f2c.c') -#XXXenv.Program(target = 'foo3', source = ['f3a.c', 'f3b.c', 'f3c.c']) +env.Program(target = 'foo3', source = ['f3a.c', 'f3b.c', 'f3c.c']) """) test.write('f1.c', """ @@ -109,15 +109,13 @@ main(int argc, char *argv[]) } """) -#XXXtest.run(arguments = '.') -test.run(arguments = 'foo1 foo2') +test.run(arguments = '.') test.run(program = test.workpath('foo1'), stdout = "f1.c\n") test.run(program = test.workpath('foo2'), stdout = "f2a.c\nf2b.c\nf2c.c\n") -#XXXtest.run(program = test.workpath('foo3'), stdout = "f3a.c\nf3b.c\nf3c.c\n") +test.run(program = test.workpath('foo3'), stdout = "f3a.c\nf3b.c\nf3c.c\n") -#XXXtest.up_to_date(arguments = '.') -test.up_to_date(arguments = 'foo1 foo2') +test.up_to_date(arguments = '.') test.write('f1.c', """ int @@ -137,22 +135,22 @@ f3b(void) } """) -#XXXtest.run(arguments = '.') -test.run(arguments = 'foo1 foo2') +test.run(arguments = '.') test.run(program = test.workpath('foo1'), stdout = "f1.c X\n") test.run(program = test.workpath('foo2'), stdout = "f2a.c\nf2b.c\nf2c.c\n") -#XXXtest.run(program = test.workpath('foo3'), stdout = "f3a.c\nf3b.c X\nf3c.c\n") +test.run(program = test.workpath('foo3'), stdout = "f3a.c\nf3b.c X\nf3c.c\n") -#XXXtest.up_to_date(arguments = '.') -test.up_to_date(arguments = 'foo1 foo2') +test.up_to_date(arguments = '.') # make sure the programs don't get rebuilt, because nothing changed: oldtime1 = os.path.getmtime(test.workpath('foo1')) oldtime2 = os.path.getmtime(test.workpath('foo2')) -time.sleep(1) # introduce a small delay, to make the test valid -test.run(arguments = 'foo1 foo2') +oldtime3 = os.path.getmtime(test.workpath('foo3')) +time.sleep(2) # introduce a small delay, to make the test valid +test.run(arguments = '.') test.fail_test(not (oldtime1 == os.path.getmtime(test.workpath('foo1')))) test.fail_test(not (oldtime2 == os.path.getmtime(test.workpath('foo2')))) +test.fail_test(not (oldtime3 == os.path.getmtime(test.workpath('foo3')))) test.pass_test() diff --git a/test/option-c.py b/test/option-c.py index 0e41ffd..b188d53 100644 --- a/test/option-c.py +++ b/test/option-c.py @@ -78,14 +78,13 @@ test.fail_test(os.path.exists(test.workpath('foo1.out'))) test.fail_test(os.path.exists(test.workpath('foo2.out'))) test.fail_test(os.path.exists(test.workpath('foo3.out'))) -test.run(arguments = 'foo1.out foo2.out foo3.out') +test.run(arguments = '.') test.fail_test(test.read(test.workpath('foo1.out')) != "foo1.in\n") test.fail_test(test.read(test.workpath('foo2.out')) != "foo2.in\n") test.fail_test(test.read(test.workpath('foo3.out')) != "foo3.in\n") -#XXXtest.run(arguments = '-c .', -test.run(arguments = '-c foo1.out foo2.out foo3.out', +test.run(arguments = '-c .', stdout = "Removed foo1.out\nRemoved foo2.out\nRemoved foo3.out\n") test.fail_test(os.path.exists(test.workpath('foo1.out'))) diff --git a/test/subdir.py b/test/subdir.py new file mode 100644 index 0000000..7071b54 --- /dev/null +++ b/test/subdir.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# +# Copyright (c) 2001 Steven Knight +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import TestSCons +import os.path + +test = TestSCons.TestSCons() + +test.subdir('subdir') + +test.write('build.py', r""" +import sys +contents = open(sys.argv[2], 'r').read() +file = open(sys.argv[1], 'w') +file.write(contents) +file.close() +""") + +test.write('SConstruct', """ +B = Builder(name = "B", action = "python build.py $targets $sources") +env = Environment(BUILDERS = [B]) +env.B(target = 'subdir/f1.out', source = 'subdir/f1.in') +env.B(target = 'subdir/f2.out', source = 'subdir/f2.in') +env.B(target = 'subdir/f3.out', source = 'subdir/f3.in') +env.B(target = 'subdir/f4.out', source = 'subdir/f4.in') +""") + +test.write('subdir/f1.in', "f1.in\n") +test.write('subdir/f2.in', "f2.in\n") +test.write('subdir/f3.in', "f3.in\n") +test.write('subdir/f4.in', "f4.in\n") + +test.run(arguments = 'subdir') + +test.fail_test(test.read('subdir/f1.out') != "f1.in\n") +test.fail_test(test.read('subdir/f2.out') != "f2.in\n") +test.fail_test(test.read('subdir/f3.out') != "f3.in\n") +test.fail_test(test.read('subdir/f4.out') != "f4.in\n") + +test.up_to_date(arguments = 'subdir') + +test.pass_test() -- cgit v0.12