diff options
Diffstat (limited to 'src/engine/SCons/Node')
| -rw-r--r-- | src/engine/SCons/Node/FS.py | 232 | ||||
| -rw-r--r-- | src/engine/SCons/Node/FSTests.py | 264 | ||||
| -rw-r--r-- | src/engine/SCons/Node/NodeTests.py | 3 |
3 files changed, 150 insertions, 349 deletions
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index ad21a4d..74dc655 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -223,77 +223,6 @@ def get_MkdirBuilder(): name = "MkdirBuilder") return MkdirBuilder -def CacheRetrieveFunc(target, source, env): - t = target[0] - fs = t.fs - cachedir, cachefile = t.cachepath() - if not fs.exists(cachefile): - fs.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) - return 1 - fs.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) - if SCons.Action.execute_actions: - if fs.islink(cachefile): - fs.symlink(fs.readlink(cachefile), t.path) - else: - fs.copy2(cachefile, t.path) - st = fs.stat(cachefile) - fs.chmod(t.path, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) - return 0 - -def CacheRetrieveString(target, source, env): - t = target[0] - cachedir, cachefile = t.cachepath() - if t.fs.exists(cachefile): - return "Retrieved `%s' from cache" % t.path - return None - -CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) - -CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) - -def CachePushFunc(target, source, env): - t = target[0] - if t.nocache: - return - fs = t.fs - cachedir, cachefile = t.cachepath() - if fs.exists(cachefile): - # Don't bother copying it if it's already there. Note that - # usually this "shouldn't happen" because if the file already - # existed in cache, we'd have retrieved the file from there, - # not built it. This can happen, though, in a race, if some - # other person running the same build pushes their copy to - # the cache after we decide we need to build it but before our - # build completes. - fs.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) - return - - fs.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) - - if not fs.isdir(cachedir): - fs.makedirs(cachedir) - - tempfile = cachefile+'.tmp' - try: - if fs.islink(t.path): - fs.symlink(fs.readlink(t.path), tempfile) - else: - fs.copy2(t.path, tempfile) - fs.rename(tempfile, cachefile) - st = fs.stat(t.path) - fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) - except (IOError, OSError): - # It's possible someone else tried writing the file at the - # same time we did, or else that there was some problem like - # the CacheDir being on a separate file system that's full. - # In any case, inability to push a file to cache doesn't affect - # the correctness of the build, so just print a warning. - SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, - "Unable to copy %s to cache. Cache file is %s" - % (str(target), cachefile)) - -CachePush = SCons.Action.Action(CachePushFunc, None) - class _Null: pass @@ -738,6 +667,15 @@ class Base(SCons.Node.Node): return ret def target_from_source(self, prefix, suffix, splitext=SCons.Util.splitext): + """ + + Generates a target entry that corresponds to this entry (usually + a source file) with the specified prefix and suffix. + + Note that this method can be overridden dynamically for generated + files that need different behavior. See Tool/swig.py for + an example. + """ return self.dir.Entry(prefix + splitext(self.name)[0] + suffix) def _Rfindalldirs_key(self, pathlist): @@ -783,6 +721,26 @@ class Base(SCons.Node.Node): cwd = self.cwd or self.fs._cwd return cwd.Rfindalldirs(pathlist) + memoizer_counters.append(SCons.Memoize.CountValue('rentry')) + + def rentry(self): + try: + return self._memo['rentry'] + except KeyError: + pass + result = self + if not self.exists(): + norm_name = _my_normcase(self.name) + for dir in self.dir.get_all_rdirs(): + try: + node = dir.entries[norm_name] + except KeyError: + if dir.entry_exists_on_disk(self.name): + result = dir.Entry(self.name) + break + self._memo['rentry'] = result + return result + class Entry(Base): """This is the class for generic Node.FS entries--that is, things that could be a File or a Dir, but we're just not sure yet. @@ -993,9 +951,6 @@ class FS(LocalFS): self.Root = {} self.SConstruct_dir = None - self.CachePath = None - self.cache_force = None - self.cache_show = None self.max_drift = default_max_drift self.Top = None @@ -1259,30 +1214,6 @@ class FS(LocalFS): d = self.Dir(d) self.Top.addRepository(d) - def CacheDebugWrite(self, fmt, target, cachefile): - self.CacheDebugFP.write(fmt % (target, os.path.split(cachefile)[1])) - - def CacheDebugQuiet(self, fmt, target, cachefile): - pass - - CacheDebug = CacheDebugQuiet - - def CacheDebugEnable(self, file): - if file == '-': - self.CacheDebugFP = sys.stdout - else: - self.CacheDebugFP = open(file, 'w') - self.CacheDebug = self.CacheDebugWrite - - def CacheDir(self, path): - try: - import SCons.Sig.MD5 - except ImportError: - msg = "No MD5 module available, CacheDir() not supported" - SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) - else: - self.CachePath = path - def build_dir_target_climb(self, orig, dir, tail): """Create targets in corresponding build directories @@ -1681,9 +1612,9 @@ class Dir(Base): def srcdir_duplicate(self, name): for dir in self.srcdir_list(): if dir.entry_exists_on_disk(name): - srcnode = dir.File(name) + srcnode = dir.Entry(name).disambiguate() if self.duplicate: - node = self.File(name) + node = self.Entry(name).disambiguate() node.do_duplicate(srcnode) return node else: @@ -1750,7 +1681,37 @@ class Dir(Base): diskcheck_sccs(self, name): try: return self.File(name) except TypeError: pass - return self.srcdir_duplicate(name) + node = self.srcdir_duplicate(name) + if isinstance(node, Dir): + node = None + return node + + def walk(self, func, arg): + """ + Walk this directory tree by calling the specified function + for each directory in the tree. + + This behaves like the os.path.walk() function, but for in-memory + Node.FS.Dir objects. The function takes the same arguments as + the functions passed to os.path.walk(): + + func(arg, dirname, fnames) + + Except that "dirname" will actually be the directory *Node*, + not the string. The '.' and '..' entries are excluded from + fnames. The fnames list may be modified in-place to filter the + subdirectories visited or otherwise impose a specific order. + The "arg" argument is always passed to func() and may be used + in any way (or ignored, passing None is common). + """ + entries = self.entries + names = entries.keys() + names.remove('.') + names.remove('..') + func(arg, self, names) + select_dirs = lambda n, e=entries: isinstance(e[n], Dir) + for dirname in filter(select_dirs, names): + entries[dirname].walk(func, arg) class RootDir(Dir): """A class for the root directory of a file system. @@ -2048,53 +2009,17 @@ class File(Base): so only do thread safe stuff here. Do thread unsafe stuff in built(). - Note that there's a special trick here with the execute flag - (one that's not normally done for other actions). Basically - if the user requested a no_exec (-n) build, then - SCons.Action.execute_actions is set to 0 and when any action - is called, it does its showing but then just returns zero - instead of actually calling the action execution operation. - The problem for caching is that if the file does NOT exist in - cache then the CacheRetrieveString won't return anything to - show for the task, but the Action.__call__ won't call - CacheRetrieveFunc; instead it just returns zero, which makes - the code below think that the file *was* successfully - retrieved from the cache, therefore it doesn't do any - subsequent building. However, the CacheRetrieveString didn't - print anything because it didn't actually exist in the cache, - and no more build actions will be performed, so the user just - sees nothing. The fix is to tell Action.__call__ to always - execute the CacheRetrieveFunc and then have the latter - explicitly check SCons.Action.execute_actions itself. - Returns true iff the node was successfully retrieved. """ if self.nocache: return None - b = self.is_derived() - if not b and not self.has_src_builder(): + if not self.is_derived(): return None - - retrieved = None - if b and self.fs.CachePath: - if self.fs.cache_show: - if CacheRetrieveSilent(self, [], None, execute=1) == 0: - self.build(presub=0, execute=0) - retrieved = 1 - else: - if CacheRetrieve(self, [], None, execute=1) == 0: - retrieved = 1 - if retrieved: - # Record build signature information, but don't - # push it out to cache. (We just got it from there!) - self.set_state(SCons.Node.executed) - SCons.Node.Node.built(self) - - return retrieved - + return self.get_build_env().get_CacheDir().retrieve(self) def built(self): - """Called just after this node is successfully built. + """ + Called just after this node is successfully built. """ # Push this file out to cache before the superclass Node.built() # method has a chance to clear the build signature, which it @@ -2104,13 +2029,13 @@ class File(Base): # cache so that the memoization of the self.exists() return # value doesn't interfere. self.clear_memoized_values() - if self.fs.CachePath and self.exists(): - CachePush(self, [], None) + if self.exists(): + self.get_build_env().get_CacheDir().push(self) SCons.Node.Node.built(self) def visited(self): - if self.fs.CachePath and self.fs.cache_force and self.exists(): - CachePush(self, None, None) + if self.exists(): + self.get_build_env().get_CacheDir().push_if_forced(self) def has_src_builder(self): """Return whether this Node has a source builder or not. @@ -2333,9 +2258,8 @@ class File(Base): def rstr(self): return str(self.rfile()) - def cachepath(self): - if self.nocache or not self.fs.CachePath: - return None, None + def get_cachedir_bsig(self): + import SCons.Sig.MD5 ninfo = self.get_binfo().ninfo if not hasattr(ninfo, 'bsig'): import SCons.Errors @@ -2346,14 +2270,16 @@ class File(Base): # Add the path to the cache signature, because multiple # targets built by the same action will all have the same # build signature, and we have to differentiate them somehow. - import SCons.Sig.MD5 - cache_sig = SCons.Sig.MD5.collect([ninfo.bsig, self.path]) - subdir = string.upper(cache_sig[0]) - dir = os.path.join(self.fs.CachePath, subdir) - return dir, os.path.join(dir, cache_sig) + return SCons.Sig.MD5.collect([ninfo.bsig, self.path]) default_fs = None +def get_default_fs(): + global default_fs + if not default_fs: + default_fs = FS() + return default_fs + class FileFinder: """ """ diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 0b60fb9..ed8d6ec 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -89,6 +89,7 @@ class Action: pass def strfunction(self, targets, sources, env): return "" + class Builder: def __init__(self, factory, action=Action()): self.factory = factory @@ -2041,6 +2042,74 @@ class RepositoryTestCase(_tempdirTestCase): r = map(lambda x, np=os.path.normpath: np(str(x)), rep) assert r == expect, r + def test_rentry(self): + """Test the Base.entry() method""" + return_true = lambda: 1 + return_false = lambda: 0 + + d1 = self.fs.Dir('d1') + d2 = self.fs.Dir('d2') + d3 = self.fs.Dir('d3') + + e1 = self.fs.Entry('e1') + e2 = self.fs.Entry('e2') + e3 = self.fs.Entry('e3') + + f1 = self.fs.File('f1') + f2 = self.fs.File('f2') + f3 = self.fs.File('f3') + + self.test.write([self.rep1, 'd2'], "") + self.test.subdir([self.rep2, 'd3']) + self.test.write([self.rep3, 'd3'], "") + + self.test.write([self.rep1, 'e2'], "") + self.test.subdir([self.rep2, 'e3']) + self.test.write([self.rep3, 'e3'], "") + + self.test.write([self.rep1, 'f2'], "") + self.test.subdir([self.rep2, 'f3']) + self.test.write([self.rep3, 'f3'], "") + + r = d1.rentry() + assert r is d1, r + + r = d2.rentry() + assert not r is d2, r + r = str(r) + assert r == os.path.join(self.rep1, 'd2'), r + + r = d3.rentry() + assert not r is d3, r + r = str(r) + assert r == os.path.join(self.rep2, 'd3'), r + + r = e1.rentry() + assert r is e1, r + + r = e2.rentry() + assert not r is e2, r + r = str(r) + assert r == os.path.join(self.rep1, 'e2'), r + + r = e3.rentry() + assert not r is e3, r + r = str(r) + assert r == os.path.join(self.rep2, 'e3'), r + + r = f1.rentry() + assert r is f1, r + + r = f2.rentry() + assert not r is f2, r + r = str(r) + assert r == os.path.join(self.rep1, 'f2'), r + + r = f3.rentry() + assert not r is f3, r + r = str(r) + assert r == os.path.join(self.rep2, 'f3'), r + def test_rdir(self): """Test the Dir.rdir() method""" return_true = lambda: 1 @@ -2468,200 +2537,6 @@ class SConstruct_dirTestCase(unittest.TestCase): fs.set_SConstruct_dir(fs.Dir('xxx')) assert fs.SConstruct_dir.path == 'xxx' -class CacheDirTestCase(unittest.TestCase): - def runTest(self): - """Test CacheDir functionality""" - test = TestCmd(workdir='') - - global built_it - - fs = SCons.Node.FS.FS() - assert fs.CachePath is None, fs.CachePath - assert fs.cache_force is None, fs.cache_force - assert fs.cache_show is None, fs.cache_show - - fs.CacheDir('cache') - assert fs.CachePath == 'cache', fs.CachePath - - save_CacheRetrieve = SCons.Node.FS.CacheRetrieve - self.retrieved = [] - def retrieve_succeed(target, source, env, self=self, execute=1): - self.retrieved.append(target) - return 0 - def retrieve_fail(target, source, env, self=self, execute=1): - self.retrieved.append(target) - return 1 - - f1 = fs.File("cd.f1") - f1.builder_set(Builder(fs.File)) - f1.env_set(Environment()) - try: - SCons.Node.FS.CacheRetrieve = retrieve_succeed - self.retrieved = [] - built_it = None - - r = f1.retrieve_from_cache() - assert r == 1, r - assert self.retrieved == [f1], self.retrieved - assert built_it is None, built_it - - SCons.Node.FS.CacheRetrieve = retrieve_fail - self.retrieved = [] - built_it = None - - r = f1.retrieve_from_cache() - assert r is None, r - assert self.retrieved == [f1], self.retrieved - assert built_it is None, built_it - finally: - SCons.Node.FS.CacheRetrieve = save_CacheRetrieve - - save_CacheRetrieveSilent = SCons.Node.FS.CacheRetrieveSilent - - fs.cache_show = 1 - - f2 = fs.File("cd.f2") - f2.builder_set(Builder(fs.File)) - f2.env_set(Environment()) - try: - SCons.Node.FS.CacheRetrieveSilent = retrieve_succeed - self.retrieved = [] - built_it = None - - r = f2.retrieve_from_cache() - assert r == 1, r - assert self.retrieved == [f2], self.retrieved - assert built_it is None, built_it - - SCons.Node.FS.CacheRetrieveSilent = retrieve_fail - self.retrieved = [] - built_it = None - - r = f2.retrieve_from_cache() - assert r is None, r - assert self.retrieved == [f2], self.retrieved - assert built_it is None, built_it - finally: - SCons.Node.FS.CacheRetrieveSilent = save_CacheRetrieveSilent - - save_CachePush = SCons.Node.FS.CachePush - def push(target, source, env, self=self): - self.pushed.append(target) - return 0 - SCons.Node.FS.CachePush = push - - try: - self.pushed = [] - - cd_f3 = test.workpath("cd.f3") - f3 = fs.File(cd_f3) - f3.built() - assert self.pushed == [], self.pushed - test.write(cd_f3, "cd.f3\n") - f3.built() - assert self.pushed == [f3], self.pushed - - self.pushed = [] - - cd_f4 = test.workpath("cd.f4") - f4 = fs.File(cd_f4) - f4.visited() - assert self.pushed == [], self.pushed - test.write(cd_f4, "cd.f4\n") - f4.visited() - assert self.pushed == [], self.pushed - fs.cache_force = 1 - f4.visited() - assert self.pushed == [f4], self.pushed - finally: - SCons.Node.FS.CachePush = save_CachePush - - # Verify how the cachepath() method determines the name - # of the file in cache. - def my_collect(list): - return list[0] - save_collect = SCons.Sig.MD5.collect - SCons.Sig.MD5.collect = my_collect - try: - f5 = fs.File("cd.f5") - f5.binfo = f5.BuildInfo(f5) - f5.binfo.ninfo.bsig = 'a_fake_bsig' - cp = f5.cachepath() - dirname = os.path.join('cache', 'A') - filename = os.path.join(dirname, 'a_fake_bsig') - assert cp == (dirname, filename), cp - finally: - SCons.Sig.MD5.collect = save_collect - - # Verify that no bsig raises an InternalERror - f6 = fs.File("cd.f6") - f6.binfo = f6.BuildInfo(f6) - exc_caught = 0 - try: - cp = f6.cachepath() - except SCons.Errors.InternalError: - exc_caught = 1 - assert exc_caught - - # Verify that we raise a warning if we can't copy a file to cache. - save_copy2 = shutil.copy2 - def copy2(src, dst): - raise OSError - shutil.copy2 = copy2 - save_mkdir = os.mkdir - def mkdir(dir, mode=0): - pass - os.mkdir = mkdir - old_warn_exceptions = SCons.Warnings.warningAsException(1) - SCons.Warnings.enableWarningClass(SCons.Warnings.CacheWriteErrorWarning) - - try: - cd_f7 = test.workpath("cd.f7") - test.write(cd_f7, "cd.f7\n") - f7 = fs.File(cd_f7) - f7.binfo = f7.BuildInfo(f7) - f7.binfo.ninfo.bsig = 'f7_bsig' - - warn_caught = 0 - try: - f7.built() - except SCons.Warnings.CacheWriteErrorWarning: - warn_caught = 1 - assert warn_caught - finally: - shutil.copy2 = save_copy2 - os.mkdir = save_mkdir - SCons.Warnings.warningAsException(old_warn_exceptions) - SCons.Warnings.suppressWarningClass(SCons.Warnings.CacheWriteErrorWarning) - - # Verify that we don't blow up if there's no strfunction() - # for an action. - act = Action() - act.strfunction = None - f8 = fs.File("cd.f8") - f8.builder_set(Builder(fs.File, action=act)) - f8.env_set(Environment()) - try: - SCons.Node.FS.CacheRetrieveSilent = retrieve_succeed - self.retrieved = [] - built_it = None - - r = f8.retrieve_from_cache() - assert r == 1, r - assert self.retrieved == [f8], self.retrieved - assert built_it is None, built_it - - SCons.Node.FS.CacheRetrieveSilent = retrieve_fail - self.retrieved = [] - built_it = None - - r = f8.retrieve_from_cache() - assert r is None, r - assert self.retrieved == [f8], self.retrieved - assert built_it is None, built_it - finally: - SCons.Node.FS.CacheRetrieveSilent = save_CacheRetrieveSilent - class clearTestCase(unittest.TestCase): def runTest(self): """Test clearing FS nodes of cached data.""" @@ -3007,7 +2882,6 @@ if __name__ == "__main__": suite.addTest(has_src_builderTestCase()) suite.addTest(prepareTestCase()) suite.addTest(SConstruct_dirTestCase()) - suite.addTest(CacheDirTestCase()) suite.addTest(clearTestCase()) suite.addTest(disambiguateTestCase()) suite.addTest(postprocessTestCase()) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index ad9eb66..09ab5c7 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -33,6 +33,7 @@ import UserList import SCons.Errors import SCons.Node +import SCons.Util @@ -977,7 +978,7 @@ class NodeTestCase(unittest.TestCase): target = SCons.Node.Node() source = SCons.Node.Node() s = target.get_source_scanner(source) - assert s is None, s + assert isinstance(s, SCons.Util.Null), s ts1 = Scanner() ts2 = Scanner() |