From a88dd02a03343dd0a0e17f6ad589148bf77d0493 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 30 Jan 2016 22:35:38 +0000 Subject: Change the cache to use the first two characters of the md5 for the directory name (more smaller directories, because they tend to get huge otherwise) --- src/engine/SCons/CacheDir.py | 2 +- src/engine/SCons/CacheDirTests.py | 2 +- src/script/scons-rename-cachedirs.py | 62 ++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 src/script/scons-rename-cachedirs.py diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index 3b7d06f..be0163d 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -164,7 +164,7 @@ class CacheDir(object): return None, None sig = node.get_cachedir_bsig() - subdir = sig[0].upper() + subdir = sig[:2].upper() dir = os.path.join(self.path, subdir) return dir, os.path.join(dir, sig) diff --git a/src/engine/SCons/CacheDirTests.py b/src/engine/SCons/CacheDirTests.py index 7ac97ef..3b99d41 100644 --- a/src/engine/SCons/CacheDirTests.py +++ b/src/engine/SCons/CacheDirTests.py @@ -100,7 +100,7 @@ class CacheDirTestCase(BaseTestCase): try: f5 = self.File("cd.f5", 'a_fake_bsig') result = self._CacheDir.cachepath(f5) - dirname = os.path.join('cache', 'A') + dirname = os.path.join('cache', 'A_') filename = os.path.join(dirname, 'a_fake_bsig') assert result == (dirname, filename), result finally: diff --git a/src/script/scons-rename-cachedirs.py b/src/script/scons-rename-cachedirs.py new file mode 100644 index 0000000..dd76a42 --- /dev/null +++ b/src/script/scons-rename-cachedirs.py @@ -0,0 +1,62 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + +import glob +import os + +# The entire purpose of this script is to rename the files in the specified +# cache directory from the 16 single hex digit directories to 256 2 hex digit +# directories. + +# You run this in the cache directory. +expected = ['{:X}'.format(x) for x in range(0, 16)] +# check there are 16 directories, 0 - 9, A - F +if sorted(glob.glob('*')) != [x]: + raise RuntimeError("This doesn't look like a cache directory") +dirs = set() +for file in glob.iglob(os.path.join('*', '*')): + name = os.path.basename(file) + dir = name[:2].upper() + print dir, name + if dir not in dirs: + os.mkdir(dir) + dirs.add(dir) + os.rename(file, os.path.join(dir, name)) + + # Now delete the original directories + for dir in expected: + os.rmdir(dir) \ No newline at end of file -- cgit v0.12 From 7f5d85d7f4428fb2fb7271952a00c87dc8c66533 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 30 Jan 2016 22:39:17 +0000 Subject: Fix for backslash being treated as an escape character On my windows system, my python is in c:\apps\32\python. Theres a lot of places where that \32 gets turned into an ascii character and the unit tests don't run. --- QMTest/TestCmd.py | 1 + test/Actions/pre-post.py | 2 +- test/Builder/multi/different-environments.py | 2 +- test/Builder/multi/same-overrides.py | 2 +- test/Builder/srcdir.py | 2 +- test/CacheDir/option--cs.py | 3 ++- test/Command.py | 6 +++--- test/Configure/Builder-call.py | 2 +- test/Configure/custom-tests.py | 4 ++-- test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOM.py | 2 +- test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOMSTR.py | 2 +- test/Deprecated/SourceCode/CVS/CVSCOM.py | 2 +- test/Deprecated/SourceCode/CVS/CVSCOMSTR.py | 2 +- test/Deprecated/SourceCode/Perforce/P4COM.py | 2 +- test/Deprecated/SourceCode/Perforce/P4COMSTR.py | 2 +- test/Deprecated/SourceCode/RCS/RCS_COCOM.py | 2 +- test/Deprecated/SourceCode/RCS/RCS_COCOMSTR.py | 2 +- test/Deprecated/SourceCode/SCCS/SCCSCOM.py | 2 +- test/Deprecated/SourceCode/SCCS/SCCSCOMSTR.py | 2 +- test/ESCAPE.py | 2 +- test/Execute.py | 16 ++++++++-------- test/MSVC/batch.py | 4 ++-- test/Parallel/duplicate-children.py | 4 ++-- test/SConsignFile/default.py | 2 +- test/SConsignFile/explicit-file.py | 2 +- test/SConsignFile/use-dbhash.py | 2 +- test/SConsignFile/use-dumbdbm.py | 2 +- test/ignore-command.py | 14 +++++++------- test/option/debug-findlibs.py | 2 +- test/redirection.py | 8 ++++---- test/silent-command.py | 14 +++++++------- test/special-filenames.py | 2 +- test/srcchange.py | 2 +- test/subdir.py | 2 +- 34 files changed, 62 insertions(+), 60 deletions(-) diff --git a/QMTest/TestCmd.py b/QMTest/TestCmd.py index cd559b7..547522e 100644 --- a/QMTest/TestCmd.py +++ b/QMTest/TestCmd.py @@ -328,6 +328,7 @@ __all__ = [ 'match_re_dotall', 'python', '_python_', +# '_esc_python_', 'TestCmd' ] diff --git a/test/Actions/pre-post.py b/test/Actions/pre-post.py index e09ee02..38a55df 100644 --- a/test/Actions/pre-post.py +++ b/test/Actions/pre-post.py @@ -188,7 +188,7 @@ def post_action(target, source, env): env = Environment() o = env.Command(['pre-post', 'file.out'], 'file.in', - '%(_python_)s build.py ${TARGETS[1]} $SOURCE') + r'%(_python_)s build.py ${TARGETS[1]} $SOURCE') env.AddPreAction(o, pre_action) env.AddPostAction(o, post_action) """ % locals()) diff --git a/test/Builder/multi/different-environments.py b/test/Builder/multi/different-environments.py index 783b7f9..c3e96c2 100644 --- a/test/Builder/multi/different-environments.py +++ b/test/Builder/multi/different-environments.py @@ -47,7 +47,7 @@ build(sys.argv[1],sys.argv[2],sys.argv[3:]) """) test.write('SConstruct', """\ -B = Builder(action='%(_python_)s build.py $foo $TARGET $SOURCES', multi=1) +B = Builder(action=r'%(_python_)s build.py $foo $TARGET $SOURCES', multi=1) env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'file03.out', source = 'file03a.in', foo=1) env.B(target = 'file03.out', source = 'file03b.in', foo=2) diff --git a/test/Builder/multi/same-overrides.py b/test/Builder/multi/same-overrides.py index 33c4e7b..ee169e4 100644 --- a/test/Builder/multi/same-overrides.py +++ b/test/Builder/multi/same-overrides.py @@ -45,7 +45,7 @@ build(sys.argv[1],sys.argv[2],sys.argv[3:]) """) test.write('SConstruct', """\ -B = Builder(action='%(_python_)s build.py $foo $TARGET $SOURCES', multi=1) +B = Builder(action=r'%(_python_)s build.py $foo $TARGET $SOURCES', multi=1) env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'file4.out', source = 'file4a.in', foo=3) env.B(target = 'file4.out', source = 'file4b.in', foo=3) diff --git a/test/Builder/srcdir.py b/test/Builder/srcdir.py index 63732d7..669c5e1 100644 --- a/test/Builder/srcdir.py +++ b/test/Builder/srcdir.py @@ -50,7 +50,7 @@ o.close() test.write(['src', 'SConstruct'], """\ Command('output', ['file1', File('file2'), r'%(file3)s', 'file4'], - '%(_python_)s cat.py $TARGET $SOURCES', + r'%(_python_)s cat.py $TARGET $SOURCES', srcdir='foo') """ % locals()) diff --git a/test/CacheDir/option--cs.py b/test/CacheDir/option--cs.py index b6bb52a..6d22814 100644 --- a/test/CacheDir/option--cs.py +++ b/test/CacheDir/option--cs.py @@ -35,6 +35,7 @@ import shutil import TestSCons _python_ = TestSCons._python_ + _exe = TestSCons._exe _obj = TestSCons._obj @@ -62,7 +63,7 @@ def cat(env, source, target): f.write(open(str(src), "rb").read()) f.close() env = Environment(BUILDERS={'Internal':Builder(action=cat), - 'External':Builder(action='%(_python_)s build.py $TARGET $SOURCES')}) + 'External':Builder(action=r'%(_python_)s build.py $TARGET $SOURCES')}) env.External('aaa.out', 'aaa.in') env.External('bbb.out', 'bbb.in') env.Internal('ccc.out', 'ccc.in') diff --git a/test/Command.py b/test/Command.py index 74046b1..5f72c94 100644 --- a/test/Command.py +++ b/test/Command.py @@ -63,15 +63,15 @@ def sub(env, target, source): t.close() return 0 -env = Environment(COPY_THROUGH_TEMP = '%(_python_)s build.py .tmp $SOURCE\\n%(_python_)s build.py $TARGET .tmp', +env = Environment(COPY_THROUGH_TEMP = r'%(_python_)s build.py .tmp $SOURCE' + '\\n' + r'%(_python_)s build.py $TARGET .tmp', EXPAND = '$COPY_THROUGH_TEMP') env.Command(target = 'f1.out', source = 'f1.in', action = buildIt) env.Command(target = 'f2.out', source = 'f2.in', action = r'%(_python_)s build.py temp2 $SOURCES' + '\\n' + r'%(_python_)s build.py $TARGET temp2') env.Command(target = 'f3.out', source = 'f3.in', - action = [ [ r'%(python)s', 'build.py', 'temp3', '$SOURCES' ], - [ r'%(python)s', 'build.py', '$TARGET', 'temp3'] ]) + action = [ [ r'%(_python_)s', 'build.py', 'temp3', '$SOURCES' ], + [ r'%(_python_)s', 'build.py', '$TARGET', 'temp3'] ]) Command(target = 'f4.out', source = 'sub', action = sub) env.Command(target = 'f5.out', source = 'f5.in', action = buildIt, XYZZY='XYZZY is set') diff --git a/test/Configure/Builder-call.py b/test/Configure/Builder-call.py index 037a2c7..b85b039 100644 --- a/test/Configure/Builder-call.py +++ b/test/Configure/Builder-call.py @@ -48,7 +48,7 @@ def CustomTest(*args): return 0 conf = env.Configure(custom_tests = {'MyTest' : CustomTest}) if not conf.MyTest(): - env.Command("hello", [], '%(_python_)s mycommand.py $TARGET') + env.Command("hello", [], r'%(_python_)s mycommand.py $TARGET') env = conf.Finish() """ % locals()) diff --git a/test/Configure/custom-tests.py b/test/Configure/custom-tests.py index 687ba48..503eb4e 100644 --- a/test/Configure/custom-tests.py +++ b/test/Configure/custom-tests.py @@ -63,8 +63,8 @@ def CheckCustom(test): retLinkFAIL = test.TryLink( '%(linkFAIL)s', '.c' ) (retRunOK, outputRunOK) = test.TryRun( '%(runOK)s', '.c' ) (retRunFAIL, outputRunFAIL) = test.TryRun( '%(runFAIL)s', '.c' ) - (retActOK, outputActOK) = test.TryAction( '%(_python_)s pyAct.py 0 > $TARGET' ) - (retActFAIL, outputActFAIL) = test.TryAction( '%(_python_)s pyAct.py 1 > $TARGET' ) + (retActOK, outputActOK) = test.TryAction( r'%(_python_)s pyAct.py 0 > $TARGET' ) + (retActFAIL, outputActFAIL) = test.TryAction( r'%(_python_)s pyAct.py 1 > $TARGET' ) resOK = retCompileOK and retLinkOK and retRunOK and outputRunOK=="Hello" resOK = resOK and retActOK and int(outputActOK)==0 resFAIL = retCompileFAIL or retLinkFAIL or retRunFAIL or outputRunFAIL!="" diff --git a/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOM.py b/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOM.py index d5af0ea..8ff4ced 100644 --- a/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOM.py +++ b/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOM.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'BitKeeper'], BUILDERS={'Cat':Builder(action=cat)}, - BITKEEPERCOM='%(_python_)s my-bk-get.py $TARGET') + BITKEEPERCOM=r'%(_python_)s my-bk-get.py $TARGET') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') env.Cat('ccc.out', 'ccc.in') diff --git a/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOMSTR.py b/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOMSTR.py index ef70cb8..5bbdea2 100644 --- a/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOMSTR.py +++ b/test/Deprecated/SourceCode/BitKeeper/BITKEEPERCOMSTR.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(tools = ['default', 'BitKeeper'], BUILDERS={'Cat':Builder(action=cat)}, - BITKEEPERCOM='%(_python_)s my-bk-get.py $TARGET', + BITKEEPERCOM=r'%(_python_)s my-bk-get.py $TARGET', BITKEEPERCOMSTR='Checking out $TARGET from our fake BitKeeper') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/Deprecated/SourceCode/CVS/CVSCOM.py b/test/Deprecated/SourceCode/CVS/CVSCOM.py index a0f8400..f3bd29d 100644 --- a/test/Deprecated/SourceCode/CVS/CVSCOM.py +++ b/test/Deprecated/SourceCode/CVS/CVSCOM.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'CVS'], BUILDERS={'Cat':Builder(action=cat)}, - CVSCOM='%(_python_)s my-cvs-co-.py $TARGET') + CVSCOM=r'%(_python_)s my-cvs-co-.py $TARGET') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') env.Cat('ccc.out', 'ccc.in') diff --git a/test/Deprecated/SourceCode/CVS/CVSCOMSTR.py b/test/Deprecated/SourceCode/CVS/CVSCOMSTR.py index f793d66..483a0ae 100644 --- a/test/Deprecated/SourceCode/CVS/CVSCOMSTR.py +++ b/test/Deprecated/SourceCode/CVS/CVSCOMSTR.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'CVS'], BUILDERS={'Cat':Builder(action=cat)}, - CVSCOM='%(_python_)s my-cvs-co.py $TARGET', + CVSCOM=r'%(_python_)s my-cvs-co.py $TARGET', CVSCOMSTR='Checking out $TARGET from our fake CVS') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/Deprecated/SourceCode/Perforce/P4COM.py b/test/Deprecated/SourceCode/Perforce/P4COM.py index 9b9bab1..e58cb60 100644 --- a/test/Deprecated/SourceCode/Perforce/P4COM.py +++ b/test/Deprecated/SourceCode/Perforce/P4COM.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'Perforce'], BUILDERS={'Cat':Builder(action=cat)}, - P4COM='%(_python_)s my-p4.py $TARGET') + P4COM=r'%(_python_)s my-p4.py $TARGET') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') env.Cat('ccc.out', 'ccc.in') diff --git a/test/Deprecated/SourceCode/Perforce/P4COMSTR.py b/test/Deprecated/SourceCode/Perforce/P4COMSTR.py index 7a24021..7b2fbba 100644 --- a/test/Deprecated/SourceCode/Perforce/P4COMSTR.py +++ b/test/Deprecated/SourceCode/Perforce/P4COMSTR.py @@ -75,7 +75,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'Perforce'], BUILDERS={'Cat':Builder(action=cat)}, - P4COM='%(_python_)s my-p4.py $TARGET', + P4COM=r'%(_python_)s my-p4.py $TARGET', P4COMSTR='Checking out $TARGET from our fake Perforce') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/Deprecated/SourceCode/RCS/RCS_COCOM.py b/test/Deprecated/SourceCode/RCS/RCS_COCOM.py index 765c88c..c5934ac 100644 --- a/test/Deprecated/SourceCode/RCS/RCS_COCOM.py +++ b/test/Deprecated/SourceCode/RCS/RCS_COCOM.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'RCS'], BUILDERS={'Cat':Builder(action=cat)}, - RCS_COCOM='%(_python_)s my-rcs-co.py $TARGET') + RCS_COCOM=r'%(_python_)s my-rcs-co.py $TARGET') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') env.Cat('ccc.out', 'ccc.in') diff --git a/test/Deprecated/SourceCode/RCS/RCS_COCOMSTR.py b/test/Deprecated/SourceCode/RCS/RCS_COCOMSTR.py index 57088fa..cb54202 100644 --- a/test/Deprecated/SourceCode/RCS/RCS_COCOMSTR.py +++ b/test/Deprecated/SourceCode/RCS/RCS_COCOMSTR.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'RCS'], BUILDERS={'Cat':Builder(action=cat)}, - RCS_COCOM='%(_python_)s my-rcs-co.py $TARGET', + RCS_COCOM=r'%(_python_)s my-rcs-co.py $TARGET', RCS_COCOMSTR='Checking out $TARGET from our fake RCS') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/Deprecated/SourceCode/SCCS/SCCSCOM.py b/test/Deprecated/SourceCode/SCCS/SCCSCOM.py index 6ad02ac..ba89d87 100644 --- a/test/Deprecated/SourceCode/SCCS/SCCSCOM.py +++ b/test/Deprecated/SourceCode/SCCS/SCCSCOM.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(TOOLS = ['default', 'SCCS'], BUILDERS={'Cat':Builder(action=cat)}, - SCCSCOM='%(_python_)s my-sccs-get.py $TARGET') + SCCSCOM=r'%(_python_)s my-sccs-get.py $TARGET') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') env.Cat('ccc.out', 'ccc.in') diff --git a/test/Deprecated/SourceCode/SCCS/SCCSCOMSTR.py b/test/Deprecated/SourceCode/SCCS/SCCSCOMSTR.py index a757495..2351c05 100644 --- a/test/Deprecated/SourceCode/SCCS/SCCSCOMSTR.py +++ b/test/Deprecated/SourceCode/SCCS/SCCSCOMSTR.py @@ -74,7 +74,7 @@ def cat(env, source, target): f.close() env = Environment(tools = ['default', 'SCCS'], BUILDERS={'Cat':Builder(action=cat)}, - SCCSCOM='%(_python_)s my-sccs-get.py $TARGET', + SCCSCOM=r'%(_python_)s my-sccs-get.py $TARGET', SCCSCOMSTR='Checking out $TARGET from our fake SCCS') env.Cat('aaa.out', 'aaa.in') env.Cat('bbb.out', 'bbb.in') diff --git a/test/ESCAPE.py b/test/ESCAPE.py index 0651b92..819d60b 100644 --- a/test/ESCAPE.py +++ b/test/ESCAPE.py @@ -51,7 +51,7 @@ def my_escape(s): s = s.replace('file.in', 'file.xxx') return orig_escape(s) env = Environment(ESCAPE = my_escape) -env.Command('file.out', 'file.in', '%(_python_)s cat.py $TARGET $SOURCES') +env.Command('file.out', 'file.in', r'%(_python_)s cat.py $TARGET $SOURCES') """ % locals()) test.write('file.in', "file.in\n") diff --git a/test/Execute.py b/test/Execute.py index 2e53444..e63cad5 100644 --- a/test/Execute.py +++ b/test/Execute.py @@ -45,18 +45,18 @@ sys.exit(exitval) """) test.write('SConstruct', """\ -Execute('%(_python_)s my_copy.py a.in a.out') -Execute(Action('%(_python_)s my_copy.py b.in b.out')) +Execute(r'%(_python_)s my_copy.py a.in a.out') +Execute(Action(r'%(_python_)s my_copy.py b.in b.out')) env = Environment(COPY = 'my_copy.py') -env.Execute('%(_python_)s my_copy.py c.in c.out') -env.Execute(Action('%(_python_)s my_copy.py d.in d.out')) -v = env.Execute('%(_python_)s $COPY e.in e.out') +env.Execute(r'%(_python_)s my_copy.py c.in c.out') +env.Execute(Action(r'%(_python_)s my_copy.py d.in d.out')) +v = env.Execute(r'%(_python_)s $COPY e.in e.out') assert v == 0, v -v = env.Execute(Action('%(_python_)s $COPY f.in f.out')) +v = env.Execute(Action(r'%(_python_)s $COPY f.in f.out')) assert v == 0, v -v = env.Execute('%(_python_)s $COPY g.in g.out 1') +v = env.Execute(r'%(_python_)s $COPY g.in g.out 1') assert v == 1, v -v = env.Execute(Action('%(_python_)s $COPY h.in h.out 2')) +v = env.Execute(Action(r'%(_python_)s $COPY h.in h.out 2')) assert v == 2, v import shutil Execute(lambda target, source, env: shutil.copy('i.in', 'i.out')) diff --git a/test/MSVC/batch.py b/test/MSVC/batch.py index fbb3218..8648292 100644 --- a/test/MSVC/batch.py +++ b/test/MSVC/batch.py @@ -72,8 +72,8 @@ for infile in sys.argv[2:]: """) test.write('SConstruct', """ -cccom = '%(_python_)s fake_cl.py $_MSVC_OUTPUT_FLAG $CHANGED_SOURCES' -linkcom = '%(_python_)s fake_link.py ${TARGET.windows} $SOURCES' +cccom = r'%(_python_)s fake_cl.py $_MSVC_OUTPUT_FLAG $CHANGED_SOURCES' +linkcom = r'%(_python_)s fake_link.py ${TARGET.windows} $SOURCES' env = Environment(tools=['msvc', 'mslink'], CCCOM=cccom, LINKCOM=linkcom, diff --git a/test/Parallel/duplicate-children.py b/test/Parallel/duplicate-children.py index c6c5c29..8b8f4cd 100644 --- a/test/Parallel/duplicate-children.py +++ b/test/Parallel/duplicate-children.py @@ -53,8 +53,8 @@ test.write('SConstruct', """ # Test case for SCons issue #1608 # Create a file "foo.in" in the current directory before running scons. env = Environment() -env.Command('foo.out', ['foo.in'], '%(_python_)s cat.py $TARGET $SOURCE && %(_python_)s sleep.py 3') -env.Command('foobar', ['foo.out'], '%(_python_)s cat.py $TARGET $SOURCES') +env.Command('foo.out', ['foo.in'], r'%(_python_)s cat.py $TARGET $SOURCE && %(_python_)s sleep.py 3') +env.Command('foobar', ['foo.out'], r'%(_python_)s cat.py $TARGET $SOURCES') env.Depends('foobar', 'foo.out') """ % locals()) diff --git a/test/SConsignFile/default.py b/test/SConsignFile/default.py index f38aec8..4d56a5e 100644 --- a/test/SConsignFile/default.py +++ b/test/SConsignFile/default.py @@ -49,7 +49,7 @@ sys.exit(0) # test.write('SConstruct', """ SConsignFile() -B = Builder(action = '%(_python_)s build.py $TARGETS $SOURCES') +B = Builder(action = r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'f1.out', source = 'f1.in') env.B(target = 'f2.out', source = 'f2.in') diff --git a/test/SConsignFile/explicit-file.py b/test/SConsignFile/explicit-file.py index 33963c2..90c2241 100644 --- a/test/SConsignFile/explicit-file.py +++ b/test/SConsignFile/explicit-file.py @@ -49,7 +49,7 @@ file.close() test.write('SConstruct', """ e = Environment(XXX = 'scons') e.SConsignFile('my_${XXX}ign') -B = Builder(action = '%(_python_)s build.py $TARGETS $SOURCES') +B = Builder(action = r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'f5.out', source = 'f5.in') env.B(target = 'f6.out', source = 'f6.in') diff --git a/test/SConsignFile/use-dbhash.py b/test/SConsignFile/use-dbhash.py index 45e3e36..bf9868b 100644 --- a/test/SConsignFile/use-dbhash.py +++ b/test/SConsignFile/use-dbhash.py @@ -55,7 +55,7 @@ test.write('SConstruct', """ import sys import dbhash SConsignFile('.sconsign', dbhash) -B = Builder(action = '%(_python_)s build.py $TARGETS $SOURCES') +B = Builder(action = r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'f1.out', source = 'f1.in') env.B(target = 'f2.out', source = 'f2.in') diff --git a/test/SConsignFile/use-dumbdbm.py b/test/SConsignFile/use-dumbdbm.py index 9d48fe5..434435a 100644 --- a/test/SConsignFile/use-dumbdbm.py +++ b/test/SConsignFile/use-dumbdbm.py @@ -55,7 +55,7 @@ test.write('SConstruct', """ import sys import dumbdbm SConsignFile('.sconsign', dumbdbm) -B = Builder(action = '%(_python_)s build.py $TARGETS $SOURCES') +B = Builder(action = r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'f1.out', source = 'f1.in') env.B(target = 'f2.out', source = 'f2.in') diff --git a/test/ignore-command.py b/test/ignore-command.py index 1343145..9fd24ab 100644 --- a/test/ignore-command.py +++ b/test/ignore-command.py @@ -49,13 +49,13 @@ sys.exit(1) test.write('SConstruct', """\ env = Environment() -f1 = env.Command('f1.out', 'f1.in', '%(_python_)s build.py $TARGET $SOURCE') -f2 = env.Command('f2.out', 'f2.in', '-%(_python_)s build.py $TARGET $SOURCE') -f3 = env.Command('f3.out', 'f3.in', '- %(_python_)s build.py $TARGET $SOURCE') -f4 = env.Command('f4.out', 'f4.in', '@-%(_python_)s build.py $TARGET $SOURCE') -f5 = env.Command('f5.out', 'f5.in', '@- %(_python_)s build.py $TARGET $SOURCE') -f6 = env.Command('f6.out', 'f6.in', '-@%(_python_)s build.py $TARGET $SOURCE') -f7 = env.Command('f7.out', 'f7.in', '-@ %(_python_)s build.py $TARGET $SOURCE') +f1 = env.Command('f1.out', 'f1.in', r'%(_python_)s build.py $TARGET $SOURCE') +f2 = env.Command('f2.out', 'f2.in', r'-%(_python_)s build.py $TARGET $SOURCE') +f3 = env.Command('f3.out', 'f3.in', r'- %(_python_)s build.py $TARGET $SOURCE') +f4 = env.Command('f4.out', 'f4.in', r'@-%(_python_)s build.py $TARGET $SOURCE') +f5 = env.Command('f5.out', 'f5.in', r'@- %(_python_)s build.py $TARGET $SOURCE') +f6 = env.Command('f6.out', 'f6.in', r'-@%(_python_)s build.py $TARGET $SOURCE') +f7 = env.Command('f7.out', 'f7.in', r'-@ %(_python_)s build.py $TARGET $SOURCE') Default(f2, f3, f4, f5, f6, f7) """ % locals()) diff --git a/test/option/debug-findlibs.py b/test/option/debug-findlibs.py index ce97199..9d5c82a 100644 --- a/test/option/debug-findlibs.py +++ b/test/option/debug-findlibs.py @@ -47,7 +47,7 @@ env = Environment(OBJSUFFIX = '.ooo', PROGSUFFIX = '.xxx', LIBS = ['iii', 'jjj', 'kkk', 'lll', 'mmm'], LIBPREFIXES = ['a-', 'b-', 'c-'], LIBSUFFIXES = ['.aaa', '.bbb', '.ccc'], - LINKCOM = '%(_python_)s cat.py $TARGET $SOURCES') + LINKCOM = r'%(_python_)s cat.py $TARGET $SOURCES') env.Program('foo', 'a.ooo',) """ % locals()) diff --git a/test/redirection.py b/test/redirection.py index 55ffeae..98bfc04 100644 --- a/test/redirection.py +++ b/test/redirection.py @@ -43,13 +43,13 @@ sys.exit(0) test.write('SConstruct', r""" env = Environment() env.Command(target='foo1', source='bar1', - action= '%(_python_)s cat.py $SOURCES > $TARGET') + action= r'%(_python_)s cat.py $SOURCES > $TARGET') env.Command(target='foo2', source='bar2', - action= '%(_python_)s cat.py < $SOURCES > $TARGET') + action= r'%(_python_)s cat.py < $SOURCES > $TARGET') env.Command(target='foo3', source='bar3', - action='%(_python_)s cat.py $SOURCES | %(_python_)s cat.py > $TARGET') + action=r'%(_python_)s cat.py $SOURCES | %(_python_)s cat.py > $TARGET') env.Command(target='foo4', source='bar4', - action='%(_python_)s cat.py <$SOURCES |%(_python_)s cat.py >$TARGET') + action=r'%(_python_)s cat.py <$SOURCES |%(_python_)s cat.py >$TARGET') """ % locals()) test.write('bar1', 'bar1\r\n') diff --git a/test/silent-command.py b/test/silent-command.py index 7704ee5..b0e4a2e 100644 --- a/test/silent-command.py +++ b/test/silent-command.py @@ -48,13 +48,13 @@ fp.close() test.write('SConstruct', """\ env = Environment() -env.Command('f1.out', 'f1.in', '%(_python_)s build.py $TARGET $SOURCE') -env.Command('f2.out', 'f2.in', '@%(_python_)s build.py $TARGET $SOURCE') -env.Command('f3.out', 'f3.in', '@ %(_python_)s build.py $TARGET $SOURCE') -env.Command('f4.out', 'f4.in', '@-%(_python_)s build.py $TARGET $SOURCE') -env.Command('f5.out', 'f5.in', '@- %(_python_)s build.py $TARGET $SOURCE') -env.Command('f6.out', 'f6.in', '-@%(_python_)s build.py $TARGET $SOURCE') -env.Command('f7.out', 'f7.in', '-@ %(_python_)s build.py $TARGET $SOURCE') +env.Command('f1.out', 'f1.in', r'%(_python_)s build.py $TARGET $SOURCE') +env.Command('f2.out', 'f2.in', r'@%(_python_)s build.py $TARGET $SOURCE') +env.Command('f3.out', 'f3.in', r'@ %(_python_)s build.py $TARGET $SOURCE') +env.Command('f4.out', 'f4.in', r'@-%(_python_)s build.py $TARGET $SOURCE') +env.Command('f5.out', 'f5.in', r'@- %(_python_)s build.py $TARGET $SOURCE') +env.Command('f6.out', 'f6.in', r'-@%(_python_)s build.py $TARGET $SOURCE') +env.Command('f7.out', 'f7.in', r'-@ %(_python_)s build.py $TARGET $SOURCE') """ % locals()) test.write('f1.in', "f1.in\n") diff --git a/test/special-filenames.py b/test/special-filenames.py index 00e4a3d..de09a52 100644 --- a/test/special-filenames.py +++ b/test/special-filenames.py @@ -69,7 +69,7 @@ def buildFileStr(fn): xxx = '\n'.join(map(buildFileStr, file_names)) test.write("SConstruct", """ -env=Environment(BUILDERS = {'Build' : Builder(action = '%(_python_)s cat.py $TARGET $SOURCE')}) +env=Environment(BUILDERS = {'Build' : Builder(action = r'%(_python_)s cat.py $TARGET $SOURCE')}) %(xxx)s """ % locals()) diff --git a/test/srcchange.py b/test/srcchange.py index d181297..f2fde91 100644 --- a/test/srcchange.py +++ b/test/srcchange.py @@ -63,7 +63,7 @@ SubRevision = Action(subrevision) env=Environment() content_env=env.Clone() -content_env.Command('revision.in', [], '%(_python_)s getrevision > $TARGET') +content_env.Command('revision.in', [], r'%(_python_)s getrevision > $TARGET') content_env.AlwaysBuild('revision.in') env.Precious('main.c') env.Command('main.c', 'revision.in', SubRevision) diff --git a/test/subdir.py b/test/subdir.py index fa10cff..22d0912 100644 --- a/test/subdir.py +++ b/test/subdir.py @@ -42,7 +42,7 @@ file.close() """) test.write('SConstruct', """ -B = Builder(action = '%(_python_)s build.py $TARGETS $SOURCES') +B = Builder(action = r'%(_python_)s build.py $TARGETS $SOURCES') env = Environment(BUILDERS = { 'B' : B }) env.B(target = 'subdir/f1.out', source = 'subdir/f1.in') env.B(target = 'subdir/f2.out', source = 'subdir/f2.in') -- cgit v0.12 From 56db17b7c0f733d1b33e07148f927149263bfc02 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sun, 31 Jan 2016 07:44:47 +0000 Subject: *sigh* Remove commented out code which I couldn't do with an amend --- QMTest/TestCmd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/QMTest/TestCmd.py b/QMTest/TestCmd.py index 547522e..cd559b7 100644 --- a/QMTest/TestCmd.py +++ b/QMTest/TestCmd.py @@ -328,7 +328,6 @@ __all__ = [ 'match_re_dotall', 'python', '_python_', -# '_esc_python_', 'TestCmd' ] -- cgit v0.12 From a014c40490ca3353d82476ca6a1db2ad80ca57fe Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 12 Mar 2016 23:28:47 +0000 Subject: improve behaviour --- src/engine/SCons/CacheDir.py | 43 ++++++++++++++++++++++++++++++------ src/engine/SCons/CacheDirTests.py | 13 ++++++++--- src/script/scons-rename-cachedirs.py | 30 +++++++++++++++++++------ test/CacheDir/CacheDir.py | 3 ++- test/CacheDir/environment.py | 3 ++- 5 files changed, 73 insertions(+), 19 deletions(-) diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index be0163d..728871f 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -27,7 +27,10 @@ __doc__ = """ CacheDir support """ -import os.path +from collections import defaultdict + +import json +import os import stat import sys @@ -72,7 +75,8 @@ CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) def CachePushFunc(target, source, env): - if cache_readonly: return + if cache_readonly: + return t = target[0] if t.nocache: @@ -133,11 +137,36 @@ class CacheDir(object): except ImportError: msg = "No hashlib or MD5 module available, CacheDir() not supported" SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) - self.path = None - else: - self.path = path + path = None + self.path = path self.current_cache_debug = None self.debugFP = None + self.config = defaultdict() + if path is None: + return + # See if there's a config file in the cache directory + config_file = os.path.join(path, 'config') + if not os.path.exists(config_file): + # If the directory exists we're likely version 1, otherwise + # assume we're latest. + # A note: There is a race hazard here, if two processes start and + # attempt to create the cache directory at the same time. However, + # python doesn't really give you the option to do exclusive file + # creation (it doesn't even give you the option to error on opening + # an existing file for writing...). The ordering of events here + # as an attempt to alleviate this, on the basis that it's a pretty + # unlikely occurence (it'd require two builds with a brand new cache + # directory) + if os.path.isdir(path): + self.config['prefix_len'] = 1 + else: + os.makedirs(path) + self.config['prefix_len'] = 2 + if not os.path.exists(config_file): + with open(config_file, 'w') as config: + self.config = json.dump(self.config, config) + with open(config_file) as config: + self.config = json.load(config) def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: @@ -152,7 +181,7 @@ class CacheDir(object): self.debugFP.write(fmt % (target, os.path.split(cachefile)[1])) def is_enabled(self): - return (cache_enabled and not self.path is None) + return cache_enabled and not self.path is None def is_readonly(self): return cache_readonly @@ -164,7 +193,7 @@ class CacheDir(object): return None, None sig = node.get_cachedir_bsig() - subdir = sig[:2].upper() + subdir = sig[:self.config['prefix_len']].upper() dir = os.path.join(self.path, subdir) return dir, os.path.join(dir, sig) diff --git a/src/engine/SCons/CacheDirTests.py b/src/engine/SCons/CacheDirTests.py index 3b99d41..82fba8f 100644 --- a/src/engine/SCons/CacheDirTests.py +++ b/src/engine/SCons/CacheDirTests.py @@ -83,6 +83,11 @@ class BaseTestCase(unittest.TestCase): #node.binfo.ninfo.bsig = bsig return node + def tearDown(self): + os.remove(os.path.join(self._CacheDir.path, 'config')) + os.rmdir(self._CacheDir.path) + # Should that be shutil.rmtree? + class CacheDirTestCase(BaseTestCase): """ Test calling CacheDir code directly. @@ -98,10 +103,12 @@ class CacheDirTestCase(BaseTestCase): SCons.Util.MD5collect = my_collect try: - f5 = self.File("cd.f5", 'a_fake_bsig') + name = 'a_fake_bsig' + f5 = self.File("cd.f5", name) result = self._CacheDir.cachepath(f5) - dirname = os.path.join('cache', 'A_') - filename = os.path.join(dirname, 'a_fake_bsig') + len = self._CacheDir.config['prefix_len'] + dirname = os.path.join('cache', name.upper()[:len]) + filename = os.path.join(dirname, name) assert result == (dirname, filename), result finally: SCons.Util.MD5collect = save_collect diff --git a/src/script/scons-rename-cachedirs.py b/src/script/scons-rename-cachedirs.py index dd76a42..24897d1 100644 --- a/src/script/scons-rename-cachedirs.py +++ b/src/script/scons-rename-cachedirs.py @@ -36,6 +36,7 @@ __date__ = "__DATE__" __developer__ = "__DEVELOPER__" import glob +import json import os # The entire purpose of this script is to rename the files in the specified @@ -43,20 +44,35 @@ import os # directories. # You run this in the cache directory. + expected = ['{:X}'.format(x) for x in range(0, 16)] -# check there are 16 directories, 0 - 9, A - F -if sorted(glob.glob('*')) != [x]: - raise RuntimeError("This doesn't look like a cache directory") + +if not os.path.exists('config'): + # check there are 16 directories, 0 - 9, A - F + if sorted(glob.glob('*')) != expected: + raise RuntimeError("This doesn't look like a (version 1) cache directory") + config = { 'prefix_len' : 1 } +else: + with open('config') as conf: + config = json.load(conf) + if config['prefix_len'] != 1: + raise RuntimeError("This doesn't look like a (version 1) cache directory") + dirs = set() for file in glob.iglob(os.path.join('*', '*')): name = os.path.basename(file) dir = name[:2].upper() - print dir, name if dir not in dirs: os.mkdir(dir) dirs.add(dir) os.rename(file, os.path.join(dir, name)) - # Now delete the original directories - for dir in expected: - os.rmdir(dir) \ No newline at end of file +# Now delete the original directories +for dir in expected: + if os.path.exists(dir): + os.rmdir(dir) + +# and write a config file +config['prefix_len'] = 2 +with open('config', 'w') as conf: + json.dump(config, conf) \ No newline at end of file diff --git a/test/CacheDir/CacheDir.py b/test/CacheDir/CacheDir.py index 9abe0e0..4c05634 100644 --- a/test/CacheDir/CacheDir.py +++ b/test/CacheDir/CacheDir.py @@ -82,7 +82,8 @@ test.must_not_exist(src_aaa_out) test.must_not_exist(src_bbb_out) test.must_not_exist(src_ccc_out) test.must_not_exist(src_all) -test.fail_test(len(os.listdir(cache))) +# Even if you do -n, the cache will be configured. +test.fail_test(os.listdir(cache) != ['config']) # Verify that a normal build works correctly, and clean up. # This should populate the cache with our derived files. diff --git a/test/CacheDir/environment.py b/test/CacheDir/environment.py index 4fb9b51..1378bb2 100644 --- a/test/CacheDir/environment.py +++ b/test/CacheDir/environment.py @@ -85,7 +85,8 @@ test.must_not_exist(src_aaa_out) test.must_not_exist(src_bbb_out) test.must_not_exist(src_ccc_out) test.must_not_exist(src_all) -test.fail_test(len(os.listdir(cache))) +# Even if you do -n, the cache will be configured. +test.fail_test(os.listdir(cache) != ['config']) # Verify that a normal build works correctly, and clean up. # This should populate the cache with our derived files. -- cgit v0.12 From 2a1cc4e5bc95abe0cf43b811ad75bff6015fb45b Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sun, 13 Mar 2016 09:55:01 +0000 Subject: Update to produce warning message. This also means an existing empty cache directory will be treated as v2 (which is probably a good idea...) --- doc/man/scons.xml | 10 ++++++++++ src/CHANGES.txt | 7 +++++++ src/engine/SCons/CacheDir.py | 24 +++++++++++++++++++----- src/engine/SCons/Warnings.py | 4 +++- 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 5c832c2..59ac678 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1707,6 +1707,16 @@ specifies the type of warnings to be enabled or disabled: + --warn=cache-v1, --warn=no-cache-v1 + +Enables or disables warnings about the cache directory being in the +original (v1) layout. +CacheDir(). +These warnings are enabled by default. + + + + --warn=cache-write-error, --warn=no-cache-write-error Enables or disables warnings about errors trying to diff --git a/src/CHANGES.txt b/src/CHANGES.txt index dc8a281..52cff62 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -6,6 +6,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER + From Tom Tanner: + - change cache to use 2 character subdirectories, rather than one character. + For existing caches, you will need to run the scons-rename-cachedirs.py + script to update them to the new format. You will get a warning for this + every time you build. + - Fix a bunch of unit tests on windows + From Dirk Baechle: - Removed a lot of compatibility methods and workarounds for Python versions < 2.7, in order to prepare the work diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index 728871f..5b60bae 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -129,6 +129,10 @@ def CachePushFunc(target, source, env): CachePush = SCons.Action.Action(CachePushFunc, None) +# Nasty hack to cut down to one warning for each cachedir path that needs +# upgrading. +warned = dict() + class CacheDir(object): def __init__(self, path): @@ -147,8 +151,8 @@ class CacheDir(object): # See if there's a config file in the cache directory config_file = os.path.join(path, 'config') if not os.path.exists(config_file): - # If the directory exists we're likely version 1, otherwise - # assume we're latest. + # If the directory exists and is not empty, we're likely version 1. + # # A note: There is a race hazard here, if two processes start and # attempt to create the cache directory at the same time. However, # python doesn't really give you the option to do exclusive file @@ -157,16 +161,26 @@ class CacheDir(object): # as an attempt to alleviate this, on the basis that it's a pretty # unlikely occurence (it'd require two builds with a brand new cache # directory) - if os.path.isdir(path): + if os.path.isdir(path) and len(os.listdir(path)) != 0: self.config['prefix_len'] = 1 - else: - os.makedirs(path) + else: + if not os.path.isdir(path): + os.makedirs(path) self.config['prefix_len'] = 2 if not os.path.exists(config_file): with open(config_file, 'w') as config: self.config = json.dump(self.config, config) with open(config_file) as config: self.config = json.load(config) + # When building the project I was testing this on, this was output + # over 20 times. That seems excessive + global warned + if self.config['prefix_len'] == 1 and self.path not in warned: + msg = "Please update your cache by going into " + self.path +\ + " and running scons-rename-cachedirs.py" + SCons.Warnings.warn(SCons.Warnings.CacheV1Warning, msg) + warned[self.path] = True + def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: diff --git a/src/engine/SCons/Warnings.py b/src/engine/SCons/Warnings.py index 5c27825..16319b9 100644 --- a/src/engine/SCons/Warnings.py +++ b/src/engine/SCons/Warnings.py @@ -41,10 +41,12 @@ class WarningOnByDefault(Warning): # NOTE: If you add a new warning class, add it to the man page, too! - class TargetNotBuiltWarning(Warning): # Should go to OnByDefault pass +class CacheV1Warning(WarningOnByDefault): + pass + class CacheWriteErrorWarning(Warning): pass -- cgit v0.12 From b0e44d9dc69b224e75c2ce755ec1a0b7f5585e49 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 19 Mar 2016 10:59:38 +0000 Subject: Cleanup of code/comments and rename and rewrite of upgrade script --- doc/man/scons.xml | 6 +- src/CHANGES.txt | 10 +-- src/engine/SCons/CacheDir.py | 35 +++++----- src/engine/SCons/Warnings.py | 2 +- src/script/scons-configure-cache.py | 123 +++++++++++++++++++++++++++++++++++ src/script/scons-rename-cachedirs.py | 78 ---------------------- 6 files changed, 151 insertions(+), 103 deletions(-) create mode 100644 src/script/scons-configure-cache.py delete mode 100644 src/script/scons-rename-cachedirs.py diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 59ac678..27cf8b7 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1707,10 +1707,10 @@ specifies the type of warnings to be enabled or disabled: - --warn=cache-v1, --warn=no-cache-v1 + --warn=cache-version, --warn=no-cache-version -Enables or disables warnings about the cache directory being in the -original (v1) layout. +Enables or disables warnings about the cache directory not using +the latest configuration information CacheDir(). These warnings are enabled by default. diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 52cff62..2057199 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -7,10 +7,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Tom Tanner: - - change cache to use 2 character subdirectories, rather than one character. - For existing caches, you will need to run the scons-rename-cachedirs.py - script to update them to the new format. You will get a warning for this - every time you build. + - change cache to use 2 character subdirectories, rather than one character, + so as not to give huge directories for large caches, a situation which + causes issues for NFS. + For existing caches, you will need to run the scons-configure-cache.py + script to update them to the new format. You will get a warning every time + you build until you co this. - Fix a bunch of unit tests on windows From Dirk Baechle: diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index 5b60bae..a941986 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -148,11 +148,12 @@ class CacheDir(object): self.config = defaultdict() if path is None: return - # See if there's a config file in the cache directory + # See if there's a config file in the cache directory. If there is, + # use it. If there isn't, and the directory exists and isn't empty, + # produce a warning. If the directory doesn't exist or is empty, + # write a config file. config_file = os.path.join(path, 'config') if not os.path.exists(config_file): - # If the directory exists and is not empty, we're likely version 1. - # # A note: There is a race hazard here, if two processes start and # attempt to create the cache directory at the same time. However, # python doesn't really give you the option to do exclusive file @@ -163,24 +164,24 @@ class CacheDir(object): # directory) if os.path.isdir(path) and len(os.listdir(path)) != 0: self.config['prefix_len'] = 1 + # When building the project I was testing this on, the warning + # was output over 20 times. That seems excessive + global warned + if self.path not in warned: + msg = "Please upgrade your cache by running " +\ + " scons-upgrade-cache.py " + self.path + SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) + warned[self.path] = True else: if not os.path.isdir(path): os.makedirs(path) self.config['prefix_len'] = 2 - if not os.path.exists(config_file): - with open(config_file, 'w') as config: - self.config = json.dump(self.config, config) - with open(config_file) as config: - self.config = json.load(config) - # When building the project I was testing this on, this was output - # over 20 times. That seems excessive - global warned - if self.config['prefix_len'] == 1 and self.path not in warned: - msg = "Please update your cache by going into " + self.path +\ - " and running scons-rename-cachedirs.py" - SCons.Warnings.warn(SCons.Warnings.CacheV1Warning, msg) - warned[self.path] = True - + if not os.path.exists(config_file): + with open(config_file, 'w') as config: + self.config = json.dump(self.config, config) + else: + with open(config_file) as config: + self.config = json.load(config) def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: diff --git a/src/engine/SCons/Warnings.py b/src/engine/SCons/Warnings.py index 16319b9..2495b89 100644 --- a/src/engine/SCons/Warnings.py +++ b/src/engine/SCons/Warnings.py @@ -44,7 +44,7 @@ class WarningOnByDefault(Warning): class TargetNotBuiltWarning(Warning): # Should go to OnByDefault pass -class CacheV1Warning(WarningOnByDefault): +class CacheVersionWarning(WarningOnByDefault): pass class CacheWriteErrorWarning(Warning): diff --git a/src/script/scons-configure-cache.py b/src/script/scons-configure-cache.py new file mode 100644 index 0000000..ec3a940 --- /dev/null +++ b/src/script/scons-configure-cache.py @@ -0,0 +1,123 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + +import argparse +import glob +import json +import os + +def rearrange_cache_entries(current_prefix_len, new_prefix_len): + print 'Changing prefix length from', current_prefix_len, 'to', new_prefix_len + dirs = set() + old_dirs = set() + for file in glob.iglob(os.path.join('*', '*')): + name = os.path.basename(file) + dir = name[:current_prefix_len].upper() + if dir not in old_dirs: + print 'Migrating', dir + old_dirs.add(dir) + dir = name[:new_prefix_len].upper() + if dir not in dirs: + os.mkdir(dir) + dirs.add(dir) + os.rename(file, os.path.join(dir, name)) + + # Now delete the original directories + for dir in old_dirs: + os.rmdir(dir) + +parser = argparse.ArgumentParser( + description = 'Modify the configuration of an scons cache directory', + epilog = ''' + Unless you specify an option, it will not be changed (if it is + already set in the cache config), or changed to an appropriate + default (it it is not set). + ''' + ) + +parser.add_argument('cache-dir', help='Path to scons cache directory') +parser.add_argument('--prefix-len', + help='Length of cache file name used as subdirectory prefix', + metavar = '', + type=int) +parser.add_argument('--version', action='version', version='%(prog)s 1.0') + +# Get the command line as a dict without any of the unspecified entries. +args = dict(filter(lambda x: x[1], vars(parser.parse_args()).items())) + +# It seems somewhat strange to me, but positional arguments don't get the - +# in the name changed to _, whereas optional arguments do... +os.chdir(args['cache-dir']) +del args['cache-dir'] + +# If a value isn't currently configured, this contains the way it behaves +# currently (implied), and the way it should behave afer running this script +# (default) +# FIXME: I should use this to construct the parameter list and supply an +# upgrade function +implicit = { + 'prefix_len' : { 'implied' : 1, 'default' : 2 } +} + +if not os.path.exists('config'): + # Validate the only files in the directory are directories 0-9, a-f + expected = [ '{:X}'.format(x) for x in range(0, 16) ] + if not set(os.listdir('.')).issubset(expected): + raise RuntimeError("This doesn't look like a version 1 cache directory") + config = dict() +else: + with open('config') as conf: + config = json.load(conf) + +# Find any keys that aren't currently set but should be +for key in implicit: + if key not in config: + config[key] = implicit[key]['implied'] + if key not in args: + args[key] = implicit[key]['default'] + +#Now we go through each entry in args to see if it changes an existing config +#setting. +for key in args: + if args[key] != config[key]: + if key == 'prefix_len': + rearrange_cache_entries(config[key], args[key]) + config[key] = args[key] + +# and write the updated config file +with open('config', 'w') as conf: + json.dump(config, conf) \ No newline at end of file diff --git a/src/script/scons-rename-cachedirs.py b/src/script/scons-rename-cachedirs.py deleted file mode 100644 index 24897d1..0000000 --- a/src/script/scons-rename-cachedirs.py +++ /dev/null @@ -1,78 +0,0 @@ -#! /usr/bin/env python -# -# SCons - a Software Constructor -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -__version__ = "__VERSION__" - -__build__ = "__BUILD__" - -__buildsys__ = "__BUILDSYS__" - -__date__ = "__DATE__" - -__developer__ = "__DEVELOPER__" - -import glob -import json -import os - -# The entire purpose of this script is to rename the files in the specified -# cache directory from the 16 single hex digit directories to 256 2 hex digit -# directories. - -# You run this in the cache directory. - -expected = ['{:X}'.format(x) for x in range(0, 16)] - -if not os.path.exists('config'): - # check there are 16 directories, 0 - 9, A - F - if sorted(glob.glob('*')) != expected: - raise RuntimeError("This doesn't look like a (version 1) cache directory") - config = { 'prefix_len' : 1 } -else: - with open('config') as conf: - config = json.load(conf) - if config['prefix_len'] != 1: - raise RuntimeError("This doesn't look like a (version 1) cache directory") - -dirs = set() -for file in glob.iglob(os.path.join('*', '*')): - name = os.path.basename(file) - dir = name[:2].upper() - if dir not in dirs: - os.mkdir(dir) - dirs.add(dir) - os.rename(file, os.path.join(dir, name)) - -# Now delete the original directories -for dir in expected: - if os.path.exists(dir): - os.rmdir(dir) - -# and write a config file -config['prefix_len'] = 2 -with open('config', 'w') as conf: - json.dump(config, conf) \ No newline at end of file -- cgit v0.12 From 2bc0caa0e5df6d5602c5640cb54abbacbc74f15c Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 19 Mar 2016 13:09:56 +0000 Subject: Add some error recovery, cleanup scons-configure-cache --- src/engine/SCons/CacheDir.py | 28 +++++++++++++++----- src/script/scons-configure-cache.py | 52 ++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index a941986..aaf3c12 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -169,19 +169,35 @@ class CacheDir(object): global warned if self.path not in warned: msg = "Please upgrade your cache by running " +\ - " scons-upgrade-cache.py " + self.path + " scons-configure-cache.py " + self.path SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) warned[self.path] = True else: if not os.path.isdir(path): - os.makedirs(path) + try: + os.makedirs(path) + except OSError: + # If someone else is trying to create the directory at + # the same time as me, bad things will happen + msg = "Failed to create cache directory " + path + raise SCons.Errors.EnvironmentError(msg) + self.config['prefix_len'] = 2 if not os.path.exists(config_file): - with open(config_file, 'w') as config: - self.config = json.dump(self.config, config) + try: + with open(config_file, 'w') as config: + self.config = json.dump(self.config, config) + except: + msg = "Failed to write cache configuration for " + path + raise SCons.Errors.EnvironmentError(msg) else: - with open(config_file) as config: - self.config = json.load(config) + try: + with open(config_file) as config: + self.config = json.load(config) + except ValueError: + msg = "Failed to read cache configuration for " + path + raise SCons.Errors.EnvironmentError(msg) + def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: diff --git a/src/script/scons-configure-cache.py b/src/script/scons-configure-cache.py index ec3a940..c1b7d59 100644 --- a/src/script/scons-configure-cache.py +++ b/src/script/scons-configure-cache.py @@ -60,6 +60,29 @@ def rearrange_cache_entries(current_prefix_len, new_prefix_len): for dir in old_dirs: os.rmdir(dir) +# This dictionary should have one entry per entry in the cache config +# Each entry should have the following: +# implicit - (optional) This is to allow adding a new config entry and also +# changing the behaviour of the system at the same time. This +# indicates the value the config entry would have had if it had been +# specified. +# default - The value the config entry should have if it wasn't previously +# specified +# command-line - parameters to pass to ArgumentParser.add_argument +# converter - (optional) Function to call if it's necessary to do some work +# if this configuration entry changes +config_entries = { + 'prefix_len' : { + 'implicit' : 1, + 'default' : 2 , + 'command-line' : { + 'help' : 'Length of cache file name used as subdirectory prefix', + 'metavar' : '', + 'type' : int + }, + 'converter' : rearrange_cache_entries + } +} parser = argparse.ArgumentParser( description = 'Modify the configuration of an scons cache directory', epilog = ''' @@ -70,10 +93,9 @@ parser = argparse.ArgumentParser( ) parser.add_argument('cache-dir', help='Path to scons cache directory') -parser.add_argument('--prefix-len', - help='Length of cache file name used as subdirectory prefix', - metavar = '', - type=int) +for param in config_entries: + parser.add_argument('--' + param.replace('_', '-'), + **config_entries[param]['command-line']) parser.add_argument('--version', action='version', version='%(prog)s 1.0') # Get the command line as a dict without any of the unspecified entries. @@ -84,15 +106,6 @@ args = dict(filter(lambda x: x[1], vars(parser.parse_args()).items())) os.chdir(args['cache-dir']) del args['cache-dir'] -# If a value isn't currently configured, this contains the way it behaves -# currently (implied), and the way it should behave afer running this script -# (default) -# FIXME: I should use this to construct the parameter list and supply an -# upgrade function -implicit = { - 'prefix_len' : { 'implied' : 1, 'default' : 2 } -} - if not os.path.exists('config'): # Validate the only files in the directory are directories 0-9, a-f expected = [ '{:X}'.format(x) for x in range(0, 16) ] @@ -104,18 +117,21 @@ else: config = json.load(conf) # Find any keys that aren't currently set but should be -for key in implicit: +for key in config_entries: if key not in config: - config[key] = implicit[key]['implied'] + if 'implicit' in config_entries[key]: + config[key] = config_entries[key]['implicit'] + else: + config[key] = config_entries[key]['default'] if key not in args: - args[key] = implicit[key]['default'] + args[key] = config_entries[key]['default'] #Now we go through each entry in args to see if it changes an existing config #setting. for key in args: if args[key] != config[key]: - if key == 'prefix_len': - rearrange_cache_entries(config[key], args[key]) + if 'converter' in config_entries[key]: + config_entries[key]['converter'](config[key], args[key]) config[key] = args[key] # and write the updated config file -- cgit v0.12 From acb2487f98360aaab390c0bbc9d9f41faab14502 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 19 Mar 2016 17:11:19 +0000 Subject: oopa --- src/engine/SCons/CacheDir.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index aaf3c12..1690674 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -27,8 +27,6 @@ __doc__ = """ CacheDir support """ -from collections import defaultdict - import json import os import stat @@ -145,7 +143,7 @@ class CacheDir(object): self.path = path self.current_cache_debug = None self.debugFP = None - self.config = defaultdict() + self.config = dict() if path is None: return # See if there's a config file in the cache directory. If there is, @@ -186,7 +184,7 @@ class CacheDir(object): if not os.path.exists(config_file): try: with open(config_file, 'w') as config: - self.config = json.dump(self.config, config) + json.dump(self.config, config) except: msg = "Failed to write cache configuration for " + path raise SCons.Errors.EnvironmentError(msg) -- cgit v0.12 From 55ff2ce640dc465833dca3c95ce4cdd2b95428aa Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sat, 19 Mar 2016 19:20:26 +0000 Subject: Add some release notes --- src/RELEASE.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 3bff366..51f2af6 100644 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -32,6 +32,9 @@ NEW FUNCTIONALITY - List new features (presumably why a checkpoint is being released) + - Addition of script scons-configure-cache which allows you to configure + information about a specific cache directly (currently it only supports + determining how the cache files are divided into subdirectories). DEPRECATED FUNCTIONALITY @@ -41,6 +44,11 @@ - List modifications to existing features, where the previous behavior wouldn't actually be considered a bug + - Cache files are now stored in 256 subdirectories in the cache directory + by default (this stresses NFS less). Existing cache directories will + remain as current, but scons will prompt you to run scons-configure-cache + which will allow you to use the new layour, or confirm you want to use the + existing layout. FIXES -- cgit v0.12 From 8f9ad5d0e91468cbe918a7e67affae6be63ac6d5 Mon Sep 17 00:00:00 2001 From: Thomas Tanner Date: Sun, 20 Mar 2016 07:19:59 +0000 Subject: Migrate stuff to the right place --- src/Announce.txt | 8 ++++++++ src/RELEASE.txt | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Announce.txt b/src/Announce.txt index c20d7df..0b58958 100644 --- a/src/Announce.txt +++ b/src/Announce.txt @@ -44,6 +44,12 @@ Notes: a scanner found in SCANNERS (but not for built-in scanners), but now the Install builder will not scan recursively regardless in order to optimize Install behaviour and bring orthogonality to previous behaviour. +* SCons handles cache directories a bit differently/ +** Cache files are now stored in 256 subdirectories in the cache directory by + default (this stresses NFS less). Existing cache directories will remain as + current, but SCons will prompt you to run scons-configure-cache which will + allow you to migrate to the new layout, or confirm you want to use the + existing layout. +================================================================= @@ -56,6 +62,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER changes. Please note the following important changes since release 2.4.0: + - New external tool scons-configurecache which allows some configuration of + how files in the cache are controlled. - Fix to swig tool - pick-up 'swig', 'swig3.0' and 'swig2.0' (in order). - Fix to swig tool - respect env['SWIG'] provided by user. - Fix for Bug # 2791 - Setup.py fails unnecessarily under Jython. diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 51f2af6..3bff366 100644 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -32,9 +32,6 @@ NEW FUNCTIONALITY - List new features (presumably why a checkpoint is being released) - - Addition of script scons-configure-cache which allows you to configure - information about a specific cache directly (currently it only supports - determining how the cache files are divided into subdirectories). DEPRECATED FUNCTIONALITY @@ -44,11 +41,6 @@ - List modifications to existing features, where the previous behavior wouldn't actually be considered a bug - - Cache files are now stored in 256 subdirectories in the cache directory - by default (this stresses NFS less). Existing cache directories will - remain as current, but scons will prompt you to run scons-configure-cache - which will allow you to use the new layour, or confirm you want to use the - existing layout. FIXES -- cgit v0.12