From 1519a7d40f5e65c1c1c896c9c96456f8364c6907 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 10 Nov 2018 09:41:28 -0700 Subject: Some fixes to sconsign Two locations which attempt to directly print an item tipped over on py3, so they now decode(). There seem to be cases where implicit dependencies do not have signatures, so instead of looping through the dep list and indexing into the signature list (IndexError), the two lists are now zipped, which means nothing is printed, but sconsign does not die (the zip technique is used in FS.py in the engine). Minor PEP8 changes: spaces around operators; shorter lines; two-blanks rule around classes/functions. Also unused args changed to _ to show it was intentional. Manpage updated slightly - the internal whichdb function explicitly looks for the .dblite suffix, so the claim that if it's not .dbm it is assumed to be dblite was not true. sconsign still will not work on a dblite file which is not suffixed .dblite, but that is an existing problem, not a newly introduced one. Signed-off-by: Mats Wichmann --- doc/man/sconsign.xml | 16 +++++++----- src/CHANGES.txt | 2 ++ src/script/sconsign.py | 68 ++++++++++++++++++++++++++++++++++---------------- 3 files changed, 59 insertions(+), 27 deletions(-) diff --git a/doc/man/sconsign.xml b/doc/man/sconsign.xml index 44a1e5f..75eeaf1 100644 --- a/doc/man/sconsign.xml +++ b/doc/man/sconsign.xml @@ -25,7 +25,7 @@ --> - @@ -42,7 +42,7 @@ - sconsign + sconsign options file ... @@ -51,7 +51,7 @@ DESCRIPTION -The +The sconsign command displays the contents of one or more @@ -94,17 +94,21 @@ was specified by the function). Any file -argument that does not end in -.dbm +argument that ends in +.dblite is assumed to be a traditional .sconsign file containing the signature entries for a single directory. +If neither of those is true, +sconsign +attempts to guess the format (but will +probably get it wrong). An explicit format may be specified using the or - + options. diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 97c3455..bae54d5 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -176,6 +176,8 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Update (pep8) configure-cache script, add a --show option. - Fix for a couple of "what if tool not found" exceptions in framework. - Add Textfile/Substfile to default environment. (issue #3147) + - sconsign: a couple of python3 fixes; be more tolerant of implicit + entries which have no signatures; minor PEP8 changes. From Bernhard M. Wiedemann: - Update SCons' internal scons build logic to allow overriding build date diff --git a/src/script/sconsign.py b/src/script/sconsign.py index 2337f67..42424a2 100644 --- a/src/script/sconsign.py +++ b/src/script/sconsign.py @@ -164,7 +164,7 @@ else: pass else: # Split /usr/libfoo/python*/os.py to /usr/libfoo/python*. - libpath, tail = os.path.split(libpath) + libpath, _ = os.path.split(libpath) # Split /usr/libfoo/python* to /usr/libfoo libpath, tail = os.path.split(libpath) # Check /usr/libfoo/scons*. @@ -195,6 +195,7 @@ import imp import SCons.SConsign + def my_whichdb(filename): if filename[-7:] == ".dblite": return "SCons.dblite" @@ -225,14 +226,18 @@ def my_import(mname): fp, pathname, description = imp.find_module(mname) return imp.load_module(mname, fp, pathname, description) + class Flagger(object): default_value = 1 + def __setitem__(self, item, value): self.__dict__[item] = value self.default_value = 0 + def __getitem__(self, item): return self.__dict__.get(item, self.default_value) + Do_Call = None Print_Directories = [] Print_Entries = [] @@ -240,9 +245,10 @@ Print_Flags = Flagger() Verbose = 0 Readable = 0 + def default_mapper(entry, name): try: - val = eval("entry."+name) + val = eval("entry." + name) except: val = None if sys.version_info.major >= 3 and isinstance(val, bytes): @@ -252,6 +258,7 @@ def default_mapper(entry, name): val = val.decode() return str(val) + def map_action(entry, name): try: bact = entry.bact @@ -260,7 +267,7 @@ def map_action(entry, name): return None return '%s [%s]' % (bactsig, bact) -def map_timestamp(entry, name): +def map_timestamp(entry, _): try: timestamp = entry.timestamp except AttributeError: @@ -270,19 +277,20 @@ def map_timestamp(entry, name): else: return str(timestamp) -def map_bkids(entry, name): +def map_bkids(entry, _): try: bkids = entry.bsources + entry.bdepends + entry.bimplicit bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs except AttributeError: return None result = [] - for i in range(len(bkids)): - result.append(nodeinfo_string(bkids[i], bkidsigs[i], " ")) + for bkid, bkidsig in zip(bkids, bkidsigs): + result.append(nodeinfo_string(bkid, bkidsig, " ")) if not result: return None return "\n ".join(result) + map_field = { 'action' : map_action, 'timestamp' : map_timestamp, @@ -293,6 +301,7 @@ map_name = { 'implicit' : 'bkids', } + def field(name, entry, verbose=Verbose): if not Print_Flags[name]: return None @@ -303,6 +312,7 @@ def field(name, entry, verbose=Verbose): val = name + ": " + val return val + def nodeinfo_raw(name, ninfo, prefix=""): # This just formats the dictionary, which we would normally use str() # to do, except that we want the keys sorted for deterministic output. @@ -318,6 +328,7 @@ def nodeinfo_raw(name, ninfo, prefix=""): name = repr(name) return name + ': {' + ', '.join(l) + '}' + def nodeinfo_cooked(name, ninfo, prefix=""): try: field_list = ninfo.field_list @@ -325,15 +336,19 @@ def nodeinfo_cooked(name, ninfo, prefix=""): field_list = [] if '\n' in name: name = repr(name) - outlist = [name+':'] + [_f for _f in [field(x, ninfo, Verbose) for x in field_list] if _f] + outlist = [name + ':'] + [ + f for f in [field(x, ninfo, Verbose) for x in field_list] if f + ] if Verbose: sep = '\n ' + prefix else: sep = ' ' return sep.join(outlist) + nodeinfo_string = nodeinfo_cooked + def printfield(name, entry, prefix=""): outlist = field("implicit", entry, 0) if outlist: @@ -347,13 +362,15 @@ def printfield(name, entry, prefix=""): else: print(" " + outact) + def printentries(entries, location): if Print_Entries: for name in Print_Entries: try: entry = entries[name] except KeyError: - sys.stderr.write("sconsign: no entry `%s' in `%s'\n" % (name, location)) + err = "sconsign: no entry `%s' in `%s'\n" % (name, location) + sys.stderr.write(err) else: try: ninfo = entry.ninfo @@ -373,6 +390,7 @@ def printentries(entries, location): print(nodeinfo_string(name, entry.ninfo)) printfield(name, entry.binfo) + class Do_SConsignDB(object): def __init__(self, dbm_name, dbm): self.dbm_name = dbm_name @@ -407,15 +425,17 @@ class Do_SConsignDB(object): # Nope, that file doesn't even exist, so report that # fact back. print_e = e - sys.stderr.write("sconsign: %s\n" % (print_e)) + sys.stderr.write("sconsign: %s\n" % print_e) return except KeyboardInterrupt: raise except pickle.UnpicklingError: - sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" % (self.dbm_name, fname)) + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" + % (self.dbm_name, fname)) return except Exception as e: - sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" % (self.dbm_name, fname, e)) + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" + % (self.dbm_name, fname, e)) return if Print_Directories: @@ -423,35 +443,41 @@ class Do_SConsignDB(object): try: val = db[dir] except KeyError: - sys.stderr.write("sconsign: no dir `%s' in `%s'\n" % (dir, args[0])) + err = "sconsign: no dir `%s' in `%s'\n" % (dir, args[0]) + sys.stderr.write(err) else: self.printentries(dir, val) else: for dir in sorted(db.keys()): self.printentries(dir, db[dir]) - def printentries(self, dir, val): - print('=== ' + dir + ':') + @staticmethod + def printentries(dir, val): + print('=== ' + dir.decode() + ':') printentries(pickle.loads(val), dir) + def Do_SConsignDir(name): try: fp = open(name, 'rb') except (IOError, OSError) as e: - sys.stderr.write("sconsign: %s\n" % (e)) + sys.stderr.write("sconsign: %s\n" % e) return try: sconsign = SCons.SConsign.Dir(fp) except KeyboardInterrupt: raise except pickle.UnpicklingError: - sys.stderr.write("sconsign: ignoring invalid .sconsign file `%s'\n" % (name)) + err = "sconsign: ignoring invalid .sconsign file `%s'\n" % (name) + sys.stderr.write(err) return except Exception as e: - sys.stderr.write("sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e)) + err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) + sys.stderr.write(err) return printentries(sconsign.entries, args[0]) + ############################################################################## import getopt @@ -493,8 +519,7 @@ for o, a in opts: elif o in ('-f', '--format'): # Try to map the given DB format to a known module # name, that we can then try to import... - Module_Map = {'dblite' : 'SCons.dblite', - 'sconsign' : None} + Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} dbm_name = Module_Map.get(a, a) if dbm_name: try: @@ -502,6 +527,7 @@ for o, a in opts: dbm = my_import(dbm_name) else: import SCons.dblite + dbm = SCons.dblite # Ensure that we don't ignore corrupt DB files, # this was handled by calling my_import('SCons.dblite') @@ -530,7 +556,6 @@ for o, a in opts: elif o in ('-v', '--verbose'): Verbose = 1 - if Do_Call: for a in args: Do_Call(a) @@ -538,11 +563,12 @@ else: for a in args: dbm_name = whichdb(a) if dbm_name: - Map_Module = {'SCons.dblite' : 'dblite'} + Map_Module = {'SCons.dblite': 'dblite'} if dbm_name != "SCons.dblite": dbm = my_import(dbm_name) else: import SCons.dblite + dbm = SCons.dblite # Ensure that we don't ignore corrupt DB files, # this was handled by calling my_import('SCons.dblite') -- cgit v0.12 From 776672af607e359468dc87e7efcca28260f70a7d Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 10 Nov 2018 10:58:27 -0700 Subject: For PR #3238: use a try block for str/bytes problem The previous fix changed a failing line that printed something that could be bytes in Py3 to do a decode. But it turned out that value can be either str or bytes and the change failed the sconsign tests. Use a try block instead of unconditionally calling decode. Signed-off-by: Mats Wichmann --- src/script/sconsign.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/script/sconsign.py b/src/script/sconsign.py index 42424a2..567dc07 100644 --- a/src/script/sconsign.py +++ b/src/script/sconsign.py @@ -453,7 +453,10 @@ class Do_SConsignDB(object): @staticmethod def printentries(dir, val): - print('=== ' + dir.decode() + ':') + try: + print('=== ' + dir + ':') + except TypeError: + print('=== ' + dir.decode() + ':') printentries(pickle.loads(val), dir) -- cgit v0.12 From 7cf33c636a0052e661d95f4ddb8e3d6237383ee4 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 10 Nov 2018 13:14:49 -0700 Subject: For PR #3238: suggestion to use a comprehension Signed-off-by: Mats Wichmann --- src/script/sconsign.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/script/sconsign.py b/src/script/sconsign.py index 567dc07..9a10907 100644 --- a/src/script/sconsign.py +++ b/src/script/sconsign.py @@ -283,9 +283,8 @@ def map_bkids(entry, _): bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs except AttributeError: return None - result = [] - for bkid, bkidsig in zip(bkids, bkidsigs): - result.append(nodeinfo_string(bkid, bkidsig, " ")) + result = [nodeinfo_string(bkid, bkidsig, " ") + for bkid, bkidsig in zip(bkids, bkidsigs)] if not result: return None return "\n ".join(result) -- cgit v0.12 From a57317332a5c8d47d0edae322ebd3a5b5ab21784 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 12 Nov 2018 12:10:29 -0700 Subject: PR #3238: sconsign manpage wording for unknown format. Signed-off-by: Mats Wichmann --- doc/man/sconsign.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/man/sconsign.xml b/doc/man/sconsign.xml index 75eeaf1..aad2cd9 100644 --- a/doc/man/sconsign.xml +++ b/doc/man/sconsign.xml @@ -102,9 +102,9 @@ file containing the signature entries for a single directory. If neither of those is true, sconsign -attempts to guess the format (but will -probably get it wrong). -An explicit format +attempts to guess the format. +If that does not work, +an explicit format may be specified using the or -- cgit v0.12 From 1c7b5c00fbec3487ad4eee65ec8d5e1bac552e9a Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 30 Nov 2018 11:50:40 -0700 Subject: For sconsign, add a warning message if missing sigs Rather than just silently moving on, emit warning messages if id count does not match signature count; summarize at the end if there were any warnings. Signed-off-by: Mats Wichmann --- src/script/sconsign.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/src/script/sconsign.py b/src/script/sconsign.py index 9a10907..6241ae2 100644 --- a/src/script/sconsign.py +++ b/src/script/sconsign.py @@ -244,9 +244,21 @@ Print_Entries = [] Print_Flags = Flagger() Verbose = 0 Readable = 0 +Warns = 0 + def default_mapper(entry, name): + ''' + Stringify an entry that doesn't have an explicit mapping. + + Args: + entry: entry + name: field name + + Returns: str + + ''' try: val = eval("entry." + name) except: @@ -259,7 +271,17 @@ def default_mapper(entry, name): return str(val) -def map_action(entry, name): +def map_action(entry, _): + ''' + Stringify an action entry and signature. + + Args: + entry: action entry + second argument is not used + + Returns: str + + ''' try: bact = entry.bact bactsig = entry.bactsig @@ -268,6 +290,16 @@ def map_action(entry, name): return '%s [%s]' % (bactsig, bact) def map_timestamp(entry, _): + ''' + Stringify a timestamp entry. + + Args: + entry: timestamp entry + second argument is not used + + Returns: str + + ''' try: timestamp = entry.timestamp except AttributeError: @@ -278,13 +310,32 @@ def map_timestamp(entry, _): return str(timestamp) def map_bkids(entry, _): + ''' + Stringify an implicit entry. + + Args: + entry: + second argument is not used + + Returns: str + + ''' try: bkids = entry.bsources + entry.bdepends + entry.bimplicit bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs except AttributeError: return None - result = [nodeinfo_string(bkid, bkidsig, " ") - for bkid, bkidsig in zip(bkids, bkidsigs)] + + if len(bkids) != len(bkidsigs): + global Warns + Warns += 1 + # add warning to result rather than direct print so it will line up + msg = "Warning: missing information, {} ids but {} sigs" + result = [msg.format(len(bkids), len(bkidsigs))] + else: + result = [] + result += [nodeinfo_string(bkid, bkidsig, " ") + for bkid, bkidsig in zip(bkids, bkidsigs)] if not result: return None return "\n ".join(result) @@ -580,6 +631,8 @@ else: else: Do_SConsignDir(a) + if Warns: + print("NOTE: there were %d warnings, please check output" % Warns) sys.exit(0) # Local Variables: -- cgit v0.12