From 897ab9da7a09ec3f4705b84ad43485697761c594 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Wed, 19 Oct 2022 10:20:50 -0400 Subject: Obtain a small perf win by using types that iterate faster for children --- CHANGES.txt | 5 +++++ SCons/Executor.py | 12 ++++++------ SCons/Util.py | 41 ++++------------------------------------- 3 files changed, 15 insertions(+), 43 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9cd653..5039eed 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -41,6 +41,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER SConscriptChdir are now listed as Global functions only; the Environment versions still work but are not documented. + From Andrew Morrow + - Avoid returning UniqueList for `children` and other `Executor` APIs. This type + iterates more slowly than the builtin types. Also simplify uniquer_hashables to + use an faster implementation under the assumption of ordered dictionaries. + RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 From Joseph Brill: diff --git a/SCons/Executor.py b/SCons/Executor.py index 492ebe3..274af6a 100644 --- a/SCons/Executor.py +++ b/SCons/Executor.py @@ -306,30 +306,30 @@ class Executor(object, metaclass=NoSlotsPyPy): over and over), so removing the duplicates once up front should save the Taskmaster a lot of work. """ - result = SCons.Util.UniqueList([]) + result = [] for target in self.get_all_targets(): result.extend(target.children()) - return result + return SCons.Util.uniquer_hashables(result) def get_all_prerequisites(self): """Returns all unique (order-only) prerequisites for all batches of this Executor. """ - result = SCons.Util.UniqueList([]) + result = [] for target in self.get_all_targets(): if target.prerequisites is not None: result.extend(target.prerequisites) - return result + return SCons.Util.uniquer_hashables(result) def get_action_side_effects(self): """Returns all side effects for all batches of this Executor used by the underlying Action. """ - result = SCons.Util.UniqueList([]) + result = [] for target in self.get_action_targets(): result.extend(target.side_effects) - return result + return SCons.Util.uniquer_hashables(result) @SCons.Memoize.CountMethodCall def get_build_env(self): diff --git a/SCons/Util.py b/SCons/Util.py index 49a3a0f..4b06ca4 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1363,44 +1363,11 @@ def unique(seq): u.append(x) return u - -# From Alex Martelli, -# https://code.activestate.com/recipes/52560 -# ASPN: Python Cookbook: Remove duplicates from a sequence -# First comment, dated 2001/10/13. -# (Also in the printed Python Cookbook.) -# This not currently used, in favor of the next function... - -def uniquer(seq, idfun=None): - def default_idfun(x): - return x - if not idfun: - idfun = default_idfun - seen = {} - result = [] - result_append = result.append # perf: avoid repeated method lookups - for item in seq: - marker = idfun(item) - if marker in seen: - continue - seen[marker] = 1 - result_append(item) - return result - -# A more efficient implementation of Alex's uniquer(), this avoids the -# idfun() argument and function-call overhead by assuming that all -# items in the sequence are hashable. Order-preserving. - +# Best way (assuming Python 3.7, but effectively 3.6) to remove +# duplicates from a list in while preserving order, according to +# https://discord.com/channels/@me/627187920079421471/1032001231952027648 def uniquer_hashables(seq): - seen = {} - result = [] - result_append = result.append # perf: avoid repeated method lookups - for item in seq: - if item not in seen: - seen[item] = 1 - result_append(item) - return result - + return list(dict.fromkeys(seq)) # Recipe 19.11 "Reading Lines with Continuation Characters", # by Alex Martelli, straight from the Python CookBook (2nd edition). -- cgit v0.12 From 742ee9a4ed63c88158a12be92b7218d4eed31207 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Fri, 21 Oct 2022 12:24:10 -0400 Subject: fix link about unique --- SCons/Util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Util.py b/SCons/Util.py index 4b06ca4..33e02f4 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1365,7 +1365,7 @@ def unique(seq): # Best way (assuming Python 3.7, but effectively 3.6) to remove # duplicates from a list in while preserving order, according to -# https://discord.com/channels/@me/627187920079421471/1032001231952027648 +# https://stackoverflow.com/questions/480214/how-do-i-remove-duplicates-from-a-list-while-preserving-order/17016257#17016257 def uniquer_hashables(seq): return list(dict.fromkeys(seq)) -- cgit v0.12