PY2/3 For msvc resource scanning, ensure we don't try to scan .tlb files which are binary. This worked fine on py2 because all strings are bytes, but failed on py3 because it wasn't valid unicode. Also fixed issue where the Classic scanner was hardcoding recursive to be 1 and not propagating any such parameter passed to its constructor. Added test for changes to Classic Scanner

3 files changed, 64 insertions, 18 deletions

diff --git a/src/engine/SCons/Scanner/RC.py b/src/engine/SCons/Scanner/RC.py
index 61393ae..82f8ddc 100644
--- a/src/engine/SCons/Scanner/RC.py
+++ b/src/engine/SCons/Scanner/RC.py
@@ -30,21 +30,32 @@ Definition Language) files.
 
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
+import re
+
 import SCons.Node.FS
 import SCons.Scanner
-import re
+
+
+def no_tlb(nodes):
+    """
+    Filter out .tlb files as they are binary and shouldn't be scanned
+    """
+    # print("Nodes:%s"%[str(n) for n in nodes])
+    return [n for n in nodes if str(n)[-4:] != '.tlb']
+
 
 def RCScan():
     """Return a prototype Scanner instance for scanning RC source files"""
- 
+
     res_re= r'^(?:\s*#\s*(?:include)|' \
             '.*?\s+(?:ICON|BITMAP|CURSOR|HTML|FONT|MESSAGETABLE|TYPELIB|REGISTRY|D3DFX)' \
             '\s*.*?)' \
             '\s*(<|"| )([^>"\s]+)(?:[>"\s])*$'
-    resScanner = SCons.Scanner.ClassicCPP( "ResourceScanner",
-                                           "$RCSUFFIXES",
-                                           "CPPPATH",
-                                           res_re )
+    resScanner = SCons.Scanner.ClassicCPP("ResourceScanner",
+                                          "$RCSUFFIXES",
+                                          "CPPPATH",
+                                          res_re,
+                                          recursive=no_tlb)
     
     return resScanner
 
diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py
index e34dcad..8050216 100644
--- a/src/engine/SCons/Scanner/ScannerTests.py
+++ b/src/engine/SCons/Scanner/ScannerTests.py
@@ -105,7 +105,7 @@ class ScannerTestCase(unittest.TestCase):
         assert str(s) == 'fooscan', str(s)
         assert s.argument == 888, s.argument
 
-        
+
 class BaseTestCase(unittest.TestCase):
 
     class skey_node(object):
@@ -447,6 +447,17 @@ class CurrentTestCase(unittest.TestCase):
         self.failUnless(ic.func_called, "did not call func()")
 
 class ClassicTestCase(unittest.TestCase):
+
+    def func(self, filename, env, target, *args):
+        self.filename = filename
+        self.env = env
+        self.target = target
+
+        if len(args) > 0:
+            self.arg = args[0]
+
+        return self.deps
+
     def test_find_include(self):
         """Test the Scanner.Classic find_include() method"""
         env = DummyEnvironment()
@@ -548,6 +559,25 @@ class ClassicTestCase(unittest.TestCase):
         ret = s.function(n, env, ('foo5',))
         assert ret == ['jkl', 'mno'], ret
 
+    def test_recursive(self):
+        """Test the Scanner.Classic class recursive flag"""
+        nodes = [1, 2, 3, 4]
+
+
+        s = SCons.Scanner.Classic("Test", [], None, "", function=self.func, recursive=1)
+        n = s.recurse_nodes(nodes)
+        self.failUnless(n == n,
+                        "recursive = 1 didn't return all nodes: %s" % n)
+
+        def odd_only(nodes):
+            return [n for n in nodes if n % 2]
+
+        s = SCons.Scanner.Classic("Test", [], None, "", function=self.func, recursive=odd_only)
+        n = s.recurse_nodes(nodes)
+        self.failUnless(n == [1, 3],
+                        "recursive = 1 didn't return all nodes: %s" % n)
+
+
         
 
 class ClassicCPPTestCase(unittest.TestCase):
diff --git a/src/engine/SCons/Scanner/__init__.py b/src/engine/SCons/Scanner/__init__.py
index 28be642..c0f1a93 100644
--- a/src/engine/SCons/Scanner/__init__.py
+++ b/src/engine/SCons/Scanner/__init__.py
@@ -188,12 +188,12 @@ class Base(object):
     def path(self, env, dir=None, target=None, source=None):
         if not self.path_function:
             return ()
-        if not self.argument is _null:
+        if self.argument is not _null:
             return self.path_function(env, dir, target, source, self.argument)
         else:
             return self.path_function(env, dir, target, source)
 
-    def __call__(self, node, env, path = ()):
+    def __call__(self, node, env, path=()):
         """
         This method scans a single object. 'node' is the node
         that will be passed to the scanner function, and 'env' is the
@@ -206,16 +206,16 @@ class Base(object):
         self = self.select(node)
 
         if not self.argument is _null:
-            list = self.function(node, env, path, self.argument)
+            node_list = self.function(node, env, path, self.argument)
         else:
-            list = self.function(node, env, path)
+            node_list = self.function(node, env, path)
 
         kw = {}
         if hasattr(node, 'dir'):
             kw['directory'] = node.dir
         node_factory = env.get_factory(self.node_factory)
         nodes = []
-        for l in list:
+        for l in node_list:
             if self.node_class and not isinstance(l, self.node_class):
                 l = node_factory(l, **kw)
             nodes.append(l)
@@ -259,7 +259,7 @@ class Base(object):
     def _recurse_no_nodes(self, nodes):
         return []
 
-    recurse_nodes = _recurse_no_nodes
+    # recurse_nodes = _recurse_no_nodes
 
     def add_scanner(self, skey, scanner):
         self.function[skey] = scanner
@@ -283,7 +283,7 @@ class Selector(Base):
         self.dict = dict
         self.skeys = list(dict.keys())
 
-    def __call__(self, node, env, path = ()):
+    def __call__(self, node, env, path=()):
         return self.select(node)(node, env, path)
 
     def select(self, node):
@@ -326,7 +326,7 @@ class Classic(Current):
 
         self.cre = re.compile(regex, re.M)
 
-        def _scan(node, env, path=(), self=self):
+        def _scan(node, _, path=(), self=self):
             node = node.rfile()
             if not node.exists():
                 return []
@@ -334,7 +334,12 @@ class Classic(Current):
 
         kw['function'] = _scan
         kw['path_function'] = FindPathDirs(path_variable)
-        kw['recursive'] = 1
+
+        # Allow recursive to propagate if child class specifies.
+        # In this case resource scanner needs to specify a filter on which files
+        # get recursively processed.  Previously was hardcoded to 1 instead of
+        # defaulted to 1.
+        kw['recursive'] = kw.get('recursive', 1)
         kw['skeys'] = suffixes
         kw['name'] = name
 
@@ -356,7 +361,7 @@ class Classic(Current):
         if node.includes is not None:
             includes = node.includes
         else:
-            includes = self.find_include_names (node)
+            includes = self.find_include_names(node)
             # Intern the names of the include files. Saves some memory
             # if the same header is included many times.
             node.includes = list(map(SCons.Util.silent_intern, includes))
@@ -393,7 +398,7 @@ class ClassicCPP(Classic):
     the contained filename in group 1.
     """
     def find_include(self, include, source_dir, path):
-        include = list (map (SCons.Util.to_str, include))
+        include = list(map(SCons.Util.to_str, include))
         if include[0] == '"':
             paths = (source_dir,) + tuple(path)
         else: