First draft of teaching Python scanner about dynamic files

8 files changed, 94 insertions, 31 deletions

diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py
index 967f007..cc2e75a 100644
--- a/SCons/Node/FS.py
+++ b/SCons/Node/FS.py
@@ -2664,6 +2664,8 @@ class File(Base):
     def _morph(self):
         """Turn a file system node into a File object."""
         self.scanner_paths = {}
+        if self.abspath.endswith('package1'):
+            raise Exception(self.abspath)
         if not hasattr(self, '_local'):
             self._local = 0
         if not hasattr(self, 'released_target_info'):
@@ -3724,7 +3726,10 @@ class FileFinder:
         return None
 
     def _find_file_key(self, filename, paths, verbose=None):
-        return (filename, paths)
+        # Note: paths could be a list, which is not hashable. If it is, convert
+        # it to a tuple.
+        paths_entry = tuple(paths) if isinstance(paths, list) else paths
+        return (filename, paths_entry)
 
     @SCons.Memoize.CountDictCall(_find_file_key)
     def find_file(self, filename, paths, verbose=None):
@@ -3773,6 +3778,8 @@ class FileFinder:
                 result = node
                 break
 
+        print('value: %s (%s)' % (result, type(result)))
+        print('key: %s (%s)' % (memo_key, type(memo_key)))
         memo_dict[memo_key] = result
 
         return result
diff --git a/SCons/Scanner/Python.py b/SCons/Scanner/Python.py
index dc6812c..965b9c1 100644
--- a/SCons/Scanner/Python.py
+++ b/SCons/Scanner/Python.py
@@ -108,49 +108,41 @@ def scan(node, env, path=()):
             for i in itertools.repeat(None, num_parents):
                 current_dir = current_dir.up()
 
-            search_paths = [current_dir.abspath]
+            search_paths = [current_dir]
             search_string = module_lstripped
         else:
-            search_paths = path
+            search_paths = [env.Dir(p) for p in tuple(path)]
             search_string = module
 
         module_components = search_string.split('.')
-        for search_path in search_paths:
-            candidate_path = os.path.join(search_path, *module_components)
-            # The import stored in "module" could refer to a directory or file.
-            import_dirs = []
-            if os.path.isdir(candidate_path):
-                import_dirs = module_components
-
-                # Because this resolved to a directory, there is a chance that
-                # additional imports (e.g. from module import A, B) could refer
-                # to files to import.
-                if imports:
-                    for imp in imports:
-                        file = os.path.join(candidate_path, imp + '.py')
-                        if os.path.isfile(file):
-                            nodes.append(file)
-            elif os.path.isfile(candidate_path + '.py'):
-                nodes.append(candidate_path + '.py')
-                import_dirs = module_components[:-1]
-
-                # We can ignore imports because this resolved to a file. Any
-                # additional imports (e.g. from module.file import A, B) would
-                # only refer to functions in this file.
+        module_joined = '/'.join(module_components)
+        print('%s - %s' % (module_joined, [str(s) for s in search_paths]))
+        node = SCons.Node.FS.find_file(module_joined, search_paths, verbose=True)
+        if node:
+            # The fact that we were able to find the node without appending .py
+            # means that this is a directory import.
+            nodes.append(env.Dir(node).File('__init__.py'))
 
             # Take a dependency on all __init__.py files from all imported
             # packages unless it's a relative import. If it's a relative
             # import, we don't need to take the dependency because Python
             # requires that all referenced packages have already been imported,
             # which means that the dependency has already been established.
-            if import_dirs and not is_relative:
+            # XXX TODO: This part is broken and needs to be fixed.
+            if not is_relative and len(module_components) > 1:
+                import_dirs = module_components
                 for i in range(len(import_dirs)):
-                    init_components = module_components[:i+1] + ['__init__.py']
-                    init_path = os.path.join(search_path, *(init_components))
-                    if os.path.isfile(init_path):
-                        nodes.append(init_path)
-                break
+                    init_path = '/'.join(module_components[:i+1] + ['__init__.py'])
+                    # TODO: Passing search_paths is not correct.
+                    init_node = SCons.Node.FS.find_file(init_path, search_paths, verbose=True)
+                    if init_node:
+                        nodes.append(init_node)
+        else:
+            node = SCons.Node.FS.find_file(module_joined + '.py', search_paths, verbose=True)
+            if node:
+                nodes.append(node)
 
+    print('nodes: %s' % [str(n) for n in nodes])
     return sorted(nodes)
 
 
diff --git a/test/Scanner/Python.py b/test/Scanner/Python.py
new file mode 100644
index 0000000..74c8a87
--- /dev/null
+++ b/test/Scanner/Python.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#
+# MIT License
+#
+# Copyright The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Template for end-to-end test file.
+Replace this with a description of the test.
+"""
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+test.dir_fixture('Python')
+test.run(arguments = '--debug=stacktrace .')
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/Scanner/Python/SConstruct b/test/Scanner/Python/SConstruct
new file mode 100644
index 0000000..63fc44b
--- /dev/null
+++ b/test/Scanner/Python/SConstruct
@@ -0,0 +1,13 @@
+import sys
+
+env = Environment(tools=['python'])
+c = []
+for source, target in [
+	('to_be_copied', 'package1'),
+	('to_be_copied', 'package2'),
+]:
+	c += env.Command(target, source, Copy('$TARGET', '$SOURCE'))
+# Don't set a dependency on the copy actions on purpose. Scanner should find
+# the dependencies automatically.
+s = env.Command('a.out', 'script.py', '$PYTHON $SOURCE $TARGET', PYTHON=sys.executable)
+env.Depends(s, c)
\ No newline at end of file
diff --git a/test/Scanner/Python/sconstest.skip b/test/Scanner/Python/sconstest.skip
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/Scanner/Python/sconstest.skip
diff --git a/test/Scanner/Python/script.py b/test/Scanner/Python/script.py
new file mode 100644
index 0000000..3970f8d
--- /dev/null
+++ b/test/Scanner/Python/script.py
@@ -0,0 +1,8 @@
+import package1
+import package2
+import sys
+
+raise Exception(sys.argv)
+
+with open(sys.argv[1], 'w') as f:
+    f.write('test')
diff --git a/test/Scanner/Python/to_be_copied/__init__.py b/test/Scanner/Python/to_be_copied/__init__.py
new file mode 100644
index 0000000..34fdc42
--- /dev/null
+++ b/test/Scanner/Python/to_be_copied/__init__.py
@@ -0,0 +1 @@
+from . import helper
\ No newline at end of file
diff --git a/test/Scanner/Python/to_be_copied/helper.py b/test/Scanner/Python/to_be_copied/helper.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/Scanner/Python/to_be_copied/helper.py