Merged in ricklupton/scons (pull request #374)

Find dependencies using LaTeX "import" package commands (updated)

5 files changed, 319 insertions, 31 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 8d7aa28..83f979a 100644
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -65,6 +65,9 @@ RELEASE 2.5.1 - Mon, 03 Nov 2016 13:37:42 -0400
     - Use memoization to optimize PATH evaluation across all dependencies per
       node. (PR #345)
 
+  From Rick Lupton:
+    - Update LaTeX scanner to understand \import and related commands
+
 RELEASE 2.5.0 - Mon, 09 Apr 2016 11:27:42 -0700
 
   From Dirk Baechle:
diff --git a/src/engine/SCons/Scanner/LaTeX.py b/src/engine/SCons/Scanner/LaTeX.py
index 1e0fea1..cb89cf0 100644
--- a/src/engine/SCons/Scanner/LaTeX.py
+++ b/src/engine/SCons/Scanner/LaTeX.py
@@ -166,6 +166,9 @@ class LaTeX(SCons.Scanner.Base):
                      'usepackage': 'TEXINPUTS',
                      'lstinputlisting': 'TEXINPUTS'}
     env_variables = SCons.Util.unique(list(keyword_paths.values()))
+    two_arg_commands = ['import', 'subimport',
+                        'includefrom', 'subincludefrom',
+                        'inputfrom', 'subinputfrom']
 
     def __init__(self, name, suffixes, graphics_extensions, *args, **kw):
 
@@ -175,8 +178,29 @@ class LaTeX(SCons.Scanner.Base):
         # line followed by one or more newline characters (i.e. blank
         # lines), interfering with a match on the next line.
         # add option for whitespace before the '[options]' or the '{filename}'
-        regex = r'^[^%\n]*\\(include|includegraphics(?:\s*\[[^\]]+\])?|lstinputlisting(?:\[[^\]]+\])?|input|bibliography|addbibresource|addglobalbib|addsectionbib|usepackage)\s*{([^}]*)}'
-        self.cre = re.compile(regex, re.M)
+        regex = r'''
+            ^[^%\n]*
+            \\(
+                include
+              | includegraphics(?:\s*\[[^\]]+\])?
+              | lstinputlisting(?:\[[^\]]+\])?
+              | input
+              | import
+              | subimport
+              | includefrom
+              | subincludefrom
+              | inputfrom
+              | subinputfrom
+              | bibliography
+              | addbibresource
+              | addglobalbib
+              | addsectionbib
+              | usepackage
+              )
+                  \s*{([^}]*)}       # first arg
+              (?: \s*{([^}]*)} )?    # maybe another arg
+        '''
+        self.cre = re.compile(regex, re.M | re.X)
         self.comment_re = re.compile(r'^((?:(?:\\%)|[^%\n])*)(.*)$', re.M)
 
         self.graphics_extensions = graphics_extensions
@@ -236,23 +260,26 @@ class LaTeX(SCons.Scanner.Base):
 
         SCons.Scanner.Base.__init__(self, *args, **kw)
 
-    def _latex_names(self, include):
-        filename = include[1]
-        if include[0] == 'input':
+    def _latex_names(self, include_type, filename):
+        if include_type == 'input':
             base, ext = os.path.splitext( filename )
             if ext == "":
                 return [filename + '.tex']
-        if (include[0] == 'include'):
-            return [filename + '.tex']
-        if include[0] == 'bibliography':
+        if include_type in ('include', 'import', 'subimport',
+                            'includefrom', 'subincludefrom',
+                            'inputfrom', 'subinputfrom'):
+            base, ext = os.path.splitext( filename )
+            if ext == "":
+                return [filename + '.tex']
+        if include_type == 'bibliography':
             base, ext = os.path.splitext( filename )
             if ext == "":
                 return [filename + '.bib']
-        if include[0] == 'usepackage':
+        if include_type == 'usepackage':
             base, ext = os.path.splitext( filename )
             if ext == "":
                 return [filename + '.sty']
-        if include[0] == 'includegraphics':
+        if include_type == 'includegraphics':
             base, ext = os.path.splitext( filename )
             if ext == "":
                 #return [filename+e for e in self.graphics_extensions + TexGraphics]
@@ -267,21 +294,26 @@ class LaTeX(SCons.Scanner.Base):
         return SCons.Node.FS._my_normcase(str(include))
 
     def find_include(self, include, source_dir, path):
+        inc_type, inc_subdir, inc_filename = include
         try:
-            sub_path = path[include[0]]
+            sub_paths = path[inc_type]
         except (IndexError, KeyError):
-            sub_path = ()
-        try_names = self._latex_names(include)
+            sub_paths = ((), ())
+        try_names = self._latex_names(inc_type, inc_filename)
+
+        # There are three search paths to try:
+        #  1. current directory "source_dir"
+        #  2. env[var]
+        #  3. env['ENV'][var]
+        search_paths = [(source_dir,)] + list(sub_paths)
+
         for n in try_names:
-            # see if we find it using the path in env[var]
-            i = SCons.Node.FS.find_file(n, (source_dir,) + sub_path[0])
-            if i:
-                return i, include
-            # see if we find it using the path in env['ENV'][var]
-            i = SCons.Node.FS.find_file(n, (source_dir,) + sub_path[1])
-            if i:
-                return i, include
-        return i, include
+            for search_path in search_paths:
+                paths = tuple([d.Dir(inc_subdir) for d in search_path])
+                i = SCons.Node.FS.find_file(n, paths)
+                if i:
+                    return i, include
+        return None, include
 
     def canonical_text(self, text):
         """Standardize an input TeX-file contents.
@@ -300,7 +332,7 @@ class LaTeX(SCons.Scanner.Base):
             line_continues_a_comment = len(comment) > 0
         return '\n'.join(out).rstrip()+'\n'
 
-    def scan(self, node):
+    def scan(self, node, subdir='.'):
         # Modify the default scan function to allow for the regular
         # expression to return a comma separated list of file names
         # as can be the case with the bibliography keyword.
@@ -326,9 +358,14 @@ class LaTeX(SCons.Scanner.Base):
             split_includes = []
             for include in includes:
                 inc_type = noopt_cre.sub('', include[0])
-                inc_list = include[1].split(',')
+                inc_subdir = subdir
+                if inc_type in self.two_arg_commands:
+                    inc_subdir = os.path.join(subdir, include[1])
+                    inc_list = include[2].split(',')
+                else:
+                    inc_list = include[1].split(',')
                 for j in range(len(inc_list)):
-                    split_includes.append( (inc_type, inc_list[j]) )
+                    split_includes.append( (inc_type, inc_subdir, inc_list[j]) )
             #
             includes = split_includes
             node.includes = includes
@@ -359,11 +396,12 @@ class LaTeX(SCons.Scanner.Base):
         while queue:
             
             include = queue.pop()
+            inc_type, inc_subdir, inc_filename = include
             try:
-                if seen[include[1]] == 1:
+                if seen[inc_filename] == 1:
                     continue
             except KeyError:
-                seen[include[1]] = 1
+                seen[inc_filename] = 1
 
             #
             # Handle multiple filenames in include[1]
@@ -372,14 +410,14 @@ class LaTeX(SCons.Scanner.Base):
             if n is None:
                 # Do not bother with 'usepackage' warnings, as they most
                 # likely refer to system-level files
-                if include[0] != 'usepackage':
+                if inc_type != 'usepackage':
                     SCons.Warnings.warn(SCons.Warnings.DependencyWarning,
                                         "No dependency generated for file: %s (included from: %s) -- file not found" % (i, node))
             else:
                 sortkey = self.sort_key(n)
                 nodes.append((sortkey, n))
-                # recurse down 
-                queue.extend( self.scan(n) )
+                # recurse down
+                queue.extend( self.scan(n, inc_subdir) )
 
         return [pair[1] for pair in sorted(nodes)]
 
diff --git a/src/engine/SCons/Scanner/LaTeXTests.py b/src/engine/SCons/Scanner/LaTeXTests.py
index 49553cf..213e89e 100644
--- a/src/engine/SCons/Scanner/LaTeXTests.py
+++ b/src/engine/SCons/Scanner/LaTeXTests.py
@@ -44,6 +44,12 @@ test.write('test1.latex',"""
 include{incNO}
 %\include{incNO}
 xyzzy \include{inc6}
+\subimport{subdir}{inc3}
+\import{subdir}{inc3a}
+\includefrom{subdir}{inc3b}
+\subincludefrom{subdir}{inc3c}
+\inputfrom{subdir}{inc3d}
+\subinputfrom{subdir}{inc3e}
 """)
 
 test.write('test2.latex',"""
@@ -61,6 +67,10 @@ test.subdir('subdir')
 test.write('inc1.tex',"\n")
 test.write('inc2.tex',"\n")
 test.write(['subdir', 'inc3.tex'], "\n")
+for suffix in 'abcde':
+    test.write(['subdir', 'inc3%s.tex' % suffix], "\n")
+test.write(['subdir', 'inc3b.tex'], "\n")
+test.write(['subdir', 'inc3c.tex'], "\n")
 test.write(['subdir', 'inc4.eps'], "\n")
 test.write('inc5.xyz', "\n")
 test.write('inc6.tex', "\n")
@@ -122,7 +132,10 @@ class LaTeXScannerTestCase1(unittest.TestCase):
         s = SCons.Scanner.LaTeX.LaTeXScanner()
         path = s.path(env)
         deps = s(env.File('test1.latex'), env, path)
-        headers = ['inc1.tex', 'inc2.tex', 'inc6.tex']
+        headers = ['inc1.tex', 'inc2.tex', 'inc6.tex',
+                   'subdir/inc3.tex', 'subdir/inc3a.tex',
+                   'subdir/inc3b.tex', 'subdir/inc3c.tex',
+                   'subdir/inc3d.tex', 'subdir/inc3e.tex']
         deps_match(self, deps, headers)
 
 class LaTeXScannerTestCase2(unittest.TestCase):
diff --git a/test/TEX/recursive_scanner_dependencies_import.py b/test/TEX/recursive_scanner_dependencies_import.py
new file mode 100644
index 0000000..d9d2625
--- /dev/null
+++ b/test/TEX/recursive_scanner_dependencies_import.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+"""Verify that we re-run LaTeX after changing a nested \import. This
+checks that recursive implicit dependencies are found correctly.
+
+This is a separate test from the
+recursive_scanner_dependencies_input.py test because \input and
+\include are handled specially by the PDF builder, whereas \import
+dependencies are found only by the scanner.
+
+"""
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+pdflatex = test.where_is('pdflatex')
+
+if not pdflatex:
+    test.skip_test("Could not find pdflatex; skipping test(s).\n")
+
+test.subdir('subdir')
+test.subdir('subdir/subdir2')
+
+test.write(['SConstruct'], """\
+env = Environment(tools=['pdftex', 'tex'])
+env.PDF('master.tex')
+""")
+
+test.write(['master.tex'], r"""
+\documentclass{article}
+\usepackage{import}
+\begin{document}
+\subinputfrom{subdir/}{sub1}
+\end{document}
+""")
+
+test.write(['subdir', 'sub1.tex'], r"""
+\subinputfrom{subdir2/}{sub2}
+""")
+
+test.write(['subdir', 'subdir2', 'sub2.tex'], r"""
+Sub-document 2 content
+""")
+
+test.run()
+
+pdf_output_1 = test.read('master.pdf')
+
+# Change sub2.tex, see if master.pdf is changed
+test.write(['subdir', 'subdir2', 'sub2.tex'], r"""
+Sub-document 2 content -- updated
+""")
+
+test.run()
+
+pdf_output_2 = test.read('master.pdf')
+
+# If the PDF file is the same as it was previously, then it didn't
+# pick up the change in sub2.tex, so fail.
+test.fail_test(pdf_output_1 == pdf_output_2)
+
+# Double-check:  clean everything and rebuild from scratch, which
+# should force the PDF file to be the 1982 version.
+
+test.run(arguments='-c')
+test.run()
+
+pdf_output_3 = test.read('master.pdf')
+
+# If the PDF file is now different than the second run, modulo the
+# creation timestamp and the ID and some other PDF garp, then something
+# else odd has happened, so fail.
+
+pdf_output_2 = test.normalize_pdf(pdf_output_2)
+pdf_output_3 = test.normalize_pdf(pdf_output_3)
+
+if pdf_output_2 != pdf_output_3:
+    import sys
+    test.write('master.normalized.2.pdf', pdf_output_2)
+    test.write('master.normalized.3.pdf', pdf_output_3)
+    sys.stdout.write("***** 2 and 3 are different!\n")
+    sys.stdout.write(test.diff_substr(pdf_output_2, pdf_output_3, 80, 80)
+                     + '\n')
+    sys.stdout.write("Output from run 2:\n")
+    sys.stdout.write(test.stdout(-2) + '\n')
+    sys.stdout.write("Output from run 3:\n")
+    sys.stdout.write(test.stdout() + '\n')
+    sys.stdout.flush()
+    test.fail_test()
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4:
diff --git a/test/TEX/recursive_scanner_dependencies_input.py b/test/TEX/recursive_scanner_dependencies_input.py
new file mode 100644
index 0000000..257051e
--- /dev/null
+++ b/test/TEX/recursive_scanner_dependencies_input.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+#
+# __COPYRIGHT__
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
+
+"""Verify that we re-run LaTeX after changing a nested \input. This
+checks that recursive implicit dependencies are found correctly.
+"""
+
+import TestSCons
+
+test = TestSCons.TestSCons()
+
+pdflatex = test.where_is('pdflatex')
+
+if not pdflatex:
+    test.skip_test("Could not find pdflatex; skipping test(s).\n")
+
+test.write(['SConstruct'], """\
+env = Environment(tools=['pdftex', 'tex'])
+env.PDF('master.tex')
+""")
+
+test.write(['master.tex'], r"""
+\documentclass{article}
+\begin{document}
+\input{sub1}
+\end{document}
+""")
+
+test.write(['sub1.tex'], r"""
+\input{sub2}
+""")
+
+test.write(['sub2.tex'], r"""
+Sub-document 2 content
+""")
+
+test.run()
+
+pdf_output_1 = test.read('master.pdf')
+
+# Change sub2.tex, see if master.pdf is changed
+test.write(['sub2.tex'], r"""
+Sub-document 2 content -- updated
+""")
+
+test.run()
+
+pdf_output_2 = test.read('master.pdf')
+
+# If the PDF file is the same as it was previously, then it didn't
+# pick up the change in sub2.tex, so fail.
+test.fail_test(pdf_output_1 == pdf_output_2)
+
+# Double-check:  clean everything and rebuild from scratch, which
+# should force the PDF file to be the 1982 version.
+
+test.run(arguments='-c')
+test.run()
+
+pdf_output_3 = test.read('master.pdf')
+
+# If the PDF file is now different than the second run, modulo the
+# creation timestamp and the ID and some other PDF garp, then something
+# else odd has happened, so fail.
+
+pdf_output_2 = test.normalize_pdf(pdf_output_2)
+pdf_output_3 = test.normalize_pdf(pdf_output_3)
+
+if pdf_output_2 != pdf_output_3:
+    import sys
+    test.write('master.normalized.2.pdf', pdf_output_2)
+    test.write('master.normalized.3.pdf', pdf_output_3)
+    sys.stdout.write("***** 2 and 3 are different!\n")
+    sys.stdout.write(test.diff_substr(pdf_output_2, pdf_output_3, 80, 80)
+                     + '\n')
+    sys.stdout.write("Output from run 2:\n")
+    sys.stdout.write(test.stdout(-2) + '\n')
+    sys.stdout.write("Output from run 3:\n")
+    sys.stdout.write(test.stdout() + '\n')
+    sys.stdout.flush()
+    test.fail_test()
+
+test.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4: