From 06f27e5d358ec5fd7045b98be25aef04031c587e Mon Sep 17 00:00:00 2001 From: Rick Lupton Date: Wed, 23 Nov 2016 21:53:03 +0000 Subject: Find dependencies using LaTeX "import" package commands The import package adds new commands for including files, similar to \input and \include, but with better handling of subdirectories. These changes extend the LaTeX Scanner to look for these commands. --- src/engine/SCons/Scanner/LaTeX.py | 98 +++++++++++------ src/engine/SCons/Scanner/LaTeXTests.py | 15 ++- test/TEX/recursive_scanner_dependencies_import.py | 122 ++++++++++++++++++++++ test/TEX/recursive_scanner_dependencies_input.py | 112 ++++++++++++++++++++ 4 files changed, 316 insertions(+), 31 deletions(-) create mode 100644 test/TEX/recursive_scanner_dependencies_import.py create mode 100644 test/TEX/recursive_scanner_dependencies_input.py diff --git a/src/engine/SCons/Scanner/LaTeX.py b/src/engine/SCons/Scanner/LaTeX.py index 2cb1ed5..0c54bf3 100644 --- a/src/engine/SCons/Scanner/LaTeX.py +++ b/src/engine/SCons/Scanner/LaTeX.py @@ -166,6 +166,9 @@ class LaTeX(SCons.Scanner.Base): 'usepackage': 'TEXINPUTS', 'lstinputlisting': 'TEXINPUTS'} env_variables = SCons.Util.unique(list(keyword_paths.values())) + two_arg_commands = ['import', 'subimport', + 'includefrom', 'subincludefrom', + 'inputfrom', 'subinputfrom'] def __init__(self, name, suffixes, graphics_extensions, *args, **kw): @@ -175,8 +178,29 @@ class LaTeX(SCons.Scanner.Base): # line followed by one or more newline characters (i.e. blank # lines), interfering with a match on the next line. # add option for whitespace before the '[options]' or the '{filename}' - regex = r'^[^%\n]*\\(include|includegraphics(?:\s*\[[^\]]+\])?|lstinputlisting(?:\[[^\]]+\])?|input|bibliography|addbibresource|addglobalbib|addsectionbib|usepackage)\s*{([^}]*)}' - self.cre = re.compile(regex, re.M) + regex = r''' + ^[^%\n]* + \\( + include + | includegraphics(?:\s*\[[^\]]+\])? + | lstinputlisting(?:\[[^\]]+\])? + | input + | import + | subimport + | includefrom + | subincludefrom + | inputfrom + | subinputfrom + | bibliography + | addbibresource + | addglobalbib + | addsectionbib + | usepackage + ) + \s*{([^}]*)} # first arg + (?: \s*{([^}]*)} )? # maybe another arg + ''' + self.cre = re.compile(regex, re.M | re.X) self.comment_re = re.compile(r'^((?:(?:\\%)|[^%\n])*)(.*)$', re.M) self.graphics_extensions = graphics_extensions @@ -236,23 +260,26 @@ class LaTeX(SCons.Scanner.Base): SCons.Scanner.Base.__init__(self, *args, **kw) - def _latex_names(self, include): - filename = include[1] - if include[0] == 'input': + def _latex_names(self, include_type, filename): + if include_type == 'input': base, ext = os.path.splitext( filename ) if ext == "": return [filename + '.tex'] - if (include[0] == 'include'): - return [filename + '.tex'] - if include[0] == 'bibliography': + if include_type in ('include', 'import', 'subimport', + 'includefrom', 'subincludefrom', + 'inputfrom', 'subinputfrom'): + base, ext = os.path.splitext( filename ) + if ext == "": + return [filename + '.tex'] + if include_type == 'bibliography': base, ext = os.path.splitext( filename ) if ext == "": return [filename + '.bib'] - if include[0] == 'usepackage': + if include_type == 'usepackage': base, ext = os.path.splitext( filename ) if ext == "": return [filename + '.sty'] - if include[0] == 'includegraphics': + if include_type == 'includegraphics': base, ext = os.path.splitext( filename ) if ext == "": #return [filename+e for e in self.graphics_extensions + TexGraphics] @@ -267,21 +294,26 @@ class LaTeX(SCons.Scanner.Base): return SCons.Node.FS._my_normcase(str(include)) def find_include(self, include, source_dir, path): + inc_type, inc_subdir, inc_filename = include try: - sub_path = path[include[0]] + sub_paths = path[inc_type] except (IndexError, KeyError): - sub_path = () - try_names = self._latex_names(include) + sub_paths = ((), ()) + try_names = self._latex_names(inc_type, inc_filename) + + # There are three search paths to try: + # 1. current directory "source_dir" + # 2. env[var] + # 3. env['ENV'][var] + search_paths = [(source_dir,)] + list(sub_paths) + for n in try_names: - # see if we find it using the path in env[var] - i = SCons.Node.FS.find_file(n, (source_dir,) + sub_path[0]) - if i: - return i, include - # see if we find it using the path in env['ENV'][var] - i = SCons.Node.FS.find_file(n, (source_dir,) + sub_path[1]) - if i: - return i, include - return i, include + for search_path in search_paths: + paths = tuple([d.Dir(inc_subdir) for d in search_path]) + i = SCons.Node.FS.find_file(n, paths) + if i: + return i, include + return None, include def canonical_text(self, text): """Standardize an input TeX-file contents. @@ -300,7 +332,7 @@ class LaTeX(SCons.Scanner.Base): line_continues_a_comment = len(comment) > 0 return '\n'.join(out).rstrip()+'\n' - def scan(self, node): + def scan(self, node, subdir='.'): # Modify the default scan function to allow for the regular # expression to return a comma separated list of file names # as can be the case with the bibliography keyword. @@ -326,9 +358,14 @@ class LaTeX(SCons.Scanner.Base): split_includes = [] for include in includes: inc_type = noopt_cre.sub('', include[0]) - inc_list = include[1].split(',') + inc_subdir = subdir + if inc_type in self.two_arg_commands: + inc_subdir = os.path.join(subdir, include[1]) + inc_list = include[2].split(',') + else: + inc_list = include[1].split(',') for j in range(len(inc_list)): - split_includes.append( (inc_type, inc_list[j]) ) + split_includes.append( (inc_type, inc_subdir, inc_list[j]) ) # includes = split_includes node.includes = includes @@ -359,11 +396,12 @@ class LaTeX(SCons.Scanner.Base): while queue: include = queue.pop() + inc_type, inc_subdir, inc_filename = include try: - if seen[include[1]] == 1: + if seen[inc_filename] == 1: continue except KeyError: - seen[include[1]] = 1 + seen[inc_filename] = 1 # # Handle multiple filenames in include[1] @@ -372,14 +410,14 @@ class LaTeX(SCons.Scanner.Base): if n is None: # Do not bother with 'usepackage' warnings, as they most # likely refer to system-level files - if include[0] != 'usepackage': + if inc_type != 'usepackage': SCons.Warnings.warn(SCons.Warnings.DependencyWarning, "No dependency generated for file: %s (included from: %s) -- file not found" % (i, node)) else: sortkey = self.sort_key(n) nodes.append((sortkey, n)) - # recurse down - queue.extend( self.scan(n) ) + # recurse down + queue.extend( self.scan(n, inc_subdir) ) return [pair[1] for pair in sorted(nodes)] diff --git a/src/engine/SCons/Scanner/LaTeXTests.py b/src/engine/SCons/Scanner/LaTeXTests.py index 49553cf..213e89e 100644 --- a/src/engine/SCons/Scanner/LaTeXTests.py +++ b/src/engine/SCons/Scanner/LaTeXTests.py @@ -44,6 +44,12 @@ test.write('test1.latex',""" include{incNO} %\include{incNO} xyzzy \include{inc6} +\subimport{subdir}{inc3} +\import{subdir}{inc3a} +\includefrom{subdir}{inc3b} +\subincludefrom{subdir}{inc3c} +\inputfrom{subdir}{inc3d} +\subinputfrom{subdir}{inc3e} """) test.write('test2.latex',""" @@ -61,6 +67,10 @@ test.subdir('subdir') test.write('inc1.tex',"\n") test.write('inc2.tex',"\n") test.write(['subdir', 'inc3.tex'], "\n") +for suffix in 'abcde': + test.write(['subdir', 'inc3%s.tex' % suffix], "\n") +test.write(['subdir', 'inc3b.tex'], "\n") +test.write(['subdir', 'inc3c.tex'], "\n") test.write(['subdir', 'inc4.eps'], "\n") test.write('inc5.xyz', "\n") test.write('inc6.tex', "\n") @@ -122,7 +132,10 @@ class LaTeXScannerTestCase1(unittest.TestCase): s = SCons.Scanner.LaTeX.LaTeXScanner() path = s.path(env) deps = s(env.File('test1.latex'), env, path) - headers = ['inc1.tex', 'inc2.tex', 'inc6.tex'] + headers = ['inc1.tex', 'inc2.tex', 'inc6.tex', + 'subdir/inc3.tex', 'subdir/inc3a.tex', + 'subdir/inc3b.tex', 'subdir/inc3c.tex', + 'subdir/inc3d.tex', 'subdir/inc3e.tex'] deps_match(self, deps, headers) class LaTeXScannerTestCase2(unittest.TestCase): diff --git a/test/TEX/recursive_scanner_dependencies_import.py b/test/TEX/recursive_scanner_dependencies_import.py new file mode 100644 index 0000000..d9d2625 --- /dev/null +++ b/test/TEX/recursive_scanner_dependencies_import.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +"""Verify that we re-run LaTeX after changing a nested \import. This +checks that recursive implicit dependencies are found correctly. + +This is a separate test from the +recursive_scanner_dependencies_input.py test because \input and +\include are handled specially by the PDF builder, whereas \import +dependencies are found only by the scanner. + +""" + +import TestSCons + +test = TestSCons.TestSCons() + +pdflatex = test.where_is('pdflatex') + +if not pdflatex: + test.skip_test("Could not find pdflatex; skipping test(s).\n") + +test.subdir('subdir') +test.subdir('subdir/subdir2') + +test.write(['SConstruct'], """\ +env = Environment(tools=['pdftex', 'tex']) +env.PDF('master.tex') +""") + +test.write(['master.tex'], r""" +\documentclass{article} +\usepackage{import} +\begin{document} +\subinputfrom{subdir/}{sub1} +\end{document} +""") + +test.write(['subdir', 'sub1.tex'], r""" +\subinputfrom{subdir2/}{sub2} +""") + +test.write(['subdir', 'subdir2', 'sub2.tex'], r""" +Sub-document 2 content +""") + +test.run() + +pdf_output_1 = test.read('master.pdf') + +# Change sub2.tex, see if master.pdf is changed +test.write(['subdir', 'subdir2', 'sub2.tex'], r""" +Sub-document 2 content -- updated +""") + +test.run() + +pdf_output_2 = test.read('master.pdf') + +# If the PDF file is the same as it was previously, then it didn't +# pick up the change in sub2.tex, so fail. +test.fail_test(pdf_output_1 == pdf_output_2) + +# Double-check: clean everything and rebuild from scratch, which +# should force the PDF file to be the 1982 version. + +test.run(arguments='-c') +test.run() + +pdf_output_3 = test.read('master.pdf') + +# If the PDF file is now different than the second run, modulo the +# creation timestamp and the ID and some other PDF garp, then something +# else odd has happened, so fail. + +pdf_output_2 = test.normalize_pdf(pdf_output_2) +pdf_output_3 = test.normalize_pdf(pdf_output_3) + +if pdf_output_2 != pdf_output_3: + import sys + test.write('master.normalized.2.pdf', pdf_output_2) + test.write('master.normalized.3.pdf', pdf_output_3) + sys.stdout.write("***** 2 and 3 are different!\n") + sys.stdout.write(test.diff_substr(pdf_output_2, pdf_output_3, 80, 80) + + '\n') + sys.stdout.write("Output from run 2:\n") + sys.stdout.write(test.stdout(-2) + '\n') + sys.stdout.write("Output from run 3:\n") + sys.stdout.write(test.stdout() + '\n') + sys.stdout.flush() + test.fail_test() + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/TEX/recursive_scanner_dependencies_input.py b/test/TEX/recursive_scanner_dependencies_input.py new file mode 100644 index 0000000..257051e --- /dev/null +++ b/test/TEX/recursive_scanner_dependencies_input.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +"""Verify that we re-run LaTeX after changing a nested \input. This +checks that recursive implicit dependencies are found correctly. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +pdflatex = test.where_is('pdflatex') + +if not pdflatex: + test.skip_test("Could not find pdflatex; skipping test(s).\n") + +test.write(['SConstruct'], """\ +env = Environment(tools=['pdftex', 'tex']) +env.PDF('master.tex') +""") + +test.write(['master.tex'], r""" +\documentclass{article} +\begin{document} +\input{sub1} +\end{document} +""") + +test.write(['sub1.tex'], r""" +\input{sub2} +""") + +test.write(['sub2.tex'], r""" +Sub-document 2 content +""") + +test.run() + +pdf_output_1 = test.read('master.pdf') + +# Change sub2.tex, see if master.pdf is changed +test.write(['sub2.tex'], r""" +Sub-document 2 content -- updated +""") + +test.run() + +pdf_output_2 = test.read('master.pdf') + +# If the PDF file is the same as it was previously, then it didn't +# pick up the change in sub2.tex, so fail. +test.fail_test(pdf_output_1 == pdf_output_2) + +# Double-check: clean everything and rebuild from scratch, which +# should force the PDF file to be the 1982 version. + +test.run(arguments='-c') +test.run() + +pdf_output_3 = test.read('master.pdf') + +# If the PDF file is now different than the second run, modulo the +# creation timestamp and the ID and some other PDF garp, then something +# else odd has happened, so fail. + +pdf_output_2 = test.normalize_pdf(pdf_output_2) +pdf_output_3 = test.normalize_pdf(pdf_output_3) + +if pdf_output_2 != pdf_output_3: + import sys + test.write('master.normalized.2.pdf', pdf_output_2) + test.write('master.normalized.3.pdf', pdf_output_3) + sys.stdout.write("***** 2 and 3 are different!\n") + sys.stdout.write(test.diff_substr(pdf_output_2, pdf_output_3, 80, 80) + + '\n') + sys.stdout.write("Output from run 2:\n") + sys.stdout.write(test.stdout(-2) + '\n') + sys.stdout.write("Output from run 3:\n") + sys.stdout.write(test.stdout() + '\n') + sys.stdout.flush() + test.fail_test() + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12