diff options
Diffstat (limited to 'src/engine')
| -rw-r--r-- | src/engine/SCons/Node/FS.py | 2 | ||||
| -rw-r--r-- | src/engine/SCons/Node/__init__.py | 3 | ||||
| -rw-r--r-- | src/engine/SCons/Taskmaster.py | 31 | ||||
| -rw-r--r-- | src/engine/SCons/TaskmasterTests.py | 91 | ||||
| -rw-r--r-- | src/engine/SCons/Tool/tex.py | 38 |
5 files changed, 143 insertions, 22 deletions
diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f106d46..f31ca83 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -2715,7 +2715,7 @@ class File(Base): so only do thread safe stuff here. Do thread unsafe stuff in built(). - Returns true iff the node was successfully retrieved. + Returns true if the node was successfully retrieved. """ if self.nocache: return None diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 2fbe2c6..992284d 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -216,6 +216,7 @@ class Node(object): self.precious = None self.noclean = 0 self.nocache = 0 + self.cached = 0 # is this node pulled from cache? self.always_build = None self.includes = None self.attributes = self.Attrs() # Generic place to stick information about the Node. @@ -306,7 +307,7 @@ class Node(object): so only do thread safe stuff here. Do thread unsafe stuff in built(). - Returns true iff the node was successfully retrieved. + Returns true if the node was successfully retrieved. """ return 0 diff --git a/src/engine/SCons/Taskmaster.py b/src/engine/SCons/Taskmaster.py index 58a8d90..64ab84d 100644 --- a/src/engine/SCons/Taskmaster.py +++ b/src/engine/SCons/Taskmaster.py @@ -227,20 +227,26 @@ class Task(object): if T: T.write(self.trace_message(u'Task.execute()', self.node)) try: - everything_was_cached = 1 + cached_targets = [] for t in self.targets: - if t.retrieve_from_cache(): - # Call the .built() method without calling the - # .push_to_cache() method, since we just got the - # target from the cache and don't need to push - # it back there. - t.set_state(NODE_EXECUTED) - t.built() - else: - everything_was_cached = 0 + if not t.retrieve_from_cache(): break - if not everything_was_cached: + cached_targets.append(t) + if len(cached_targets) < len(self.targets): + # Remove targets before building. It's possible that we + # partially retrieved targets from the cache, leaving + # them in read-only mode. That might cause the command + # to fail. + # + for t in cached_targets: + try: + t.fs.unlink(t.path) + except (IOError, OSError): + pass self.targets[0].build() + else: + for t in cached_targets: + t.cached = 1 except SystemExit: exc_value = sys.exc_info()[1] raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) @@ -292,7 +298,8 @@ class Task(object): for side_effect in t.side_effects: side_effect.set_state(NODE_NO_STATE) t.set_state(NODE_EXECUTED) - t.push_to_cache() + if not t.cached: + t.push_to_cache() t.built() t.visited() diff --git a/src/engine/SCons/TaskmasterTests.py b/src/engine/SCons/TaskmasterTests.py index 66c6b9c..85ade8d 100644 --- a/src/engine/SCons/TaskmasterTests.py +++ b/src/engine/SCons/TaskmasterTests.py @@ -86,16 +86,61 @@ class Node(object): def prepare(self): self.prepared = 1 + self.get_binfo() def build(self): global built_text built_text = self.name + " built" + def remove(self): + pass + + # The following four methods new_binfo(), del_binfo(), + # get_binfo(), clear() as well as its calls have been added + # to support the cached_execute() test (issue #2720). + # They are full copies (or snippets) of their actual + # counterparts in the Node class... + def new_binfo(self): + binfo = "binfo" + return binfo + + def del_binfo(self): + """Delete the build info from this node.""" + try: + delattr(self, 'binfo') + except AttributeError: + pass + + def get_binfo(self): + """Fetch a node's build information.""" + try: + return self.binfo + except AttributeError: + pass + + binfo = self.new_binfo() + self.binfo = binfo + + return binfo + + def clear(self): + # The del_binfo() call here isn't necessary for normal execution, + # but is for interactive mode, where we might rebuild the same + # target and need to start from scratch. + self.del_binfo() + def built(self): global built_text if not self.cached: built_text = built_text + " really" - + + # Clear the implicit dependency caches of any Nodes + # waiting for this Node to be built. + for parent in self.waiting_parents: + parent.implicit = None + + self.clear() + def has_builder(self): return not self.builder is None @@ -954,6 +999,50 @@ class TaskmasterTestCase(unittest.TestCase): assert built_text is None, built_text assert cache_text == ["n7 retrieved", "n8 retrieved"], cache_text + def test_cached_execute(self): + """Test executing a task with cached targets + """ + # In issue #2720 Alexei Klimkin detected that the previous + # workflow for execute() led to problems in a multithreaded build. + # We have: + # task.prepare() + # task.execute() + # task.executed() + # -> node.visited() + # for the Serial flow, but + # - Parallel - - Worker - + # task.prepare() + # requestQueue.put(task) + # task = requestQueue.get() + # task.execute() + # resultQueue.put(task) + # task = resultQueue.get() + # task.executed() + # ->node.visited() + # in parallel. Since execute() used to call built() when a target + # was cached, it could unblock dependent nodes before the binfo got + # restored again in visited(). This resulted in spurious + # "file not found" build errors, because files fetched from cache would + # be seen as not up to date and wouldn't be scanned for implicit + # dependencies. + # + # The following test ensures that execute() only marks targets as cached, + # but the actual call to built() happens in executed() only. + # Like this, the binfo should still be intact after calling execute()... + global cache_text + + n1 = Node("n1") + # Mark the node as being cached + n1.cached = 1 + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + t.prepare() + t.execute() + assert cache_text == ["n1 retrieved"], cache_text + # If no binfo exists anymore, something has gone wrong... + has_binfo = hasattr(n1, 'binfo') + assert has_binfo == True, has_binfo + def test_exception(self): """Test generic Taskmaster exception handling diff --git a/src/engine/SCons/Tool/tex.py b/src/engine/SCons/Tool/tex.py index b0c518e..0d871ad 100644 --- a/src/engine/SCons/Tool/tex.py +++ b/src/engine/SCons/Tool/tex.py @@ -86,6 +86,7 @@ tableofcontents_re = re.compile(r"^[^%\n]*\\tableofcontents", re.MULTILINE) makeindex_re = re.compile(r"^[^%\n]*\\makeindex", re.MULTILINE) bibliography_re = re.compile(r"^[^%\n]*\\bibliography", re.MULTILINE) bibunit_re = re.compile(r"^[^%\n]*\\begin\{bibunit\}", re.MULTILINE) +multibib_re = re.compile(r"^[^%\n]*\\newcites\{([^\}]*)\}", re.MULTILINE) listoffigures_re = re.compile(r"^[^%\n]*\\listoffigures", re.MULTILINE) listoftables_re = re.compile(r"^[^%\n]*\\listoftables", re.MULTILINE) hyperref_re = re.compile(r"^[^%\n]*\\usepackage.*\{hyperref\}", re.MULTILINE) @@ -236,6 +237,9 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None must_rerun_latex = True + # .aux files already processed by BibTex + already_bibtexed = [] + # # routine to update MD5 hash and compare # @@ -298,21 +302,23 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None # The information that bibtex reads from the .aux file is # pass-independent. If we find (below) that the .bbl file is unchanged, # then the last latex saw a correct bibliography. - # Therefore only do this on the first pass - if count == 1: - for auxfilename in auxfiles: + # Therefore only do this once + # Go through all .aux files and remember the files already done. + for auxfilename in auxfiles: + if auxfilename not in already_bibtexed: + already_bibtexed.append(auxfilename) target_aux = os.path.join(targetdir, auxfilename) if os.path.isfile(target_aux): content = open(target_aux, "rb").read() if content.find("bibdata") != -1: if Verbose: - print "Need to run bibtex" + print "Need to run bibtex on ",auxfilename bibfile = env.fs.File(SCons.Util.splitext(target_aux)[0]) result = BibTeXAction(bibfile, bibfile, env) if result != 0: check_file_error_message(env['BIBTEX'], 'blg') - must_rerun_latex = must_rerun_latex or check_MD5(suffix_nodes['.bbl'],'.bbl') - + #must_rerun_latex = must_rerun_latex or check_MD5(suffix_nodes['.bbl'],'.bbl') + must_rerun_latex = True # Now decide if latex will need to be run again due to index. if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex): # We must run makeindex @@ -553,6 +559,8 @@ def ScanFiles(theFile, target, paths, file_tests, file_tests_search, env, graphi for i in range(len(file_tests_search)): if file_tests[i][0] is None: file_tests[i][0] = file_tests_search[i].search(content) + if Verbose and file_tests[i][0]: + print " found match for ",file_tests[i][-1][-1] incResult = includeOnly_re.search(content) if incResult: @@ -621,6 +629,7 @@ def tex_emitter_core(target, source, env, graphics_extensions): makeindex_re, bibliography_re, bibunit_re, + multibib_re, tableofcontents_re, listoffigures_re, listoftables_re, @@ -636,6 +645,7 @@ def tex_emitter_core(target, source, env, graphics_extensions): ['.idx', '.ind', '.ilg','makeindex'], ['.bbl', '.blg','bibliography'], ['.bbl', '.blg','bibunit'], + ['.bbl', '.blg','multibib'], ['.toc','contents'], ['.lof','figures'], ['.lot','tables'], @@ -678,6 +688,8 @@ def tex_emitter_core(target, source, env, graphics_extensions): for (theSearch,suffix_list) in file_tests: # add side effects if feature is present.If file is to be generated,add all side effects + if Verbose and theSearch: + print "check side effects for ",suffix_list[-1] if (theSearch != None) or (not source[0].exists() ): file_list = [targetbase,] # for bibunit we need a list of files @@ -687,6 +699,18 @@ def tex_emitter_core(target, source, env, graphics_extensions): # remove the suffix '.aux' for i in range(len(file_list)): file_list[i] = SCons.Util.splitext(file_list[i])[0] + # for multibib we need a list of files + if suffix_list[-1] == 'multibib': + file_list = [] + for multibibmatch in multibib_re.finditer(content): + if Verbose: + print "multibib match ",multibibmatch.group(1) + if multibibmatch != None: + baselist = multibibmatch.group(1).split(',') + if Verbose: + print "multibib list ", baselist + for i in range(len(baselist)): + file_list.append(os.path.join(targetdir, baselist[i])) # now define the side effects for file_name in file_list: for suffix in suffix_list[:-1]: @@ -826,7 +850,7 @@ def generate_common(env): env['LATEX'] = 'latex' env['LATEXFLAGS'] = SCons.Util.CLVar('-interaction=nonstopmode -recorder') env['LATEXCOM'] = CDCOM + '${TARGET.dir} && $LATEX $LATEXFLAGS ${SOURCE.file}' - env['LATEXRETRIES'] = 3 + env['LATEXRETRIES'] = 4 env['PDFLATEX'] = 'pdflatex' env['PDFLATEXFLAGS'] = SCons.Util.CLVar('-interaction=nonstopmode -recorder') |