Merge pull request #3472 from maiphi/latex_latin1

Fix crash when Latin-1 encoded Latex log file is read with Python 3

4 files changed, 28 insertions, 3 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index af05d35..617b3d0 100755
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -6,6 +6,11 @@
 
 RELEASE  VERSION/DATE TO BE FILLED IN LATER
 
+  From Philipp Maierhöfer:
+    - Avoid crash with UnicodeDecodeError on Python 3 when a Latex log file in
+      non-UTF-8 encoding (e.g. containing umlauts in Latin-1 encoding when
+      the fontenc package is included with \usepackage[T1]{fontenc}) is read.
+
   From Mathew Robinson:
 
     - Improved threading performance by ensuring NodeInfo is shared
diff --git a/src/engine/SCons/Tool/tex.py b/src/engine/SCons/Tool/tex.py
index 64b9d3b..5cf7bca 100644
--- a/src/engine/SCons/Tool/tex.py
+++ b/src/engine/SCons/Tool/tex.py
@@ -297,8 +297,8 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None
         logfilename = targetbase + '.log'
         logContent = ''
         if os.path.isfile(logfilename):
-            with open(logfilename, "r") as f:
-                logContent = f.read()
+            with open(logfilename, "rb") as f:
+                logContent = f.read().decode(errors='replace')
 
 
         # Read the fls file to find all .aux files
diff --git a/test/TEX/LATEX.py b/test/TEX/LATEX.py
index 553313e..dabe8b1 100644
--- a/test/TEX/LATEX.py
+++ b/test/TEX/LATEX.py
@@ -28,6 +28,8 @@ r"""
 Validate that we can set the LATEX string to our own utility, that
 the produced .dvi, .aux and .log files get removed by the -c option,
 and that we can use this to wrap calls to the real latex utility.
+Check that a log file with a warning encoded in non-UTF-8 (here: Latin-1)
+is read without throwing an error.
 """
 
 import TestSCons
@@ -193,6 +195,24 @@ This is the include file. mod %s
     test.must_not_exist('latexi.ilg')
 
 
+    test.write('SConstruct', """
+env = Environment()
+env.DVI('latin1log.tex')
+""")
+
+    # This will trigger an overfull hbox warning in the log file,
+    # containing the umlaut "o in Latin-1 ("T1 fontenc") encoding.
+    test.write('latin1log.tex', r"""
+\documentclass[12pt,a4paper]{article}
+\usepackage[T1]{fontenc}
+\begin{document}
+\"oxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+\end{document}
+""")
+
+    test.run(arguments = 'latin1log.dvi', stderr = None)
+    test.must_exist('latin1log.dvi')
+
 test.pass_test()
 
 # Local Variables:
diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py
index 81e03f3..9218f60 100644
--- a/testing/framework/TestCmd.py
+++ b/testing/framework/TestCmd.py
@@ -1528,7 +1528,7 @@ class TestCmd(object):
         # TODO: Run full tests on both platforms and see if this fixes failures
         # It seems that py3.6 still sets text mode if you set encoding.
         elif sys.version_info[0] == 3:  # TODO and sys.version_info[1] < 6:
-            stream = stream.decode('utf-8')
+            stream = stream.decode('utf-8', errors='replace')
             stream = stream.replace('\r\n', '\n')
         elif sys.version_info[0] == 2:
             stream = stream.replace('\r\n', '\n')