Merged in bdbaddog/scons (pull request #505)

Fix handling of non ascii/not utf-8 file contents for PY3 in get_text_contents()

2 files changed, 14 insertions, 3 deletions

diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py
index 8c1161d..c31ac6c 100644
--- a/src/engine/SCons/Node/FS.py
+++ b/src/engine/SCons/Node/FS.py
@@ -2654,9 +2654,12 @@ class File(Base):
         if contents[:len(codecs.BOM_UTF16_BE)] == codecs.BOM_UTF16_BE:
             return contents[len(codecs.BOM_UTF16_BE):].decode('utf-16-be')
         try:
-            return contents.decode()
-        except (UnicodeDecodeError, AttributeError) as e:
-            return contents
+            return contents.decode('utf-8')
+        except UnicodeDecodeError as e:
+            try:
+                return contents.decode('latin-1')
+            except UnicodeDecodeError as e:
+                return contents.decode('utf-8', error='backslashreplace')
 
 
     def get_content_hash(self):
diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py
index 399ac06..273f809 100644
--- a/src/engine/SCons/Node/FSTests.py
+++ b/src/engine/SCons/Node/FSTests.py
@@ -1315,6 +1315,14 @@ class FSTestCase(_tempdirTestCase):
         assert eval('f1.get_text_contents() == u"Foo\x1aBar"'), \
                f1.get_text_contents()
 
+        # Check for string which doesn't have BOM and isn't valid
+        # ASCII
+        test_string = b'Gan\xdfauge'
+        test.write('latin1_file', test_string)
+        f1 = fs.File(test.workpath("latin1_file"))
+        assert f1.get_text_contents() == test_string.decode('latin-1'), \
+               f1.get_text_contents()
+
         def nonexistent(method, s):
             try:
                 x = method(s, create = 0)