Merge pull request #3568 from datalogics-robb/master

Specify encoding for Java files, as the default varies

3 files changed, 29 insertions, 1 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 20b99c4..7c5c9c2 100755
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -10,6 +10,10 @@ NOTE: Please include a reference to any Issues resolved by your changes in the b
 
 RELEASE  VERSION/DATE TO BE FILLED IN LATER
 
+From Rob Boehne
+    - Specify UTF-8 encoding when opening Java source file as text.  By default, encoding is the output
+    of locale.getpreferredencoding(False), and varies by platform.
+
   From William Deegan:
     - Fix broken clang + MSVC 2019 combination by using MSVC configuration logic to 
       propagate'VCINSTALLDIR' and 'VCToolsInstallDir' which clang tools use to locate
diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py
index 1711de1..c6c19ae 100644
--- a/src/engine/SCons/Tool/JavaCommon.py
+++ b/src/engine/SCons/Tool/JavaCommon.py
@@ -404,7 +404,7 @@ if java_parsing:
 
 
     def parse_java_file(fn, version=default_java_version):
-        with open(fn, 'r') as f:
+        with open(fn, 'r', encoding='utf-8') as f:
             data = f.read()
         return parse_java(data, version)
 
diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py
index 9242624..b0a788e 100644
--- a/src/engine/SCons/Tool/JavaCommonTests.py
+++ b/src/engine/SCons/Tool/JavaCommonTests.py
@@ -68,6 +68,30 @@ public class Foo
         assert classes == ['Foo'], classes
 
 
+    def test_file_parser(self):
+        """Test the file parser"""
+        input = """\
+package com.sub.bar;
+
+public class Foo
+{
+     public static void main(String[] args)
+     {
+        /* This tests that unicde is handled . */
+        String hello1 = new String("ఎత్తువెడల్పు");
+     }
+}
+"""
+        file_name = 'test_file_parser.java'
+        with open(file_name, 'w', encoding='UTF-8') as jf:
+            print(input, file=jf)
+
+        pkg_dir, classes = SCons.Tool.JavaCommon.parse_java_file(file_name)
+        if os.path.exists(file_name):
+            os.remove(file_name)
+        assert pkg_dir == os.path.join('com', 'sub', 'bar'), pkg_dir
+        assert classes == ['Foo'], classes
+
 
     def test_dollar_sign(self):
         """Test class names with $ in them"""