Issue #18037: 2to3 now escapes '\u' and '\U' in native strings.

3 files changed, 64 insertions, 7 deletions

diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py
index d2b3cee..6555397 100644
--- a/Lib/lib2to3/fixes/fix_unicode.py
+++ b/Lib/lib2to3/fixes/fix_unicode.py
@@ -1,25 +1,43 @@
-"""Fixer that changes unicode to str, unichr to chr, and u"..." into "...".
+r"""Fixer for unicode.
+
+* Changes unicode to str and unichr to chr.
+
+* If "...\u..." is not unicode literal change it into "...\\u...".
+
+* Change u"..." into "...".
 
 """
 
-import re
 from ..pgen2 import token
 from .. import fixer_base
 
 _mapping = {"unichr" : "chr", "unicode" : "str"}
-_literal_re = re.compile(r"[uU][rR]?[\'\"]")
 
 class FixUnicode(fixer_base.BaseFix):
     BM_compatible = True
     PATTERN = "STRING | 'unicode' | 'unichr'"
 
+    def start_tree(self, tree, filename):
+        super(FixUnicode, self).start_tree(tree, filename)
+        self.unicode_literals = 'unicode_literals' in tree.future_features
+
     def transform(self, node, results):
         if node.type == token.NAME:
             new = node.clone()
             new.value = _mapping[node.value]
             return new
         elif node.type == token.STRING:
-            if _literal_re.match(node.value):
-                new = node.clone()
-                new.value = new.value[1:]
-                return new
+            val = node.value
+            if (not self.unicode_literals and val[0] in 'rR\'"' and
+                '\\' in val):
+                val = r'\\'.join([
+                    v.replace('\\u', r'\\u').replace('\\U', r'\\U')
+                    for v in val.split(r'\\')
+                ])
+            if val[0] in 'uU':
+                val = val[1:]
+            if val == node.value:
+                return node
+            new = node.clone()
+            new.value = val
+            return new
diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py
index 2d20b08..bffb741 100644
--- a/Lib/lib2to3/tests/test_fixers.py
+++ b/Lib/lib2to3/tests/test_fixers.py
@@ -2824,6 +2824,43 @@ class Test_unicode(FixerTestCase):
         a = """R'''x''' """
         self.check(b, a)
 
+    def test_native_literal_escape_u(self):
+        b = r"""'\\\u20ac\U0001d121\\u20ac'"""
+        a = r"""'\\\\u20ac\\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+        b = r"""r'\\\u20ac\U0001d121\\u20ac'"""
+        a = r"""r'\\\\u20ac\\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+    def test_bytes_literal_escape_u(self):
+        b = r"""b'\\\u20ac\U0001d121\\u20ac'"""
+        a = r"""b'\\\u20ac\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+        b = r"""br'\\\u20ac\U0001d121\\u20ac'"""
+        a = r"""br'\\\u20ac\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+    def test_unicode_literal_escape_u(self):
+        b = r"""u'\\\u20ac\U0001d121\\u20ac'"""
+        a = r"""'\\\u20ac\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+        b = r"""ur'\\\u20ac\U0001d121\\u20ac'"""
+        a = r"""r'\\\u20ac\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+    def test_native_unicode_literal_escape_u(self):
+        f = 'from __future__ import unicode_literals\n'
+        b = f + r"""'\\\u20ac\U0001d121\\u20ac'"""
+        a = f + r"""'\\\u20ac\U0001d121\\u20ac'"""
+        self.check(b, a)
+
+        b = f + r"""r'\\\u20ac\U0001d121\\u20ac'"""
+        a = f + r"""r'\\\u20ac\U0001d121\\u20ac'"""
+        self.check(b, a)
+
 class Test_callable(FixerTestCase):
     fixer = "callable"
 
diff --git a/Misc/NEWS b/Misc/NEWS
index 48d5898..ad44fe7 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -71,6 +71,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #18037: 2to3 now escapes '\u' and '\U' in native strings.
+
 - Issue #19137: The pprint module now correctly formats instances of set and
   frozenset subclasses.