Patch #2477: Added from __future__ import unicode_literals

The new PyParser_*Ex() functions are based on Neal's suggestion and initial patch. The new __future__ feature makes all '' and r'' unicode strings. b'' and br'' stay (byte) strings.

4 files changed, 41 insertions, 19 deletions

diff --git a/Python/ast.c b/Python/ast.c
index 1fc2324..bc91805 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -18,6 +18,7 @@
 /* Data structure used internally */
 struct compiling {
     char *c_encoding; /* source encoding */
+    int c_future_unicode; /* __future__ unicode literals flag */
     PyArena *c_arena; /* arena for allocating memeory */
     const char *c_filename; /* filename */
 };
@@ -36,7 +37,7 @@ static expr_ty ast_for_testlist_gexp(struct compiling *, const node *);
 static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
 
 static PyObject *parsenumber(const char *);
-static PyObject *parsestr(const char *s, const char *encoding);
+static PyObject *parsestr(struct compiling *, const char *);
 static PyObject *parsestrplus(struct compiling *, const node *n);
 
 #ifndef LINENO
@@ -198,6 +199,7 @@ PyAST_FromNode(const node *n, PyCompilerFlags *flags, const char *filename,
     } else {
         c.c_encoding = NULL;
     }
+    c.c_future_unicode = flags && flags->cf_flags & CO_FUTURE_UNICODE_LITERALS;
     c.c_arena = arena;
     c.c_filename = filename;
 
@@ -3247,13 +3249,13 @@ decode_unicode(const char *s, size_t len, int rawmode, const char *encoding)
  * parsestr parses it, and returns the decoded Python string object.
  */
 static PyObject *
-parsestr(const char *s, const char *encoding)
+parsestr(struct compiling *c, const char *s)
 {
         size_t len;
         int quote = Py_CHARMASK(*s);
         int rawmode = 0;
         int need_encoding;
-        int unicode = 0;
+        int unicode = c->c_future_unicode;
 
         if (isalpha(quote) || quote == '_') {
                 if (quote == 'u' || quote == 'U') {
@@ -3262,6 +3264,7 @@ parsestr(const char *s, const char *encoding)
                 }
                 if (quote == 'b' || quote == 'B') {
                         quote = *++s;
+                        unicode = 0;
                 }
                 if (quote == 'r' || quote == 'R') {
                         quote = *++s;
@@ -3293,12 +3296,12 @@ parsestr(const char *s, const char *encoding)
         }
 #ifdef Py_USING_UNICODE
         if (unicode || Py_UnicodeFlag) {
-                return decode_unicode(s, len, rawmode, encoding);
+                return decode_unicode(s, len, rawmode, c->c_encoding);
         }
 #endif
-        need_encoding = (encoding != NULL &&
-                         strcmp(encoding, "utf-8") != 0 &&
-                         strcmp(encoding, "iso-8859-1") != 0);
+        need_encoding = (c->c_encoding != NULL &&
+                         strcmp(c->c_encoding, "utf-8") != 0 &&
+                         strcmp(c->c_encoding, "iso-8859-1") != 0);
         if (rawmode || strchr(s, '\\') == NULL) {
                 if (need_encoding) {
 #ifndef Py_USING_UNICODE
@@ -3310,7 +3313,7 @@ parsestr(const char *s, const char *encoding)
                         PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
                         if (u == NULL)
                                 return NULL;
-                        v = PyUnicode_AsEncodedString(u, encoding, NULL);
+                        v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL);
                         Py_DECREF(u);
                         return v;
 #endif
@@ -3320,7 +3323,7 @@ parsestr(const char *s, const char *encoding)
         }
 
         return PyString_DecodeEscape(s, len, NULL, unicode,
-                                     need_encoding ? encoding : NULL);
+                                     need_encoding ? c->c_encoding : NULL);
 }
 
 /* Build a Python string object out of a STRING atom.  This takes care of
@@ -3333,11 +3336,11 @@ parsestrplus(struct compiling *c, const node *n)
         PyObject *v;
         int i;
         REQ(CHILD(n, 0), STRING);
-        if ((v = parsestr(STR(CHILD(n, 0)), c->c_encoding)) != NULL) {
+        if ((v = parsestr(c, STR(CHILD(n, 0)))) != NULL) {
                 /* String literal concatenation */
                 for (i = 1; i < NCH(n); i++) {
                         PyObject *s;
-                        s = parsestr(STR(CHILD(n, i)), c->c_encoding);
+                        s = parsestr(c, STR(CHILD(n, i)));
                         if (s == NULL)
                                 goto onError;
                         if (PyString_Check(v) && PyString_Check(s)) {
diff --git a/Python/future.c b/Python/future.c
index 267e1b7..2c6aaa2 100644
--- a/Python/future.c
+++ b/Python/future.c
@@ -35,6 +35,8 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename)
 			ff->ff_features |= CO_FUTURE_WITH_STATEMENT;
 		} else if (strcmp(feature, FUTURE_PRINT_FUNCTION) == 0) {
 			ff->ff_features |= CO_FUTURE_PRINT_FUNCTION;
+		} else if (strcmp(feature, FUTURE_UNICODE_LITERALS) == 0) {
+			ff->ff_features |= CO_FUTURE_UNICODE_LITERALS;
 		} else if (strcmp(feature, "braces") == 0) {
 			PyErr_SetString(PyExc_SyntaxError,
 					"not a chance");
diff --git a/Python/import.c b/Python/import.c
index ecbec15..95cd20d 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -818,11 +818,12 @@ parse_source_module(const char *pathname, FILE *fp)
 {
 	PyCodeObject *co = NULL;
 	mod_ty mod;
+	PyCompilerFlags flags;
 	PyArena *arena = PyArena_New();
 	if (arena == NULL)
 		return NULL;
 
-	mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0, 
+	mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, &flags, 
 				   NULL, arena);
 	if (mod) {
 		co = PyAST_Compile(mod, pathname, NULL, arena);
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 226fee3..423aae1 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -774,8 +774,11 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 #define PARSER_FLAGS(flags) \
 	((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
 		      PyPARSE_DONT_IMPLY_DEDENT : 0) \
-		    | ((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION ? \
-		       PyPARSE_PRINT_IS_FUNCTION : 0)) : 0)
+		    | (((flags)->cf_flags & CO_FUTURE_PRINT_FUNCTION) ? \
+		       PyPARSE_PRINT_IS_FUNCTION : 0) \
+		    | (((flags)->cf_flags & CO_FUTURE_UNICODE_LITERALS) ? \
+		       PyPARSE_UNICODE_LITERALS : 0) \
+		    ) : 0)
 #endif
 
 int
@@ -1390,11 +1393,12 @@ Py_SymtableString(const char *str, const char *filename, int start)
 {
 	struct symtable *st;
 	mod_ty mod;
+	PyCompilerFlags flags;
 	PyArena *arena = PyArena_New();
 	if (arena == NULL)
 		return NULL;
 
-	mod = PyParser_ASTFromString(str, filename, start, NULL, arena);
+	mod = PyParser_ASTFromString(str, filename, start, &flags, arena);
 	if (mod == NULL) {
 		PyArena_Free(arena);
 		return NULL;
@@ -1411,10 +1415,16 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
 {
 	mod_ty mod;
 	perrdetail err;
-	node *n = PyParser_ParseStringFlagsFilename(s, filename,
+	int iflags;
+	iflags = PARSER_FLAGS(flags);
+
+	node *n = PyParser_ParseStringFlagsFilenameEx(s, filename,
 					&_PyParser_Grammar, start, &err,
-					PARSER_FLAGS(flags));
+					&iflags);
 	if (n) {
+		if (flags) {
+			flags->cf_flags |= iflags & PyCF_MASK;
+		}
 		mod = PyAST_FromNode(n, flags, filename, arena);
 		PyNode_Free(n);
 		return mod;
@@ -1432,9 +1442,15 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
 {
 	mod_ty mod;
 	perrdetail err;
-	node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
-				start, ps1, ps2, &err, PARSER_FLAGS(flags));
+	int iflags;
+
+	iflags = PARSER_FLAGS(flags);
+	node *n = PyParser_ParseFileFlagsEx(fp, filename, &_PyParser_Grammar,
+				start, ps1, ps2, &err, &iflags);
 	if (n) {
+		if (flags) {
+			flags->cf_flags |= iflags & PyCF_MASK;
+		}
 		mod = PyAST_FromNode(n, flags, filename, arena);
 		PyNode_Free(n);
 		return mod;