Implement PEP 393.

14 files changed, 1824 insertions, 350 deletions

diff --git a/Python/_warnings.c b/Python/_warnings.c
index f8a7175..2bcca91 100644
--- a/Python/_warnings.c
+++ b/Python/_warnings.c
@@ -498,17 +498,19 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno,
     *filename = PyDict_GetItemString(globals, "__file__");
     if (*filename != NULL && PyUnicode_Check(*filename)) {
         Py_ssize_t len = PyUnicode_GetSize(*filename);
-        Py_UNICODE *unicode = PyUnicode_AS_UNICODE(*filename);
+        int kind = PyUnicode_KIND(*filename);
+        void *data = PyUnicode_DATA(*filename);
 
         /* if filename.lower().endswith((".pyc", ".pyo")): */
         if (len >= 4 &&
-            unicode[len-4] == '.' &&
-            Py_UNICODE_TOLOWER(unicode[len-3]) == 'p' &&
-            Py_UNICODE_TOLOWER(unicode[len-2]) == 'y' &&
-            (Py_UNICODE_TOLOWER(unicode[len-1]) == 'c' ||
-                Py_UNICODE_TOLOWER(unicode[len-1]) == 'o'))
+            PyUnicode_READ(kind, data, len-4) == '.' &&
+            Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-3)) == 'p' &&
+            Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-2)) == 'y' &&
+            (Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-1)) == 'c' ||
+                Py_UNICODE_TOLOWER(PyUnicode_READ(kind, data, len-1)) == 'o'))
         {
-            *filename = PyUnicode_FromUnicode(unicode, len-1);
+            *filename = PyUnicode_Substring(*filename, 0,
+                                            PyUnicode_GET_LENGTH(*filename)-1);
             if (*filename == NULL)
                 goto handle_error;
         }
diff --git a/Python/ast.c b/Python/ast.c
index 14500ee..a52fd09 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -528,26 +528,21 @@ static identifier
 new_identifier(const char* n, PyArena *arena)
 {
     PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
-    Py_UNICODE *u;
-    if (!id)
+    if (!id || PyUnicode_READY(id) == -1)
         return NULL;
-    u = PyUnicode_AS_UNICODE(id);
     /* Check whether there are non-ASCII characters in the
        identifier; if so, normalize to NFKC. */
-    for (; *u; u++) {
-        if (*u >= 128) {
-            PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
-            PyObject *id2;
-            if (!m)
-                return NULL;
-            id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
-            Py_DECREF(m);
-            if (!id2)
-                return NULL;
-            Py_DECREF(id);
-            id = id2;
-            break;
-        }
+    if (PyUnicode_MAX_CHAR_VALUE((PyUnicodeObject *)id) >= 128) {
+        PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
+        PyObject *id2;
+        if (!m)
+            return NULL;
+        id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
+        Py_DECREF(m);
+        if (!id2)
+            return NULL;
+        Py_DECREF(id);
+        id = id2;
     }
     PyUnicode_InternInPlace(&id);
     PyArena_AddPyObject(arena, id);
@@ -3660,20 +3655,14 @@ parsenumber(struct compiling *c, const char *s)
 }
 
 static PyObject *
-decode_utf8(struct compiling *c, const char **sPtr, const char *end, char* encoding)
+decode_utf8(struct compiling *c, const char **sPtr, const char *end)
 {
-    PyObject *u, *v;
     char *s, *t;
     t = s = (char *)*sPtr;
     /* while (s < end && *s != '\\') s++; */ /* inefficient for u".." */
     while (s < end && (*s & 0x80)) s++;
     *sPtr = s;
-    u = PyUnicode_DecodeUTF8(t, s - t, NULL);
-    if (u == NULL)
-        return NULL;
-    v = PyUnicode_AsEncodedString(u, encoding, NULL);
-    Py_DECREF(u);
-    return v;
+    return PyUnicode_DecodeUTF8(t, s - t, NULL);
 }
 
 static PyObject *
@@ -3707,22 +3696,20 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons
             }
             if (*s & 0x80) { /* XXX inefficient */
                 PyObject *w;
-                char *r;
-                Py_ssize_t rn, i;
-                w = decode_utf8(c, &s, end, "utf-32-be");
+                int kind;
+                void *data;
+                Py_ssize_t len, i;
+                w = decode_utf8(c, &s, end);
                 if (w == NULL) {
                     Py_DECREF(u);
                     return NULL;
                 }
-                r = PyBytes_AS_STRING(w);
-                rn = Py_SIZE(w);
-                assert(rn % 4 == 0);
-                for (i = 0; i < rn; i += 4) {
-                    sprintf(p, "\\U%02x%02x%02x%02x",
-                            r[i + 0] & 0xFF,
-                            r[i + 1] & 0xFF,
-                            r[i + 2] & 0xFF,
-                            r[i + 3] & 0xFF);
+                kind = PyUnicode_KIND(w);
+                data = PyUnicode_DATA(w);
+                len = PyUnicode_GET_LENGTH(w);
+                for (i = 0; i < len; i++) {
+                    Py_UCS4 chr = PyUnicode_READ(kind, data, i);
+                    sprintf(p, "\\U%08x", chr);
                     p += 10;
                 }
                 /* Should be impossible to overflow */
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 9de8ca0..04f5231 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -508,8 +508,8 @@ source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf)
 
     if (PyUnicode_Check(cmd)) {
         cf->cf_flags |= PyCF_IGNORE_COOKIE;
-        cmd = _PyUnicode_AsDefaultEncodedString(cmd);
-        if (cmd == NULL)
+        str = PyUnicode_AsUTF8AndSize(cmd, &size);
+        if (str == NULL)
             return NULL;
     }
     else if (!PyObject_CheckReadBuffer(cmd)) {
@@ -518,9 +518,10 @@ source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf)
           funcname, what);
         return NULL;
     }
-    if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) {
+    else if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) {
         return NULL;
     }
+
     if (strlen(str) != size) {
         PyErr_SetString(PyExc_TypeError,
                         "source code string cannot contain null bytes");
@@ -1395,24 +1396,13 @@ builtin_ord(PyObject *self, PyObject* obj)
         }
     }
     else if (PyUnicode_Check(obj)) {
-        size = PyUnicode_GET_SIZE(obj);
+        if (PyUnicode_READY(obj) == -1)
+            return NULL;
+        size = PyUnicode_GET_LENGTH(obj);
         if (size == 1) {
-            ord = (long)*PyUnicode_AS_UNICODE(obj);
+            ord = (long)PyUnicode_READ_CHAR(obj, 0);
             return PyLong_FromLong(ord);
         }
-#ifndef Py_UNICODE_WIDE
-        if (size == 2) {
-            /* Decode a valid surrogate pair */
-            int c0 = PyUnicode_AS_UNICODE(obj)[0];
-            int c1 = PyUnicode_AS_UNICODE(obj)[1];
-            if (0xD800 <= c0 && c0 <= 0xDBFF &&
-                0xDC00 <= c1 && c1 <= 0xDFFF) {
-                ord = ((((c0 & 0x03FF) << 10) | (c1 & 0x03FF)) +
-                       0x00010000);
-                return PyLong_FromLong(ord);
-            }
-        }
-#endif
     }
     else if (PyByteArray_Check(obj)) {
         /* XXX Hopefully this is temporary */
diff --git a/Python/ceval.c b/Python/ceval.c
index 1c9153e..0a346f4 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -2054,7 +2054,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 /* Inline the PyDict_GetItem() calls.
                    WARNING: this is an extreme speed hack.
                    Do not try this at home. */
-                Py_hash_t hash = ((PyUnicodeObject *)w)->hash;
+                Py_hash_t hash = ((PyASCIIObject *)w)->hash;
                 if (hash != -1) {
                     PyDictObject *d;
                     PyDictEntry *e;
@@ -4456,7 +4456,8 @@ import_all_from(PyObject *locals, PyObject *v)
         }
         if (skip_leading_underscores &&
             PyUnicode_Check(name) &&
-            PyUnicode_AS_UNICODE(name)[0] == '_')
+            PyUnicode_READY(name) != -1 &&
+            PyUnicode_READ_CHAR(name, 0) == '_')
         {
             Py_DECREF(name);
             continue;
@@ -4520,6 +4521,14 @@ unicode_concatenate(PyObject *v, PyObject *w,
 {
     /* This function implements 'variable += expr' when both arguments
        are (Unicode) strings. */
+
+    w = PyUnicode_Concat(v, w);
+    Py_DECREF(v);
+    return w;
+
+    /* XXX: This optimization is currently disabled as unicode objects in the
+       new flexible representation are not in-place resizable anymore. */
+#if 0
     Py_ssize_t v_len = PyUnicode_GET_SIZE(v);
     Py_ssize_t w_len = PyUnicode_GET_SIZE(w);
     Py_ssize_t new_len = v_len + w_len;
@@ -4570,7 +4579,8 @@ unicode_concatenate(PyObject *v, PyObject *w,
         }
     }
 
-    if (Py_REFCNT(v) == 1 && !PyUnicode_CHECK_INTERNED(v)) {
+    if (Py_REFCNT(v) == 1 && !PyUnicode_CHECK_INTERNED(v) &&
+        !PyUnicode_IS_COMPACT((PyUnicodeObject *)v)) {
         /* Now we own the last reference to 'v', so we can resize it
          * in-place.
          */
@@ -4594,6 +4604,7 @@ unicode_concatenate(PyObject *v, PyObject *w,
         Py_DECREF(v);
         return w;
     }
+#endif
 }
 
 #ifdef DYNAMIC_EXECUTION_PROFILE
diff --git a/Python/codecs.c b/Python/codecs.c
index 1a3e457..be2e833 100644
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -513,27 +513,25 @@ PyObject *PyCodec_IgnoreErrors(PyObject *exc)
 
 PyObject *PyCodec_ReplaceErrors(PyObject *exc)
 {
-    PyObject *restuple;
-    Py_ssize_t start;
-    Py_ssize_t end;
-    Py_ssize_t i;
+    Py_ssize_t start, end, i, len;
 
     if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
         PyObject *res;
-        Py_UNICODE *p;
+        int kind;
+        void *data;
         if (PyUnicodeEncodeError_GetStart(exc, &start))
             return NULL;
         if (PyUnicodeEncodeError_GetEnd(exc, &end))
             return NULL;
-        res = PyUnicode_FromUnicode(NULL, end-start);
+        len = end - start;
+        res = PyUnicode_New(len, '?');
         if (res == NULL)
             return NULL;
-        for (p = PyUnicode_AS_UNICODE(res), i = start;
-            i<end; ++p, ++i)
-            *p = '?';
-        restuple = Py_BuildValue("(On)", res, end);
-        Py_DECREF(res);
-        return restuple;
+        kind = PyUnicode_KIND(res);
+        data = PyUnicode_DATA(res);
+        for (i = 0; i < len; ++i)
+            PyUnicode_WRITE(kind, data, i, '?');
+        return Py_BuildValue("(Nn)", res, end);
     }
     else if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) {
         Py_UNICODE res = Py_UNICODE_REPLACEMENT_CHARACTER;
@@ -543,20 +541,21 @@ PyObject *PyCodec_ReplaceErrors(PyObject *exc)
     }
     else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) {
         PyObject *res;
-        Py_UNICODE *p;
+        int kind;
+        void *data;
         if (PyUnicodeTranslateError_GetStart(exc, &start))
             return NULL;
         if (PyUnicodeTranslateError_GetEnd(exc, &end))
             return NULL;
-        res = PyUnicode_FromUnicode(NULL, end-start);
+        len = end - start;
+        res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER);
         if (res == NULL)
             return NULL;
-        for (p = PyUnicode_AS_UNICODE(res), i = start;
-            i<end; ++p, ++i)
-            *p = Py_UNICODE_REPLACEMENT_CHARACTER;
-        restuple = Py_BuildValue("(On)", res, end);
-        Py_DECREF(res);
-        return restuple;
+        kind = PyUnicode_KIND(res);
+        data = PyUnicode_DATA(res);
+        for (i=0; i < len; i++)
+            PyUnicode_WRITE(kind, data, i, Py_UNICODE_REPLACEMENT_CHARACTER);
+        return Py_BuildValue("(Nn)", res, end);
     }
     else {
         wrong_exception_type(exc);
@@ -671,10 +670,7 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
     }
 }
 
-static Py_UNICODE hexdigits[] = {
-    '0', '1', '2', '3', '4', '5', '6', '7',
-    '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
-};
+static const char *hexdigits = "0123456789abcdef";
 
 PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
 {
diff --git a/Python/compile.c b/Python/compile.c
index 567b230..5f03792 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -197,16 +197,17 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident)
 {
     /* Name mangling: __private becomes _classname__private.
        This is independent from how the name is used. */
-    const Py_UNICODE *p, *name = PyUnicode_AS_UNICODE(ident);
-    Py_UNICODE *buffer;
-    size_t nlen, plen;
+    PyObject *result;
+    size_t nlen, plen, ipriv;
+    Py_UCS4 maxchar;
     if (privateobj == NULL || !PyUnicode_Check(privateobj) ||
-        name == NULL || name[0] != '_' || name[1] != '_') {
+        PyUnicode_READ_CHAR(ident, 0) != '_' ||
+        PyUnicode_READ_CHAR(ident, 1) != '_') {
         Py_INCREF(ident);
         return ident;
     }
-    p = PyUnicode_AS_UNICODE(privateobj);
-    nlen = Py_UNICODE_strlen(name);
+    nlen = PyUnicode_GET_LENGTH(ident);
+    plen = PyUnicode_GET_LENGTH(privateobj);
     /* Don't mangle __id__ or names with dots.
 
        The only time a name with a dot can occur is when
@@ -216,32 +217,37 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident)
        TODO(jhylton): Decide whether we want to support
        mangling of the module name, e.g. __M.X.
     */
-    if ((name[nlen-1] == '_' && name[nlen-2] == '_')
-        || Py_UNICODE_strchr(name, '.')) {
+    if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
+         PyUnicode_READ_CHAR(ident, nlen-2) == '_') ||
+        PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) {
         Py_INCREF(ident);
         return ident; /* Don't mangle __whatever__ */
     }
     /* Strip leading underscores from class name */
-    while (*p == '_')
-        p++;
-    if (*p == 0) {
+    ipriv = 0;
+    while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_')
+        ipriv++;
+    if (ipriv == plen) {
         Py_INCREF(ident);
         return ident; /* Don't mangle if class is just underscores */
     }
-    plen = Py_UNICODE_strlen(p);
+    plen -= ipriv;
 
     assert(1 <= PY_SSIZE_T_MAX - nlen);
     assert(1 + nlen <= PY_SSIZE_T_MAX - plen);
 
-    ident = PyUnicode_FromStringAndSize(NULL, 1 + nlen + plen);
-    if (!ident)
+    maxchar = PyUnicode_MAX_CHAR_VALUE(ident);
+    if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar)
+        maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj);
+
+    result = PyUnicode_New(1 + nlen + plen, maxchar);
+    if (!result)
         return 0;
-    /* ident = "_" + p[:plen] + name # i.e. 1+plen+nlen bytes */
-    buffer = PyUnicode_AS_UNICODE(ident);
-    buffer[0] = '_';
-    Py_UNICODE_strncpy(buffer+1, p, plen);
-    Py_UNICODE_strcpy(buffer+1+plen, name);
-    return ident;
+    /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */
+    PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_');
+    PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen);
+    PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen);
+    return result;
 }
 
 static int
@@ -2085,22 +2091,27 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname)
        If there is a dot in name, we need to split it and emit a
        LOAD_ATTR for each name.
     */
-    const Py_UNICODE *src = PyUnicode_AS_UNICODE(name);
-    const Py_UNICODE *dot = Py_UNICODE_strchr(src, '.');
-    if (dot) {
+    Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
+                                        PyUnicode_GET_LENGTH(name), 1);
+    if (dot == -2)
+        return -1;
+    if (dot != -1) {
         /* Consume the base module name to get the first attribute */
-        src = dot + 1;
-        while (dot) {
-            /* NB src is only defined when dot != NULL */
+        Py_ssize_t pos = dot + 1;
+        while (dot != -1) {
             PyObject *attr;
-            dot = Py_UNICODE_strchr(src, '.');
-            attr = PyUnicode_FromUnicode(src,
-                                dot ? dot - src : Py_UNICODE_strlen(src));
+            dot = PyUnicode_FindChar(name, '.', pos,
+                                     PyUnicode_GET_LENGTH(name), 1);
+            if (dot == -2)
+                return -1;
+            attr = PyUnicode_Substring(name, pos,
+                                       (dot != -1) ? dot :
+                                       PyUnicode_GET_LENGTH(name));
             if (!attr)
                 return -1;
             ADDOP_O(c, LOAD_ATTR, attr, names);
             Py_DECREF(attr);
-            src = dot + 1;
+            pos = dot + 1;
         }
     }
     return compiler_nameop(c, asname, Store);
@@ -2139,13 +2150,12 @@ compiler_import(struct compiler *c, stmt_ty s)
         }
         else {
             identifier tmp = alias->name;
-            const Py_UNICODE *base = PyUnicode_AS_UNICODE(alias->name);
-            Py_UNICODE *dot = Py_UNICODE_strchr(base, '.');
-            if (dot)
-                tmp = PyUnicode_FromUnicode(base,
-                                            dot - base);
+            Py_ssize_t dot = PyUnicode_FindChar(
+                alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1);
+            if (dot != -1)
+                tmp = PyUnicode_Substring(alias->name, 0, dot);
             r = compiler_nameop(c, tmp, Store);
-            if (dot) {
+            if (dot != -1) {
                 Py_DECREF(tmp);
             }
             if (!r)
@@ -2208,7 +2218,7 @@ compiler_from_import(struct compiler *c, stmt_ty s)
         alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
         identifier store_name;
 
-        if (i == 0 && *PyUnicode_AS_UNICODE(alias->name) == '*') {
+        if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') {
             assert(n == 1);
             ADDOP(c, IMPORT_STAR);
             return 1;
@@ -2522,7 +2532,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
     }
 
     /* XXX Leave assert here, but handle __doc__ and the like better */
-    assert(scope || PyUnicode_AS_UNICODE(name)[0] == '_');
+    assert(scope || PyUnicode_READ_CHAR(name, 0) == '_');
 
     switch (optype) {
     case OP_DEREF:
@@ -3045,8 +3055,7 @@ expr_constant(struct compiler *c, expr_ty e)
         return PyObject_IsTrue(e->v.Str.s);
     case Name_kind:
         /* optimize away names that can't be reassigned */
-        id = PyBytes_AS_STRING(
-            _PyUnicode_AsDefaultEncodedString(e->v.Name.id));
+        id = PyUnicode_AsUTF8(e->v.Name.id);
         if (strcmp(id, "True") == 0) return 1;
         if (strcmp(id, "False") == 0) return 0;
         if (strcmp(id, "None") == 0) return 0;
diff --git a/Python/errors.c b/Python/errors.c
index 5a9a624..149151e 100644
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -395,7 +395,7 @@ PyErr_SetFromErrnoWithFilenameObject(PyObject *exc, PyObject *filenameObject)
                 /* remove trailing cr/lf and dots */
                 while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.'))
                     s_buf[--len] = L'\0';
-                message = PyUnicode_FromUnicode(s_buf, len);
+                message = PyUnicode_FromWideChar(s_buf, len);
             }
         }
     }
@@ -487,7 +487,7 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilenameObject(
         /* remove trailing cr/lf and dots */
         while (len > 0 && (s_buf[len-1] <= L' ' || s_buf[len-1] == L'.'))
             s_buf[--len] = L'\0';
-        message = PyUnicode_FromUnicode(s_buf, len);
+        message = PyUnicode_FromWideChar(s_buf, len);
     }
 
     if (message == NULL)
diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c
index c350907..9fb840b 100644
--- a/Python/formatter_unicode.c
+++ b/Python/formatter_unicode.c
@@ -3,12 +3,1445 @@
    of int.__float__, etc., that take and return unicode objects */
 
 #include "Python.h"
-#include "../Objects/stringlib/unicodedefs.h"
+#include <locale.h>
 
+/* Raises an exception about an unknown presentation type for this
+ * type. */
 
-#define FORMAT_STRING  _PyUnicode_FormatAdvanced
-#define FORMAT_LONG    _PyLong_FormatAdvanced
-#define FORMAT_FLOAT   _PyFloat_FormatAdvanced
-#define FORMAT_COMPLEX _PyComplex_FormatAdvanced
+static void
+unknown_presentation_type(Py_UCS4 presentation_type,
+                          const char* type_name)
+{
+    /* %c might be out-of-range, hence the two cases. */
+    if (presentation_type > 32 && presentation_type < 128)
+        PyErr_Format(PyExc_ValueError,
+                     "Unknown format code '%c' "
+                     "for object of type '%.200s'",
+                     (char)presentation_type,
+                     type_name);
+    else
+        PyErr_Format(PyExc_ValueError,
+                     "Unknown format code '\\x%x' "
+                     "for object of type '%.200s'",
+                     (unsigned int)presentation_type,
+                     type_name);
+}
 
-#include "../Objects/stringlib/formatter.h"
+static void
+invalid_comma_type(Py_UCS4 presentation_type)
+{
+    if (presentation_type > 32 && presentation_type < 128)
+        PyErr_Format(PyExc_ValueError,
+                     "Cannot specify ',' with '%c'.",
+                     (char)presentation_type);
+    else
+        PyErr_Format(PyExc_ValueError,
+                     "Cannot specify ',' with '\\x%x'.",
+                     (unsigned int)presentation_type);
+}
+
+/*
+    get_integer consumes 0 or more decimal digit characters from an
+    input string, updates *result with the corresponding positive
+    integer, and returns the number of digits consumed.
+
+    returns -1 on error.
+*/
+static int
+get_integer(PyObject *str, Py_ssize_t *pos, Py_ssize_t end,
+                  Py_ssize_t *result)
+{
+    Py_ssize_t accumulator, digitval, oldaccumulator;
+    int numdigits;
+    accumulator = numdigits = 0;
+    for (;;(*pos)++, numdigits++) {
+        if (*pos >= end)
+            break;
+        digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str, *pos));
+        if (digitval < 0)
+            break;
+        /*
+           This trick was copied from old Unicode format code.  It's cute,
+           but would really suck on an old machine with a slow divide
+           implementation.  Fortunately, in the normal case we do not
+           expect too many digits.
+        */
+        oldaccumulator = accumulator;
+        accumulator *= 10;
+        if ((accumulator+10)/10 != oldaccumulator+1) {
+            PyErr_Format(PyExc_ValueError,
+                         "Too many decimal digits in format string");
+            return -1;
+        }
+        accumulator += digitval;
+    }
+    *result = accumulator;
+    return numdigits;
+}
+
+/************************************************************************/
+/*********** standard format specifier parsing **************************/
+/************************************************************************/
+
+/* returns true if this character is a specifier alignment token */
+Py_LOCAL_INLINE(int)
+is_alignment_token(Py_UCS4 c)
+{
+    switch (c) {
+    case '<': case '>': case '=': case '^':
+        return 1;
+    default:
+        return 0;
+    }
+}
+
+/* returns true if this character is a sign element */
+Py_LOCAL_INLINE(int)
+is_sign_element(Py_UCS4 c)
+{
+    switch (c) {
+    case ' ': case '+': case '-':
+        return 1;
+    default:
+        return 0;
+    }
+}
+
+
+typedef struct {
+    Py_UCS4 fill_char;
+    Py_UCS4 align;
+    int alternate;
+    Py_UCS4 sign;
+    Py_ssize_t width;
+    int thousands_separators;
+    Py_ssize_t precision;
+    Py_UCS4 type;
+} InternalFormatSpec;
+
+#if 0
+/* Occassionally useful for debugging. Should normally be commented out. */
+static void
+DEBUG_PRINT_FORMAT_SPEC(InternalFormatSpec *format)
+{
+    printf("internal format spec: fill_char %d\n", format->fill_char);
+    printf("internal format spec: align %d\n", format->align);
+    printf("internal format spec: alternate %d\n", format->alternate);
+    printf("internal format spec: sign %d\n", format->sign);
+    printf("internal format spec: width %zd\n", format->width);
+    printf("internal format spec: thousands_separators %d\n",
+           format->thousands_separators);
+    printf("internal format spec: precision %zd\n", format->precision);
+    printf("internal format spec: type %c\n", format->type);
+    printf("\n");
+}
+#endif
+
+
+/*
+  ptr points to the start of the format_spec, end points just past its end.
+  fills in format with the parsed information.
+  returns 1 on success, 0 on failure.
+  if failure, sets the exception
+*/
+static int
+parse_internal_render_format_spec(PyObject *format_spec,
+                                  Py_ssize_t start, Py_ssize_t end,
+                                  InternalFormatSpec *format,
+                                  char default_type,
+                                  char default_align)
+{
+    Py_ssize_t pos = start;
+    /* end-pos is used throughout this code to specify the length of
+       the input string */
+#define READ_spec(index) PyUnicode_READ_CHAR(format_spec, index)
+
+    Py_ssize_t consumed;
+    int align_specified = 0;
+
+    format->fill_char = '\0';
+    format->align = default_align;
+    format->alternate = 0;
+    format->sign = '\0';
+    format->width = -1;
+    format->thousands_separators = 0;
+    format->precision = -1;
+    format->type = default_type;
+
+    /* If the second char is an alignment token,
+       then parse the fill char */
+    if (end-pos >= 2 && is_alignment_token(READ_spec(pos+1))) {
+        format->align = READ_spec(pos+1);
+        format->fill_char = READ_spec(pos);
+        align_specified = 1;
+        pos += 2;
+    }
+    else if (end-pos >= 1 && is_alignment_token(READ_spec(pos))) {
+        format->align = READ_spec(pos);
+        align_specified = 1;
+        ++pos;
+    }
+
+    /* Parse the various sign options */
+    if (end-pos >= 1 && is_sign_element(READ_spec(pos))) {
+        format->sign = READ_spec(pos);
+        ++pos;
+    }
+
+    /* If the next character is #, we're in alternate mode.  This only
+       applies to integers. */
+    if (end-pos >= 1 && READ_spec(pos) == '#') {
+        format->alternate = 1;
+        ++pos;
+    }
+
+    /* The special case for 0-padding (backwards compat) */
+    if (format->fill_char == '\0' && end-pos >= 1 && READ_spec(pos) == '0') {
+        format->fill_char = '0';
+        if (!align_specified) {
+            format->align = '=';
+        }
+        ++pos;
+    }
+
+    consumed = get_integer(format_spec, &pos, end, &format->width);
+    if (consumed == -1)
+        /* Overflow error. Exception already set. */
+        return 0;
+
+    /* If consumed is 0, we didn't consume any characters for the
+       width. In that case, reset the width to -1, because
+       get_integer() will have set it to zero. -1 is how we record
+       that the width wasn't specified. */
+    if (consumed == 0)
+        format->width = -1;
+
+    /* Comma signifies add thousands separators */
+    if (end-pos && READ_spec(pos) == ',') {
+        format->thousands_separators = 1;
+        ++pos;
+    }
+
+    /* Parse field precision */
+    if (end-pos && READ_spec(pos) == '.') {
+        ++pos;
+
+        consumed = get_integer(format_spec, &pos, end, &format->precision);
+        if (consumed == -1)
+            /* Overflow error. Exception already set. */
+            return 0;
+
+        /* Not having a precision after a dot is an error. */
+        if (consumed == 0) {
+            PyErr_Format(PyExc_ValueError,
+                         "Format specifier missing precision");
+            return 0;
+        }
+
+    }
+
+    /* Finally, parse the type field. */
+
+    if (end-pos > 1) {
+        /* More than one char remain, invalid conversion spec. */
+        PyErr_Format(PyExc_ValueError, "Invalid conversion specification");
+        return 0;
+    }
+
+    if (end-pos == 1) {
+        format->type = READ_spec(pos);
+        ++pos;
+    }
+
+    /* Do as much validating as we can, just by looking at the format
+       specifier.  Do not take into account what type of formatting
+       we're doing (int, float, string). */
+
+    if (format->thousands_separators) {
+        switch (format->type) {
+        case 'd':
+        case 'e':
+        case 'f':
+        case 'g':
+        case 'E':
+        case 'G':
+        case '%':
+        case 'F':
+        case '\0':
+            /* These are allowed. See PEP 378.*/
+            break;
+        default:
+            invalid_comma_type(format->type);
+            return 0;
+        }
+    }
+
+    if (format->fill_char > 127 || format->align > 127 ||
+        format->sign > 127) {
+        PyErr_SetString(PyExc_ValueError, "fill character too large");
+        return 0;
+    }
+
+    return 1;
+}
+
+/* Calculate the padding needed. */
+static void
+calc_padding(Py_ssize_t nchars, Py_ssize_t width, Py_UCS4 align,
+             Py_ssize_t *n_lpadding, Py_ssize_t *n_rpadding,
+             Py_ssize_t *n_total)
+{
+    if (width >= 0) {
+        if (nchars > width)
+            *n_total = nchars;
+        else
+            *n_total = width;
+    }
+    else {
+        /* not specified, use all of the chars and no more */
+        *n_total = nchars;
+    }
+
+    /* Figure out how much leading space we need, based on the
+       aligning */
+    if (align == '>')
+        *n_lpadding = *n_total - nchars;
+    else if (align == '^')
+        *n_lpadding = (*n_total - nchars) / 2;
+    else if (align == '<' || align == '=')
+        *n_lpadding = 0;
+    else {
+        /* We should never have an unspecified alignment. */
+        *n_lpadding = 0;
+        assert(0);
+    }
+
+    *n_rpadding = *n_total - nchars - *n_lpadding;
+}
+
+static void
+unicode_fill(PyObject *str, Py_ssize_t start, Py_ssize_t end, Py_UCS4 ch)
+{
+    int kind = PyUnicode_KIND(str);
+    void *data = PyUnicode_DATA(str);
+    while (start < end)
+        PyUnicode_WRITE(kind, data, start++, ch);
+}
+
+/* Do the padding, and return a pointer to where the caller-supplied
+   content goes. */
+static Py_ssize_t
+fill_padding(PyObject *s, Py_ssize_t start, Py_ssize_t nchars,
+             Py_UCS4 fill_char, Py_ssize_t n_lpadding,
+             Py_ssize_t n_rpadding)
+{
+    /* Pad on left. */
+    if (n_lpadding)
+        unicode_fill(s, start, start + n_lpadding, fill_char);
+
+    /* Pad on right. */
+    if (n_rpadding)
+        unicode_fill(s, start + nchars + n_lpadding, 
+                     start + nchars + n_lpadding + n_rpadding, fill_char);
+
+    /* Pointer to the user content. */
+    return start + n_lpadding;
+}
+
+/************************************************************************/
+/*********** common routines for numeric formatting *********************/
+/************************************************************************/
+
+/* Locale type codes. */
+#define LT_CURRENT_LOCALE 0
+#define LT_DEFAULT_LOCALE 1
+#define LT_NO_LOCALE 2
+
+/* Locale info needed for formatting integers and the part of floats
+   before and including the decimal. Note that locales only support
+   8-bit chars, not unicode. */
+typedef struct {
+    char *decimal_point;
+    char *thousands_sep;
+    char *grouping;
+} LocaleInfo;
+
+/* describes the layout for an integer, see the comment in
+   calc_number_widths() for details */
+typedef struct {
+    Py_ssize_t n_lpadding;
+    Py_ssize_t n_prefix;
+    Py_ssize_t n_spadding;
+    Py_ssize_t n_rpadding;
+    char sign;
+    Py_ssize_t n_sign;      /* number of digits needed for sign (0/1) */
+    Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including
+                                    any grouping chars. */
+    Py_ssize_t n_decimal;   /* 0 if only an integer */
+    Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part,
+                               excluding the decimal itself, if
+                               present. */
+
+    /* These 2 are not the widths of fields, but are needed by
+       STRINGLIB_GROUPING. */
+    Py_ssize_t n_digits;    /* The number of digits before a decimal
+                               or exponent. */
+    Py_ssize_t n_min_width; /* The min_width we used when we computed
+                               the n_grouped_digits width. */
+} NumberFieldWidths;
+
+
+/* Given a number of the form:
+   digits[remainder]
+   where ptr points to the start and end points to the end, find where
+    the integer part ends. This could be a decimal, an exponent, both,
+    or neither.
+   If a decimal point is present, set *has_decimal and increment
+    remainder beyond it.
+   Results are undefined (but shouldn't crash) for improperly
+    formatted strings.
+*/
+static void
+parse_number(PyObject *s, Py_ssize_t pos, Py_ssize_t end,
+             Py_ssize_t *n_remainder, int *has_decimal)
+{
+    Py_ssize_t remainder;
+
+    while (pos<end && isdigit(PyUnicode_READ_CHAR(s, pos)))
+        ++pos;
+    remainder = pos;
+
+    /* Does remainder start with a decimal point? */
+    *has_decimal = pos<end && PyUnicode_READ_CHAR(s, remainder) == '.';
+
+    /* Skip the decimal point. */
+    if (*has_decimal)
+        remainder++;
+
+    *n_remainder = end - remainder;
+}
+
+/* not all fields of format are used.  for example, precision is
+   unused.  should this take discrete params in order to be more clear
+   about what it does?  or is passing a single format parameter easier
+   and more efficient enough to justify a little obfuscation? */
+static Py_ssize_t
+calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix,
+                   Py_UCS4 sign_char, PyObject *number, Py_ssize_t n_start,
+                   Py_ssize_t n_end, Py_ssize_t n_remainder,
+                   int has_decimal, const LocaleInfo *locale,
+                   const InternalFormatSpec *format)
+{
+    Py_ssize_t n_non_digit_non_padding;
+    Py_ssize_t n_padding;
+
+    spec->n_digits = n_end - n_start - n_remainder - (has_decimal?1:0);
+    spec->n_lpadding = 0;
+    spec->n_prefix = n_prefix;
+    spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0;
+    spec->n_remainder = n_remainder;
+    spec->n_spadding = 0;
+    spec->n_rpadding = 0;
+    spec->sign = '\0';
+    spec->n_sign = 0;
+
+    /* the output will look like:
+       |                                                                                         |
+       | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> |
+       |                                                                                         |
+
+       sign is computed from format->sign and the actual
+       sign of the number
+
+       prefix is given (it's for the '0x' prefix)
+
+       digits is already known
+
+       the total width is either given, or computed from the
+       actual digits
+
+       only one of lpadding, spadding, and rpadding can be non-zero,
+       and it's calculated from the width and other fields
+    */
+
+    /* compute the various parts we're going to write */
+    switch (format->sign) {
+    case '+':
+        /* always put a + or - */
+        spec->n_sign = 1;
+        spec->sign = (sign_char == '-' ? '-' : '+');
+        break;
+    case ' ':
+        spec->n_sign = 1;
+        spec->sign = (sign_char == '-' ? '-' : ' ');
+        break;
+    default:
+        /* Not specified, or the default (-) */
+        if (sign_char == '-') {
+            spec->n_sign = 1;
+            spec->sign = '-';
+        }
+    }
+
+    /* The number of chars used for non-digits and non-padding. */
+    n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal +
+        spec->n_remainder;
+
+    /* min_width can go negative, that's okay. format->width == -1 means
+       we don't care. */
+    if (format->fill_char == '0' && format->align == '=')
+        spec->n_min_width = format->width - n_non_digit_non_padding;
+    else
+        spec->n_min_width = 0;
+
+    if (spec->n_digits == 0)
+        /* This case only occurs when using 'c' formatting, we need
+           to special case it because the grouping code always wants
+           to have at least one character. */
+        spec->n_grouped_digits = 0;
+    else
+        spec->n_grouped_digits = _PyUnicode_InsertThousandsGrouping(
+            PyUnicode_1BYTE_KIND, NULL, 0, NULL, 
+            spec->n_digits, spec->n_min_width,
+            locale->grouping, locale->thousands_sep);
+
+    /* Given the desired width and the total of digit and non-digit
+       space we consume, see if we need any padding. format->width can
+       be negative (meaning no padding), but this code still works in
+       that case. */
+    n_padding = format->width -
+                        (n_non_digit_non_padding + spec->n_grouped_digits);
+    if (n_padding > 0) {
+        /* Some padding is needed. Determine if it's left, space, or right. */
+        switch (format->align) {
+        case '<':
+            spec->n_rpadding = n_padding;
+            break;
+        case '^':
+            spec->n_lpadding = n_padding / 2;
+            spec->n_rpadding = n_padding - spec->n_lpadding;
+            break;
+        case '=':
+            spec->n_spadding = n_padding;
+            break;
+        case '>':
+            spec->n_lpadding = n_padding;
+            break;
+        default:
+            /* Shouldn't get here, but treat it as '>' */
+            spec->n_lpadding = n_padding;
+            assert(0);
+            break;
+        }
+    }
+    return spec->n_lpadding + spec->n_sign + spec->n_prefix +
+        spec->n_spadding + spec->n_grouped_digits + spec->n_decimal +
+        spec->n_remainder + spec->n_rpadding;
+}
+
+/* Fill in the digit parts of a numbers's string representation,
+   as determined in calc_number_widths().
+   No error checking, since we know the buffer is the correct size. */
+static void
+fill_number(PyObject *out, Py_ssize_t pos, const NumberFieldWidths *spec,
+            PyObject *digits, Py_ssize_t d_start, Py_ssize_t d_end,
+            PyObject *prefix, Py_ssize_t p_start, Py_UCS4 fill_char,
+            LocaleInfo *locale, int toupper)
+{
+    /* Used to keep track of digits, decimal, and remainder. */
+    Py_ssize_t d_pos = d_start;
+    unsigned int kind = PyUnicode_KIND(out);
+    void *data = PyUnicode_DATA(out);
+
+#ifndef NDEBUG
+    Py_ssize_t r;
+#endif
+
+    if (spec->n_lpadding) {
+        unicode_fill(out, pos, pos + spec->n_lpadding, fill_char);
+        pos += spec->n_lpadding;
+    }
+    if (spec->n_sign == 1) {
+        PyUnicode_WRITE(kind, data, pos++, spec->sign);
+    }
+    if (spec->n_prefix) {
+        PyUnicode_CopyCharacters(out, pos, prefix, p_start, spec->n_prefix);
+        if (toupper) {
+            Py_ssize_t t;
+            /* XXX if the upper-case prefix is wider than the target
+               buffer, the caller should have allocated a wider string,
+               but currently doesn't. */
+            for (t = 0; t < spec->n_prefix; ++t)
+                PyUnicode_WRITE(kind, data, pos + t,
+                                Py_UNICODE_TOUPPER(
+                                    PyUnicode_READ(kind, data, pos + t)));
+        }
+        pos += spec->n_prefix;
+    }
+    if (spec->n_spadding) {
+        unicode_fill(out, pos, pos + spec->n_spadding, fill_char);
+        pos += spec->n_spadding;
+    }
+
+    /* Only for type 'c' special case, it has no digits. */
+    if (spec->n_digits != 0) {
+        /* Fill the digits with InsertThousandsGrouping. */
+        char *pdigits = PyUnicode_DATA(digits);
+        if (PyUnicode_KIND(digits) < kind) {
+            pdigits = _PyUnicode_AsKind(digits, kind);
+            if (pdigits == NULL) {
+                /* XXX report exception */
+                Py_FatalError("out of memory");
+                return;
+            }
+        }
+#ifndef NDEBUG
+        r =
+#endif
+            _PyUnicode_InsertThousandsGrouping(
+                kind,
+                (char*)data + PyUnicode_KIND_SIZE(kind, pos),
+                spec->n_grouped_digits,
+                pdigits + PyUnicode_KIND_SIZE(kind, d_pos),
+                spec->n_digits, spec->n_min_width,
+                locale->grouping, locale->thousands_sep);
+#ifndef NDEBUG
+        assert(r == spec->n_grouped_digits);
+#endif
+        if (PyUnicode_KIND(digits) < kind)
+            PyMem_Free(pdigits);
+        d_pos += spec->n_digits;
+    }
+    if (toupper) {
+        Py_ssize_t t;
+        for (t = 0; t < spec->n_grouped_digits; ++t)
+            PyUnicode_WRITE(kind, data, pos + t,
+                            Py_UNICODE_TOUPPER(
+                                PyUnicode_READ(kind, data, pos + t)));
+    }
+    pos += spec->n_grouped_digits;
+
+    if (spec->n_decimal) {
+        Py_ssize_t t;
+        for (t = 0; t < spec->n_decimal; ++t)
+            PyUnicode_WRITE(kind, data, pos + t,
+                            locale->decimal_point[t]);
+        pos += spec->n_decimal;
+        d_pos += 1;
+    }
+
+    if (spec->n_remainder) {
+        PyUnicode_CopyCharacters(out, pos, digits, d_pos, spec->n_remainder);
+        pos += spec->n_remainder;
+        d_pos += spec->n_remainder;
+    }
+
+    if (spec->n_rpadding) {
+        unicode_fill(out, pos, pos + spec->n_rpadding, fill_char);
+        pos += spec->n_rpadding;
+    }
+}
+
+static char no_grouping[1] = {CHAR_MAX};
+
+/* Find the decimal point character(s?), thousands_separator(s?), and
+   grouping description, either for the current locale if type is
+   LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or
+   none if LT_NO_LOCALE. */
+static void
+get_locale_info(int type, LocaleInfo *locale_info)
+{
+    switch (type) {
+    case LT_CURRENT_LOCALE: {
+        struct lconv *locale_data = localeconv();
+        locale_info->decimal_point = locale_data->decimal_point;
+        locale_info->thousands_sep = locale_data->thousands_sep;
+        locale_info->grouping = locale_data->grouping;
+        break;
+    }
+    case LT_DEFAULT_LOCALE:
+        locale_info->decimal_point = ".";
+        locale_info->thousands_sep = ",";
+        locale_info->grouping = "\3"; /* Group every 3 characters.  The
+                                         (implicit) trailing 0 means repeat
+                                         infinitely. */
+        break;
+    case LT_NO_LOCALE:
+        locale_info->decimal_point = ".";
+        locale_info->thousands_sep = "";
+        locale_info->grouping = no_grouping;
+        break;
+    default:
+        assert(0);
+    }
+}
+
+/************************************************************************/
+/*********** string formatting ******************************************/
+/************************************************************************/
+
+static PyObject *
+format_string_internal(PyObject *value, const InternalFormatSpec *format)
+{
+    Py_ssize_t lpad;
+    Py_ssize_t rpad;
+    Py_ssize_t total;
+    Py_ssize_t pos;
+    Py_ssize_t len = PyUnicode_GET_SIZE(value);
+    PyObject *result = NULL;
+    int maxchar = 127;
+
+    /* sign is not allowed on strings */
+    if (format->sign != '\0') {
+        PyErr_SetString(PyExc_ValueError,
+                        "Sign not allowed in string format specifier");
+        goto done;
+    }
+
+    /* alternate is not allowed on strings */
+    if (format->alternate) {
+        PyErr_SetString(PyExc_ValueError,
+                        "Alternate form (#) not allowed in string format "
+                        "specifier");
+        goto done;
+    }
+
+    /* '=' alignment not allowed on strings */
+    if (format->align == '=') {
+        PyErr_SetString(PyExc_ValueError,
+                        "'=' alignment not allowed "
+                        "in string format specifier");
+        goto done;
+    }
+
+    /* if precision is specified, output no more that format.precision
+       characters */
+    if (format->precision >= 0 && len >= format->precision) {
+        len = format->precision;
+    }
+
+    calc_padding(len, format->width, format->align, &lpad, &rpad, &total);
+
+    /* allocate the resulting string */
+    result = PyUnicode_New(total, maxchar);
+    if (result == NULL)
+        goto done;
+
+    /* Write into that space. First the padding. */
+    pos = fill_padding(result, 0, len,
+                       format->fill_char=='\0'?' ':format->fill_char,
+                       lpad, rpad);
+
+    /* Then the source string. */
+    PyUnicode_CopyCharacters(result, pos, value, 0, len);
+
+done:
+    return result;
+}
+
+
+/************************************************************************/
+/*********** long formatting ********************************************/
+/************************************************************************/
+
+typedef PyObject*
+(*IntOrLongToString)(PyObject *value, int base);
+
+static PyObject *
+format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format,
+                            IntOrLongToString tostring)
+{
+    PyObject *result = NULL;
+    int maxchar = 127;
+    PyObject *tmp = NULL;
+    Py_ssize_t inumeric_chars;
+    Py_UCS4 sign_char = '\0';
+    Py_ssize_t n_digits;       /* count of digits need from the computed
+                                  string */
+    Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which
+                                   produces non-digits */
+    Py_ssize_t n_prefix = 0;   /* Count of prefix chars, (e.g., '0x') */
+    Py_ssize_t n_total;
+    Py_ssize_t prefix;
+    NumberFieldWidths spec;
+    long x;
+
+    /* Locale settings, either from the actual locale or
+       from a hard-code pseudo-locale */
+    LocaleInfo locale;
+
+    /* no precision allowed on integers */
+    if (format->precision != -1) {
+        PyErr_SetString(PyExc_ValueError,
+                        "Precision not allowed in integer format specifier");
+        goto done;
+    }
+
+    /* special case for character formatting */
+    if (format->type == 'c') {
+        /* error to specify a sign */
+        if (format->sign != '\0') {
+            PyErr_SetString(PyExc_ValueError,
+                            "Sign not allowed with integer"
+                            " format specifier 'c'");
+            goto done;
+        }
+
+        /* taken from unicodeobject.c formatchar() */
+        /* Integer input truncated to a character */
+/* XXX: won't work for int */
+        x = PyLong_AsLong(value);
+        if (x == -1 && PyErr_Occurred())
+            goto done;
+        if (x < 0 || x > 0x10ffff) {
+            PyErr_SetString(PyExc_OverflowError,
+                            "%c arg not in range(0x110000) "
+                            "(wide Python build)");
+            goto done;
+        }
+        tmp = PyUnicode_FromOrdinal(x);
+        inumeric_chars = 0;
+        n_digits = 1;
+        if (x > maxchar)
+            maxchar = x;
+
+        /* As a sort-of hack, we tell calc_number_widths that we only
+           have "remainder" characters. calc_number_widths thinks
+           these are characters that don't get formatted, only copied
+           into the output string. We do this for 'c' formatting,
+           because the characters are likely to be non-digits. */
+        n_remainder = 1;
+    }
+    else {
+        int base;
+        int leading_chars_to_skip = 0;  /* Number of characters added by
+                                           PyNumber_ToBase that we want to
+                                           skip over. */
+
+        /* Compute the base and how many characters will be added by
+           PyNumber_ToBase */
+        switch (format->type) {
+        case 'b':
+            base = 2;
+            leading_chars_to_skip = 2; /* 0b */
+            break;
+        case 'o':
+            base = 8;
+            leading_chars_to_skip = 2; /* 0o */
+            break;
+        case 'x':
+        case 'X':
+            base = 16;
+            leading_chars_to_skip = 2; /* 0x */
+            break;
+        default:  /* shouldn't be needed, but stops a compiler warning */
+        case 'd':
+        case 'n':
+            base = 10;
+            break;
+        }
+
+        /* The number of prefix chars is the same as the leading
+           chars to skip */
+        if (format->alternate)
+            n_prefix = leading_chars_to_skip;
+
+        /* Do the hard part, converting to a string in a given base */
+        tmp = tostring(value, base);
+        if (tmp == NULL || PyUnicode_READY(tmp) == -1)
+            goto done;
+
+        inumeric_chars = 0;
+        n_digits = PyUnicode_GET_LENGTH(tmp);
+
+        prefix = inumeric_chars;
+
+        /* Is a sign character present in the output?  If so, remember it
+           and skip it */
+        if (PyUnicode_READ_CHAR(tmp, inumeric_chars) == '-') {
+            sign_char = '-';
+            ++prefix;
+            ++leading_chars_to_skip;
+        }
+
+        /* Skip over the leading chars (0x, 0b, etc.) */
+        n_digits -= leading_chars_to_skip;
+        inumeric_chars += leading_chars_to_skip;
+    }
+
+    /* Determine the grouping, separator, and decimal point, if any. */
+    get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE :
+                    (format->thousands_separators ?
+                     LT_DEFAULT_LOCALE :
+                     LT_NO_LOCALE),
+                    &locale);
+
+    /* Calculate how much memory we'll need. */
+    n_total = calc_number_widths(&spec, n_prefix, sign_char, tmp, inumeric_chars,
+                                 inumeric_chars + n_digits, n_remainder, 0, &locale, format);
+
+    /* Allocate the memory. */
+    result = PyUnicode_New(n_total, maxchar);
+    if (!result)
+        goto done;
+
+    /* Populate the memory. */
+    fill_number(result, 0, &spec, tmp, inumeric_chars, inumeric_chars + n_digits,
+                tmp, prefix,
+                format->fill_char == '\0' ? ' ' : format->fill_char,
+                &locale, format->type == 'X');
+
+done:
+    Py_XDECREF(tmp);
+    return result;
+}
+
+/************************************************************************/
+/*********** float formatting *******************************************/
+/************************************************************************/
+
+static PyObject*
+strtounicode(char *charbuffer, Py_ssize_t len)
+{
+    return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, charbuffer, len);
+}
+
+/* much of this is taken from unicodeobject.c */
+static PyObject *
+format_float_internal(PyObject *value,
+                      const InternalFormatSpec *format)
+{
+    char *buf = NULL;       /* buffer returned from PyOS_double_to_string */
+    Py_ssize_t n_digits;
+    Py_ssize_t n_remainder;
+    Py_ssize_t n_total;
+    int has_decimal;
+    double val;
+    Py_ssize_t precision = format->precision;
+    Py_ssize_t default_precision = 6;
+    Py_UCS4 type = format->type;
+    int add_pct = 0;
+    Py_ssize_t index;
+    NumberFieldWidths spec;
+    int flags = 0;
+    PyObject *result = NULL;
+    int maxchar = 127;
+    Py_UCS4 sign_char = '\0';
+    int float_type; /* Used to see if we have a nan, inf, or regular float. */
+    PyObject *unicode_tmp = NULL;
+
+    /* Locale settings, either from the actual locale or
+       from a hard-code pseudo-locale */
+    LocaleInfo locale;
+
+    if (format->alternate)
+        flags |= Py_DTSF_ALT;
+
+    if (type == '\0') {
+        /* Omitted type specifier.  Behaves in the same way as repr(x)
+           and str(x) if no precision is given, else like 'g', but with
+           at least one digit after the decimal point. */
+        flags |= Py_DTSF_ADD_DOT_0;
+        type = 'r';
+        default_precision = 0;
+    }
+
+    if (type == 'n')
+        /* 'n' is the same as 'g', except for the locale used to
+           format the result. We take care of that later. */
+        type = 'g';
+
+    val = PyFloat_AsDouble(value);
+    if (val == -1.0 && PyErr_Occurred())
+        goto done;
+
+    if (type == '%') {
+        type = 'f';
+        val *= 100;
+        add_pct = 1;
+    }
+
+    if (precision < 0)
+        precision = default_precision;
+    else if (type == 'r')
+        type = 'g';
+
+    /* Cast "type", because if we're in unicode we need to pass a
+       8-bit char. This is safe, because we've restricted what "type"
+       can be. */
+    buf = PyOS_double_to_string(val, (char)type, precision, flags,
+                                &float_type);
+    if (buf == NULL)
+        goto done;
+    n_digits = strlen(buf);
+
+    if (add_pct) {
+        /* We know that buf has a trailing zero (since we just called
+           strlen() on it), and we don't use that fact any more. So we
+           can just write over the trailing zero. */
+        buf[n_digits] = '%';
+        n_digits += 1;
+    }
+
+    /* Since there is no unicode version of PyOS_double_to_string,
+       just use the 8 bit version and then convert to unicode. */
+    unicode_tmp = strtounicode(buf, n_digits);
+    if (unicode_tmp == NULL)
+        goto done;
+    index = 0;
+
+    /* Is a sign character present in the output?  If so, remember it
+       and skip it */
+    if (PyUnicode_READ_CHAR(unicode_tmp, index) == '-') {
+        sign_char = '-';
+        ++index;
+        --n_digits;
+    }
+
+    /* Determine if we have any "remainder" (after the digits, might include
+       decimal or exponent or both (or neither)) */
+    parse_number(unicode_tmp, index, index + n_digits, &n_remainder, &has_decimal);
+
+    /* Determine the grouping, separator, and decimal point, if any. */
+    get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE :
+                    (format->thousands_separators ?
+                     LT_DEFAULT_LOCALE :
+                     LT_NO_LOCALE),
+                    &locale);
+
+    /* Calculate how much memory we'll need. */
+    n_total = calc_number_widths(&spec, 0, sign_char, unicode_tmp, index, 
+                                 index + n_digits, n_remainder, has_decimal,
+                                 &locale, format);
+
+    /* Allocate the memory. */
+    result = PyUnicode_New(n_total, maxchar);
+    if (result == NULL)
+        goto done;
+
+    /* Populate the memory. */
+    fill_number(result, 0, &spec, unicode_tmp, index, index + n_digits,
+                NULL, 0,
+                format->fill_char == '\0' ? ' ' : format->fill_char, &locale,
+                0);
+
+done:
+    PyMem_Free(buf);
+    Py_DECREF(unicode_tmp);
+    return result;
+}
+
+/************************************************************************/
+/*********** complex formatting *****************************************/
+/************************************************************************/
+
+static PyObject *
+format_complex_internal(PyObject *value,
+                        const InternalFormatSpec *format)
+{
+    double re;
+    double im;
+    char *re_buf = NULL;       /* buffer returned from PyOS_double_to_string */
+    char *im_buf = NULL;       /* buffer returned from PyOS_double_to_string */
+
+    InternalFormatSpec tmp_format = *format;
+    Py_ssize_t n_re_digits;
+    Py_ssize_t n_im_digits;
+    Py_ssize_t n_re_remainder;
+    Py_ssize_t n_im_remainder;
+    Py_ssize_t n_re_total;
+    Py_ssize_t n_im_total;
+    int re_has_decimal;
+    int im_has_decimal;
+    Py_ssize_t precision = format->precision;
+    Py_ssize_t default_precision = 6;
+    Py_UCS4 type = format->type;
+    Py_ssize_t i_re;
+    Py_ssize_t i_im;
+    NumberFieldWidths re_spec;
+    NumberFieldWidths im_spec;
+    int flags = 0;
+    PyObject *result = NULL;
+    int maxchar = 127;
+    int rkind;
+    void *rdata;
+    Py_ssize_t index;
+    Py_UCS4 re_sign_char = '\0';
+    Py_UCS4 im_sign_char = '\0';
+    int re_float_type; /* Used to see if we have a nan, inf, or regular float. */
+    int im_float_type;
+    int add_parens = 0;
+    int skip_re = 0;
+    Py_ssize_t lpad;
+    Py_ssize_t rpad;
+    Py_ssize_t total;
+    PyObject *re_unicode_tmp = NULL;
+    PyObject *im_unicode_tmp = NULL;
+
+    /* Locale settings, either from the actual locale or
+       from a hard-code pseudo-locale */
+    LocaleInfo locale;
+
+    /* Zero padding is not allowed. */
+    if (format->fill_char == '0') {
+        PyErr_SetString(PyExc_ValueError,
+                        "Zero padding is not allowed in complex format "
+                        "specifier");
+        goto done;
+    }
+
+    /* Neither is '=' alignment . */
+    if (format->align == '=') {
+        PyErr_SetString(PyExc_ValueError,
+                        "'=' alignment flag is not allowed in complex format "
+                        "specifier");
+        goto done;
+    }
+
+    re = PyComplex_RealAsDouble(value);
+    if (re == -1.0 && PyErr_Occurred())
+        goto done;
+    im = PyComplex_ImagAsDouble(value);
+    if (im == -1.0 && PyErr_Occurred())
+        goto done;
+
+    if (format->alternate)
+        flags |= Py_DTSF_ALT;
+
+    if (type == '\0') {
+        /* Omitted type specifier. Should be like str(self). */
+        type = 'r';
+        default_precision = 0;
+        if (re == 0.0 && copysign(1.0, re) == 1.0)
+            skip_re = 1;
+        else
+            add_parens = 1;
+    }
+
+    if (type == 'n')
+        /* 'n' is the same as 'g', except for the locale used to
+           format the result. We take care of that later. */
+        type = 'g';
+
+    if (precision < 0)
+        precision = default_precision;
+    else if (type == 'r')
+        type = 'g';
+
+    /* Cast "type", because if we're in unicode we need to pass a
+       8-bit char. This is safe, because we've restricted what "type"
+       can be. */
+    re_buf = PyOS_double_to_string(re, (char)type, precision, flags,
+                                   &re_float_type);
+    if (re_buf == NULL)
+        goto done;
+    im_buf = PyOS_double_to_string(im, (char)type, precision, flags,
+                                   &im_float_type);
+    if (im_buf == NULL)
+        goto done;
+
+    n_re_digits = strlen(re_buf);
+    n_im_digits = strlen(im_buf);
+
+    /* Since there is no unicode version of PyOS_double_to_string,
+       just use the 8 bit version and then convert to unicode. */
+    re_unicode_tmp = strtounicode(re_buf, n_re_digits);
+    if (re_unicode_tmp == NULL)
+        goto done;
+    i_re = 0;
+
+    im_unicode_tmp = strtounicode(im_buf, n_im_digits);
+    if (im_unicode_tmp == NULL)
+        goto done;
+    i_im = 0;
+
+    /* Is a sign character present in the output?  If so, remember it
+       and skip it */
+    if (PyUnicode_READ_CHAR(re_unicode_tmp, i_re) == '-') {
+        re_sign_char = '-';
+        ++i_re;
+        --n_re_digits;
+    }
+    if (PyUnicode_READ_CHAR(im_unicode_tmp, i_im) == '-') {
+        im_sign_char = '-';
+        ++i_im;
+        --n_im_digits;
+    }
+
+    /* Determine if we have any "remainder" (after the digits, might include
+       decimal or exponent or both (or neither)) */
+    parse_number(re_unicode_tmp, i_re, i_re + n_re_digits, 
+                 &n_re_remainder, &re_has_decimal);
+    parse_number(im_unicode_tmp, i_im, i_im + n_im_digits, 
+                 &n_im_remainder, &im_has_decimal);
+
+    /* Determine the grouping, separator, and decimal point, if any. */
+    get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE :
+                    (format->thousands_separators ?
+                     LT_DEFAULT_LOCALE :
+                     LT_NO_LOCALE),
+                    &locale);
+
+    /* Turn off any padding. We'll do it later after we've composed
+       the numbers without padding. */
+    tmp_format.fill_char = '\0';
+    tmp_format.align = '<';
+    tmp_format.width = -1;
+
+    /* Calculate how much memory we'll need. */
+    n_re_total = calc_number_widths(&re_spec, 0, re_sign_char, re_unicode_tmp,
+                                    i_re, i_re + n_re_digits, n_re_remainder,
+                                    re_has_decimal, &locale, &tmp_format);
+
+    /* Same formatting, but always include a sign, unless the real part is
+     * going to be omitted, in which case we use whatever sign convention was
+     * requested by the original format. */
+    if (!skip_re)
+        tmp_format.sign = '+';
+    n_im_total = calc_number_widths(&im_spec, 0, im_sign_char, im_unicode_tmp,
+                                    i_im, i_im + n_im_digits, n_im_remainder,
+                                    im_has_decimal, &locale, &tmp_format);
+
+    if (skip_re)
+        n_re_total = 0;
+
+    /* Add 1 for the 'j', and optionally 2 for parens. */
+    calc_padding(n_re_total + n_im_total + 1 + add_parens * 2,
+                 format->width, format->align, &lpad, &rpad, &total);
+
+    result = PyUnicode_New(total, maxchar);
+    if (result == NULL)
+        goto done;
+    rkind = PyUnicode_KIND(result);
+    rdata = PyUnicode_DATA(result);
+
+    /* Populate the memory. First, the padding. */
+    index = fill_padding(result, 0,
+                         n_re_total + n_im_total + 1 + add_parens * 2,
+                         format->fill_char=='\0' ? ' ' : format->fill_char,
+                         lpad, rpad);
+
+    if (add_parens)
+        PyUnicode_WRITE(rkind, rdata, index++, '(');
+
+    if (!skip_re) {
+        fill_number(result, index, &re_spec, re_unicode_tmp, 
+                    i_re, i_re + n_re_digits, NULL, 0, 0, &locale, 0);
+        index += n_re_total;
+    }
+    fill_number(result, index, &im_spec, im_unicode_tmp,
+                i_im, i_im + n_im_digits, NULL, 0, 0, &locale, 0);
+    index += n_im_total;
+    PyUnicode_WRITE(rkind, rdata, index++, 'j');
+
+    if (add_parens)
+        PyUnicode_WRITE(rkind, rdata, index++, ')');
+
+done:
+    PyMem_Free(re_buf);
+    PyMem_Free(im_buf);
+    Py_XDECREF(re_unicode_tmp);
+    Py_XDECREF(im_unicode_tmp);
+    return result;
+}
+
+/************************************************************************/
+/*********** built in formatters ****************************************/
+/************************************************************************/
+PyObject *
+_PyUnicode_FormatAdvanced(PyObject *obj,
+                          PyObject *format_spec,
+                          Py_ssize_t start, Py_ssize_t end)
+{
+    InternalFormatSpec format;
+    PyObject *result = NULL;
+
+    /* check for the special case of zero length format spec, make
+       it equivalent to str(obj) */
+    if (start == end) {
+        result = PyObject_Str(obj);
+        goto done;
+    }
+
+    /* parse the format_spec */
+    if (!parse_internal_render_format_spec(format_spec, start, end,
+                                           &format, 's', '<'))
+        goto done;
+
+    /* type conversion? */
+    switch (format.type) {
+    case 's':
+        /* no type conversion needed, already a string.  do the formatting */
+        result = format_string_internal(obj, &format);
+        break;
+    default:
+        /* unknown */
+        unknown_presentation_type(format.type, obj->ob_type->tp_name);
+        goto done;
+    }
+
+done:
+    return result;
+}
+
+static PyObject*
+format_int_or_long(PyObject* obj, PyObject* format_spec,
+                   Py_ssize_t start, Py_ssize_t end,
+                   IntOrLongToString tostring)
+{
+    PyObject *result = NULL;
+    PyObject *tmp = NULL;
+    InternalFormatSpec format;
+
+    /* check for the special case of zero length format spec, make
+       it equivalent to str(obj) */
+    if (start == end) {
+        result = PyObject_Str(obj);
+        goto done;
+    }
+
+    /* parse the format_spec */
+    if (!parse_internal_render_format_spec(format_spec, start, end,
+                                           &format, 'd', '>'))
+        goto done;
+
+    /* type conversion? */
+    switch (format.type) {
+    case 'b':
+    case 'c':
+    case 'd':
+    case 'o':
+    case 'x':
+    case 'X':
+    case 'n':
+        /* no type conversion needed, already an int (or long).  do
+           the formatting */
+            result = format_int_or_long_internal(obj, &format, tostring);
+        break;
+
+    case 'e':
+    case 'E':
+    case 'f':
+    case 'F':
+    case 'g':
+    case 'G':
+    case '%':
+        /* convert to float */
+        tmp = PyNumber_Float(obj);
+        if (tmp == NULL)
+            goto done;
+        result = format_float_internal(tmp, &format);
+        break;
+
+    default:
+        /* unknown */
+        unknown_presentation_type(format.type, obj->ob_type->tp_name);
+        goto done;
+    }
+
+done:
+    Py_XDECREF(tmp);
+    return result;
+}
+
+/* Need to define long_format as a function that will convert a long
+   to a string.  In 3.0, _PyLong_Format has the correct signature. */
+#define long_format _PyLong_Format
+
+PyObject *
+_PyLong_FormatAdvanced(PyObject *obj,
+                       PyObject *format_spec,
+                       Py_ssize_t start, Py_ssize_t end)
+{
+    return format_int_or_long(obj, format_spec, start, end,
+                              long_format);
+}
+
+PyObject *
+_PyFloat_FormatAdvanced(PyObject *obj,
+                        PyObject *format_spec,
+                        Py_ssize_t start, Py_ssize_t end)
+{
+    PyObject *result = NULL;
+    InternalFormatSpec format;
+
+    /* check for the special case of zero length format spec, make
+       it equivalent to str(obj) */
+    if (start == end) {
+        result = PyObject_Str(obj);
+        goto done;
+    }
+
+    /* parse the format_spec */
+    if (!parse_internal_render_format_spec(format_spec, start, end,
+                                           &format, '\0', '>'))
+        goto done;
+
+    /* type conversion? */
+    switch (format.type) {
+    case '\0': /* No format code: like 'g', but with at least one decimal. */
+    case 'e':
+    case 'E':
+    case 'f':
+    case 'F':
+    case 'g':
+    case 'G':
+    case 'n':
+    case '%':
+        /* no conversion, already a float.  do the formatting */
+        result = format_float_internal(obj, &format);
+        break;
+
+    default:
+        /* unknown */
+        unknown_presentation_type(format.type, obj->ob_type->tp_name);
+        goto done;
+    }
+
+done:
+    return result;
+}
+
+PyObject *
+_PyComplex_FormatAdvanced(PyObject *obj,
+                          PyObject *format_spec,
+                          Py_ssize_t start, Py_ssize_t end)
+{
+    PyObject *result = NULL;
+    InternalFormatSpec format;
+
+    /* check for the special case of zero length format spec, make
+       it equivalent to str(obj) */
+    if (start == end) {
+        result = PyObject_Str(obj);
+        goto done;
+    }
+
+    /* parse the format_spec */
+    if (!parse_internal_render_format_spec(format_spec, start, end,
+                                           &format, '\0', '>'))
+        goto done;
+
+    /* type conversion? */
+    switch (format.type) {
+    case '\0': /* No format code: like 'g', but with at least one decimal. */
+    case 'e':
+    case 'E':
+    case 'f':
+    case 'F':
+    case 'g':
+    case 'G':
+    case 'n':
+        /* no conversion, already a complex.  do the formatting */
+        result = format_complex_internal(obj, &format);
+        break;
+
+    default:
+        /* unknown */
+        unknown_presentation_type(format.type, obj->ob_type->tp_name);
+        goto done;
+    }
+
+done:
+    return result;
+}
diff --git a/Python/getargs.c b/Python/getargs.c
index c3da368..0e7d9c4 100644
--- a/Python/getargs.c
+++ b/Python/getargs.c
@@ -546,9 +546,6 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 
 
 
-#define UNICODE_DEFAULT_ENCODING(arg) \
-    _PyUnicode_AsDefaultEncodedString(arg)
-
 /* Format an error message generated by convertsimple(). */
 
 static char *
@@ -611,7 +608,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 
     const char *format = *p_format;
     char c = *format++;
-    PyObject *uarg;
+    char *sarg;
 
     switch (c) {
 
@@ -838,8 +835,11 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
     case 'C': {/* unicode char */
         int *p = va_arg(*p_va, int *);
         if (PyUnicode_Check(arg) &&
-            PyUnicode_GET_SIZE(arg) == 1)
-            *p = PyUnicode_AS_UNICODE(arg)[0];
+            PyUnicode_GET_LENGTH(arg) == 1) {
+            int kind = PyUnicode_KIND(arg);
+            void *data = PyUnicode_DATA(arg);
+            *p = PyUnicode_READ(kind, data, 0);
+        }
         else
             return converterr("a unicode character", arg, msgbuf, bufsize);
         break;
@@ -889,13 +889,12 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
             if (c == 'z' && arg == Py_None)
                 PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0);
             else if (PyUnicode_Check(arg)) {
-                uarg = UNICODE_DEFAULT_ENCODING(arg);
-                if (uarg == NULL)
+                Py_ssize_t len;
+                sarg = PyUnicode_AsUTF8AndSize(arg, &len);
+                if (sarg == NULL)
                     return converterr(CONV_UNICODE,
                                       arg, msgbuf, bufsize);
-                PyBuffer_FillInfo(p, arg,
-                                  PyBytes_AS_STRING(uarg), PyBytes_GET_SIZE(uarg),
-                                  1, 0);
+                PyBuffer_FillInfo(p, arg, sarg, len, 1, 0);
             }
             else { /* any buffer-like object */
                 char *buf;
@@ -918,12 +917,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
                 STORE_SIZE(0);
             }
             else if (PyUnicode_Check(arg)) {
-                uarg = UNICODE_DEFAULT_ENCODING(arg);
-                if (uarg == NULL)
+                Py_ssize_t len;
+                sarg = PyUnicode_AsUTF8AndSize(arg, &len);
+                if (sarg == NULL)
                     return converterr(CONV_UNICODE,
                                       arg, msgbuf, bufsize);
-                *p = PyBytes_AS_STRING(uarg);
-                STORE_SIZE(PyBytes_GET_SIZE(uarg));
+                *p = sarg;
+                STORE_SIZE(len);
             }
             else { /* any buffer-like object */
                 /* XXX Really? */
@@ -937,22 +937,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
         } else {
             /* "s" or "z" */
             char **p = va_arg(*p_va, char **);
-            uarg = NULL;
+            Py_ssize_t len;
+            sarg = NULL;
 
             if (c == 'z' && arg == Py_None)
                 *p = NULL;
             else if (PyUnicode_Check(arg)) {
-                uarg = UNICODE_DEFAULT_ENCODING(arg);
-                if (uarg == NULL)
+                sarg = PyUnicode_AsUTF8AndSize(arg, &len);
+                if (sarg == NULL)
                     return converterr(CONV_UNICODE,
                                       arg, msgbuf, bufsize);
-                *p = PyBytes_AS_STRING(uarg);
+                *p = sarg;
             }
             else
                 return converterr(c == 'z' ? "str or None" : "str",
                                   arg, msgbuf, bufsize);
-            if (*p != NULL && uarg != NULL &&
-                (Py_ssize_t) strlen(*p) != PyBytes_GET_SIZE(uarg))
+            if (*p != NULL && sarg != NULL && (Py_ssize_t) strlen(*p) != len)
                 return converterr(
                     c == 'z' ? "str without null bytes or None"
                              : "str without null bytes",
@@ -976,6 +976,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
             }
             else if (PyUnicode_Check(arg)) {
                 *p = PyUnicode_AS_UNICODE(arg);
+                if (*p == NULL)
+                    RETURN_ERR_OCCURRED;
                 STORE_SIZE(PyUnicode_GET_SIZE(arg));
             }
             else
@@ -987,6 +989,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
                 *p = NULL;
             else if (PyUnicode_Check(arg)) {
                 *p = PyUnicode_AS_UNICODE(arg);
+                if (*p == NULL)
+                    RETURN_ERR_OCCURRED;
                 if (Py_UNICODE_strlen(*p) != PyUnicode_GET_SIZE(arg))
                     return converterr(
                         "str without null character or None",
diff --git a/Python/import.c b/Python/import.c
index 24df985..4afdb70 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -118,12 +118,12 @@ typedef unsigned short mode_t;
 #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24))
 #define TAG "cpython-" MAJOR MINOR;
 #define CACHEDIR "__pycache__"
-static const Py_UNICODE CACHEDIR_UNICODE[] = {
+static const Py_UCS4 CACHEDIR_UNICODE[] = {
     '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'};
 /* Current magic word and string tag as globals. */
 static long pyc_magic = MAGIC;
 static const char *pyc_tag = TAG;
-static const Py_UNICODE PYC_TAG_UNICODE[] = {
+static const Py_UCS4 PYC_TAG_UNICODE[] = {
     'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'};
 #undef QUOTE
 #undef STRIFY
@@ -762,7 +762,7 @@ remove_module(PyObject *name)
 
 static PyObject * get_sourcefile(PyObject *filename);
 static PyObject *make_source_pathname(PyObject *pathname);
-static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug);
+static PyObject* make_compiled_pathname(PyObject *pathname, int debug);
 
 /* Execute a code object in a module and return the module object
  * WITH INCREMENTED REFERENCE COUNT.  If an error occurs, name is
@@ -886,10 +886,10 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
 /* Like strrchr(string, '/') but searches for the rightmost of either SEP
    or ALTSEP, if the latter is defined.
 */
-static Py_UNICODE*
-rightmost_sep(Py_UNICODE *s)
+static Py_UCS4*
+rightmost_sep(Py_UCS4 *s)
 {
-    Py_UNICODE *found, c;
+    Py_UCS4 *found, c;
     for (found = NULL; (c = *s); s++) {
         if (c == SEP
 #ifdef ALTSEP
@@ -912,15 +912,21 @@ rightmost_sep(Py_UNICODE *s)
    foo.py -> __pycache__/foo.<tag>.pyc */
 
 static PyObject*
-make_compiled_pathname(Py_UNICODE *pathname, int debug)
+make_compiled_pathname(PyObject *pathstr, int debug)
 {
-    Py_UNICODE buf[MAXPATHLEN];
+    Py_UCS4 *pathname;
+    Py_UCS4 buf[MAXPATHLEN];
     size_t buflen = (size_t)MAXPATHLEN;
-    size_t len = Py_UNICODE_strlen(pathname);
+    size_t len;
     size_t i, save;
-    Py_UNICODE *pos;
+    Py_UCS4 *pos;
     int sep = SEP;
 
+    pathname = PyUnicode_AsUCS4Copy(pathstr);
+    if (!pathname)
+        return NULL;
+    len = Py_UCS4_strlen(pathname);
+
     /* Sanity check that the buffer has roughly enough space to hold what
        will eventually be the full path to the compiled file.  The 5 extra
        bytes include the slash afer __pycache__, the two extra dots, the
@@ -930,8 +936,10 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug)
        sanity check before writing the extension to ensure we do not
        overflow the buffer.
     */
-    if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen)
+    if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) {
+        PyMem_Free(pathname);
         return NULL;
+    }
 
     /* Find the last path separator and copy everything from the start of
        the source string up to and including the separator.
@@ -943,24 +951,28 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug)
     else {
         sep = *pos;
         i = pos - pathname + 1;
-        Py_UNICODE_strncpy(buf, pathname, i);
+        Py_UCS4_strncpy(buf, pathname, i);
     }
 
     save = i;
     buf[i++] = '\0';
     /* Add __pycache__/ */
-    Py_UNICODE_strcat(buf, CACHEDIR_UNICODE);
-    i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1;
+    Py_UCS4_strcat(buf, CACHEDIR_UNICODE);
+    i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1;
     buf[i++] = sep;
     buf[i] = '\0';
     /* Add the base filename, but remove the .py or .pyw extension, since
        the tag name must go before the extension.
     */
-    Py_UNICODE_strcat(buf, pathname + save);
-    pos = Py_UNICODE_strrchr(buf + i, '.');
+    Py_UCS4_strcat(buf, pathname + save);
+    pos = Py_UCS4_strrchr(buf + i, '.');
     if (pos != NULL)
         *++pos = '\0';
-    Py_UNICODE_strcat(buf, PYC_TAG_UNICODE);
+
+    /* pathname is not used from here on. */
+    PyMem_Free(pathname);
+
+    Py_UCS4_strcat(buf, PYC_TAG_UNICODE);
     /* The length test above assumes that we're only adding one character
        to the end of what would normally be the extension.  What if there
        is no extension, or the string ends in '.' or '.p', and otherwise
@@ -1010,7 +1022,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug)
 #if 0
     printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen);
 #endif
-    len = Py_UNICODE_strlen(buf);
+    len = Py_UCS4_strlen(buf);
     if (len + 5 > buflen)
         return NULL;
     buf[len] = '.'; len++;
@@ -1018,7 +1030,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug)
     buf[len] = 'y'; len++;
     buf[len] = debug ? 'c' : 'o'; len++;
     assert(len <= buflen);
-    return PyUnicode_FromUnicode(buf, len);
+    return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len);
 }
 
 
@@ -1033,14 +1045,16 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug)
 static PyObject*
 make_source_pathname(PyObject *pathobj)
 {
-    Py_UNICODE buf[MAXPATHLEN];
-    Py_UNICODE *pathname;
-    Py_UNICODE *left, *right, *dot0, *dot1, sep;
+    Py_UCS4 buf[MAXPATHLEN];
+    Py_UCS4 *pathname;
+    Py_UCS4 *left, *right, *dot0, *dot1, sep;
     size_t i, j;
 
-    if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN)
+    if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN)
+        return NULL;
+    pathname = PyUnicode_AsUCS4Copy(pathobj);
+    if (!pathname)
         return NULL;
-    pathname = PyUnicode_AS_UNICODE(pathobj);
 
     /* Look back two slashes from the end.  In between these two slashes
        must be the string __pycache__ or this is not a PEP 3147 style
@@ -1057,31 +1071,35 @@ make_source_pathname(PyObject *pathobj)
         left = pathname;
     else
         left++;
-    if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) ||
-        Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0)
-        return NULL;
+    if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) ||
+        Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0)
+        goto error;
 
     /* Now verify that the path component to the right of the last slash
        has two dots in it.
     */
-    if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL)
-        return NULL;
-    if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL)
-        return NULL;
+    if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL)
+       goto error;
+    if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL)
+        goto error;
     /* Too many dots? */
-    if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL)
-        return NULL;
+    if (Py_UCS4_strchr(dot1 + 1, '.') != NULL)
+        goto error;
 
     /* This is a PEP 3147 path.  Start by copying everything from the
        start of pathname up to and including the leftmost slash.  Then
        copy the file's basename, removing the magic tag and adding a .py
        suffix.
     */
-    Py_UNICODE_strncpy(buf, pathname, (i=left-pathname));
-    Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right));
+    Py_UCS4_strncpy(buf, pathname, (i=left-pathname));
+    Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right));
     buf[i+j] = 'p';
     buf[i+j+1] = 'y';
-    return PyUnicode_FromUnicode(buf, i+j+2);
+    PyMem_Free(pathname);
+    return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2);
+  error:
+    PyMem_Free(pathname);
+    return NULL;
 }
 
 /* Given a pathname for a Python source file, its time of last
@@ -1250,6 +1268,7 @@ static void
 write_compiled_module(PyCodeObject *co, PyObject *cpathname,
                       struct stat *srcstat)
 {
+    Py_UCS4 *cpathname_ucs4;
     FILE *fp;
     time_t mtime = srcstat->st_mtime;
 #ifdef MS_WINDOWS   /* since Windows uses different permissions  */
@@ -1267,18 +1286,23 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname,
     PyObject *cpathbytes;
 #endif
     PyObject *dirname;
-    Py_UNICODE *dirsep;
+    Py_UCS4 *dirsep;
     int res, ok;
 
     /* Ensure that the __pycache__ directory exists. */
-    dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname));
+    cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname);
+    if (!cpathname_ucs4)
+        return;
+    dirsep = rightmost_sep(cpathname_ucs4);
     if (dirsep == NULL) {
         if (Py_VerboseFlag)
             PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname);
         return;
     }
-    dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname),
-                                    dirsep - PyUnicode_AS_UNICODE(cpathname));
+    dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                        cpathname_ucs4,
+                                        dirsep - cpathname_ucs4);
+    PyMem_Free(cpathname_ucs4);
     if (dirname == NULL) {
         PyErr_Clear();
         return;
@@ -1461,9 +1485,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp)
         goto error;
     }
 #endif
-    cpathname = make_compiled_pathname(
-        PyUnicode_AS_UNICODE(pathname),
-        !Py_OptimizeFlag);
+    cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag);
 
     if (cpathname != NULL)
         fpc = check_compiled_module(pathname, st.st_mtime, cpathname);
@@ -1512,16 +1534,18 @@ static PyObject *
 get_sourcefile(PyObject *filename)
 {
     Py_ssize_t len;
-    Py_UNICODE *fileuni;
+    Py_UCS4 *fileuni;
     PyObject *py;
     struct stat statbuf;
 
-    len = PyUnicode_GET_SIZE(filename);
+    len = PyUnicode_GET_LENGTH(filename);
     if (len == 0)
         Py_RETURN_NONE;
 
     /* don't match *.pyc or *.pyo? */
-    fileuni = PyUnicode_AS_UNICODE(filename);
+    fileuni = PyUnicode_AsUCS4Copy(filename);
+    if (!fileuni)
+        return NULL;
     if (len < 5
         || fileuni[len-4] != '.'
         || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P')
@@ -1535,7 +1559,7 @@ get_sourcefile(PyObject *filename)
     py = make_source_pathname(filename);
     if (py == NULL) {
         PyErr_Clear();
-        py = PyUnicode_FromUnicode(fileuni, len - 1);
+        py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1);
     }
     if (py == NULL)
         goto error;
@@ -1548,6 +1572,7 @@ get_sourcefile(PyObject *filename)
 error:
     PyErr_Clear();
 unchanged:
+    PyMem_Free(fileuni);
     Py_INCREF(filename);
     return filename;
 }
@@ -1739,8 +1764,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path,
                  PyObject *path_hooks, PyObject *path_importer_cache,
                  PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd)
 {
-    Py_UNICODE buf[MAXPATHLEN+1];
-    Py_ssize_t buflen = MAXPATHLEN+1;
+    Py_UCS4 buf[MAXPATHLEN+1];
     PyObject *path_unicode, *filename;
     Py_ssize_t len;
     struct stat statbuf;
@@ -1759,15 +1783,15 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path,
     else
         return 0;
 
-    len = PyUnicode_GET_SIZE(path_unicode);
-    if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) {
+    len = PyUnicode_GET_LENGTH(path_unicode);
+    if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) {
         Py_DECREF(path_unicode);
-        return 0; /* Too long */
+        PyErr_Clear();
+        return 0;
     }
-    Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode));
     Py_DECREF(path_unicode);
 
-    if (Py_UNICODE_strlen(buf) != len)
+    if (Py_UCS4_strlen(buf) != len)
         return 0; /* path contains '\0' */
 
     /* sys.path_hooks import hook */
@@ -1804,10 +1828,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path,
 #endif
         )
         buf[len++] = SEP;
-    Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name));
-    len += PyUnicode_GET_SIZE(name);
+    if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) {
+        PyErr_Clear();
+        return 0;
+    }
+    len += PyUnicode_GET_LENGTH(name);
 
-    filename = PyUnicode_FromUnicode(buf, len);
+    filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                         buf, len);
     if (filename == NULL)
         return -1;
 
@@ -1989,6 +2017,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list,
     if (p_loader != NULL)
         *p_loader = NULL;
 
+    if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "module name is too long");
+        return NULL;
+    }
+
     /* sys.meta_path import hook */
     if (p_loader != NULL) {
         PyObject *meta_path;
@@ -2704,7 +2738,7 @@ static PyObject *get_parent(PyObject *globals,
                             int level);
 static PyObject *load_next(PyObject *mod, PyObject *altmod,
                            PyObject *inputname, PyObject **p_outputname,
-                           Py_UNICODE *buf, Py_ssize_t *p_buflen,
+                           Py_UCS4 *buf, Py_ssize_t *p_buflen,
                            Py_ssize_t bufsize);
 static int mark_miss(PyObject *name);
 static int ensure_fromlist(PyObject *mod, PyObject *fromlist,
@@ -2718,37 +2752,47 @@ static PyObject *
 import_module_level(PyObject *name, PyObject *globals, PyObject *locals,
                     PyObject *fromlist, int level)
 {
-    Py_UNICODE buf[MAXPATHLEN+1];
+    Py_UCS4 buf[MAXPATHLEN+1];
     Py_ssize_t buflen;
     Py_ssize_t bufsize = MAXPATHLEN+1;
     PyObject *parent, *head, *next, *tail, *inputname, *outputname;
     PyObject *parent_name, *ensure_name;
-    const Py_UNICODE *nameunicode;
+    Py_ssize_t sep, altsep;
 
-    nameunicode = PyUnicode_AS_UNICODE(name);
+    if (PyUnicode_READY(name))
+        return NULL;
 
-    if (Py_UNICODE_strchr(nameunicode, SEP) != NULL
+    sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1);
+    if (sep == -2)
+        return NULL;
 #ifdef ALTSEP
-        || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL
+    altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1);
+    if (altsep == -2)
+        return NULL;
+#else
+    altsep = -1;
 #endif
-        ) {
+    if (sep != -1 || altsep != -1)
+    {
         PyErr_SetString(PyExc_ImportError,
                         "Import by filename is not supported.");
         return NULL;
     }
 
     parent = get_parent(globals, &parent_name, level);
-    if (parent == NULL)
+    if (parent == NULL) {
         return NULL;
+    }
 
-    buflen = PyUnicode_GET_SIZE(parent_name);
-    if (buflen+1 > bufsize) {
+    if (PyUnicode_READY(parent_name))
+        return NULL;
+    buflen = PyUnicode_GET_LENGTH(parent_name);
+    if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) {
         Py_DECREF(parent_name);
         PyErr_SetString(PyExc_ValueError,
                         "Module name too long");
         return NULL;
     }
-    Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name));
     Py_DECREF(parent_name);
 
     head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname,
@@ -2799,7 +2843,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals,
 
     Py_DECREF(head);
 
-    ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf));
+    ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                            buf, Py_UCS4_strlen(buf));
     if (ensure_name == NULL) {
         Py_DECREF(tail);
         return NULL;
@@ -2859,8 +2904,6 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals
 static PyObject *
 get_parent(PyObject *globals, PyObject **p_name, int level)
 {
-    Py_UNICODE name[MAXPATHLEN+1];
-    const Py_ssize_t bufsize = MAXPATHLEN+1;
     PyObject *nameobj;
 
     static PyObject *namestr = NULL;
@@ -2897,7 +2940,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level)
                             "__package__ set to non-string");
             return NULL;
         }
-        if (PyUnicode_GET_SIZE(pkgname) == 0) {
+        if (PyUnicode_GET_LENGTH(pkgname) == 0) {
             if (level > 0) {
                 PyErr_SetString(PyExc_ValueError,
                     "Attempted relative import in non-package");
@@ -2905,12 +2948,8 @@ get_parent(PyObject *globals, PyObject **p_name, int level)
             }
             goto return_none;
         }
-        if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) {
-            PyErr_SetString(PyExc_ValueError,
-                            "Package name too long");
-            return NULL;
-        }
-        Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname));
+        Py_INCREF(pkgname);
+        nameobj = pkgname;
     } else {
         /* __package__ not set, so figure it out and set it */
         modname = PyDict_GetItem(globals, namestr);
@@ -2922,74 +2961,71 @@ get_parent(PyObject *globals, PyObject **p_name, int level)
             /* __path__ is set, so modname is already the package name */
             int error;
 
-            if (PyUnicode_GET_SIZE(modname)+1 > bufsize) {
-                PyErr_SetString(PyExc_ValueError,
-                                "Module name too long");
-                return NULL;
-            }
-            Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname));
             error = PyDict_SetItem(globals, pkgstr, modname);
             if (error) {
                 PyErr_SetString(PyExc_ValueError,
                                 "Could not set __package__");
                 return NULL;
             }
+            Py_INCREF(modname);
+            nameobj = modname;
         } else {
             /* Normal module, so work out the package name if any */
-            Py_UNICODE *start = PyUnicode_AS_UNICODE(modname);
-            Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.');
             Py_ssize_t len;
-            int error;
-            if (lastdot == NULL && level > 0) {
-                PyErr_SetString(PyExc_ValueError,
-                    "Attempted relative import in non-package");
+            len = PyUnicode_FindChar(modname, '.',
+                                     0, PyUnicode_GET_LENGTH(modname), -1);
+            if (len == -2)
                 return NULL;
-            }
-            if (lastdot == NULL) {
-                error = PyDict_SetItem(globals, pkgstr, Py_None);
-                if (error) {
+            if (len < 0) {
+                if (level > 0) {
+                    PyErr_SetString(PyExc_ValueError,
+                        "Attempted relative import in non-package");
+                    return NULL;
+                }
+                if (PyDict_SetItem(globals, pkgstr, Py_None)) {
                     PyErr_SetString(PyExc_ValueError,
                         "Could not set __package__");
                     return NULL;
                 }
                 goto return_none;
             }
-            len = lastdot - start;
-            if (len+1 > bufsize) {
+            pkgname = PyUnicode_Substring(modname, 0, len);
+            if (pkgname == NULL)
+                return NULL;
+            if (PyDict_SetItem(globals, pkgstr, pkgname)) {
+                Py_DECREF(pkgname);
                 PyErr_SetString(PyExc_ValueError,
-                                "Module name too long");
+                                "Could not set __package__");
                 return NULL;
             }
-            Py_UNICODE_strncpy(name, start, len);
-            name[len] = '\0';
-            pkgname = PyUnicode_FromUnicode(name, len);
-            if (pkgname == NULL) {
+            nameobj = pkgname;
+        }
+    }
+    if (level > 1) {
+        Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj);
+        PyObject *newname;
+        while (--level > 0) {
+            dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1);
+            if (dot == -2) {
+                Py_DECREF(nameobj);
                 return NULL;
             }
-            error = PyDict_SetItem(globals, pkgstr, pkgname);
-            Py_DECREF(pkgname);
-            if (error) {
+            if (dot < 0) {
+                Py_DECREF(nameobj);
                 PyErr_SetString(PyExc_ValueError,
-                                "Could not set __package__");
+                    "Attempted relative import beyond "
+                    "toplevel package");
                 return NULL;
             }
+            end = dot;
         }
-    }
-    while (--level > 0) {
-        Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.');
-        if (dot == NULL) {
-            PyErr_SetString(PyExc_ValueError,
-                "Attempted relative import beyond "
-                "toplevel package");
+        newname = PyUnicode_Substring(nameobj, 0, end);
+        Py_DECREF(nameobj);
+        if (newname == NULL)
             return NULL;
-        }
-        *dot = '\0';
+        nameobj = newname;
     }
 
-    nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name));
-    if (nameobj == NULL)
-        return NULL;
-
     modules = PyImport_GetModuleDict();
     parent = PyDict_GetItem(modules, nameobj);
     if (parent == NULL) {
@@ -3021,7 +3057,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level)
        If this is violated...  Who cares? */
 
 return_none:
-    nameobj = PyUnicode_FromUnicode(NULL, 0);
+    nameobj = PyUnicode_New(0, 0);
     if (nameobj == NULL)
         return NULL;
     *p_name = nameobj;
@@ -3032,28 +3068,28 @@ return_none:
 static PyObject *
 load_next(PyObject *mod, PyObject *altmod,
           PyObject *inputname, PyObject **p_outputname,
-          Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize)
+          Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize)
 {
-    const Py_UNICODE *dot;
+    Py_UCS4 *dot;
     Py_ssize_t len;
-    Py_UNICODE *p;
+    Py_UCS4 *p;
     PyObject *fullname, *name, *result, *mark_name;
-    const Py_UNICODE *nameuni;
+    const Py_UCS4 *nameuni;
 
     *p_outputname = NULL;
 
-    if (PyUnicode_GET_SIZE(inputname) == 0) {
+    if (PyUnicode_GET_LENGTH(inputname) == 0) {
         /* completely empty module name should only happen in
            'from . import' (or '__import__("")')*/
         Py_INCREF(mod);
         return mod;
     }
 
-    nameuni = PyUnicode_AS_UNICODE(inputname);
+    nameuni = PyUnicode_AsUCS4Copy(inputname);
     if (nameuni == NULL)
         return NULL;
 
-    dot = Py_UNICODE_strchr(nameuni, '.');
+    dot = Py_UCS4_strchr(nameuni, '.');
     if (dot != NULL) {
         len = dot - nameuni;
         if (len == 0) {
@@ -3063,7 +3099,7 @@ load_next(PyObject *mod, PyObject *altmod,
         }
     }
     else
-        len = PyUnicode_GET_SIZE(inputname);
+        len = PyUnicode_GET_LENGTH(inputname);
 
     if (*p_buflen+len+1 >= bufsize) {
         PyErr_SetString(PyExc_ValueError,
@@ -3076,14 +3112,16 @@ load_next(PyObject *mod, PyObject *altmod,
         *p++ = '.';
         *p_buflen += 1;
     }
-    Py_UNICODE_strncpy(p, nameuni, len);
+    Py_UCS4_strncpy(p, nameuni, len);
     p[len] = '\0';
     *p_buflen += len;
 
-    fullname = PyUnicode_FromUnicode(buf, *p_buflen);
+    fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                         buf, *p_buflen);
     if (fullname == NULL)
         return NULL;
-    name = PyUnicode_FromUnicode(p, len);
+    name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                     p, len);
     if (name == NULL) {
         Py_DECREF(fullname);
         return NULL;
@@ -3096,7 +3134,8 @@ load_next(PyObject *mod, PyObject *altmod,
         result = import_submodule(altmod, name, name);
         Py_DECREF(name);
         if (result != NULL && result != Py_None) {
-            mark_name = PyUnicode_FromUnicode(buf, *p_buflen);
+            mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                                  buf, *p_buflen);
             if (mark_name == NULL) {
                 Py_DECREF(result);
                 return NULL;
@@ -3107,7 +3146,7 @@ load_next(PyObject *mod, PyObject *altmod,
                 return NULL;
             }
             Py_DECREF(mark_name);
-            Py_UNICODE_strncpy(buf, nameuni, len);
+            Py_UCS4_strncpy(buf, nameuni, len);
             buf[len] = '\0';
             *p_buflen = len;
         }
@@ -3125,7 +3164,8 @@ load_next(PyObject *mod, PyObject *altmod,
     }
 
     if (dot != NULL) {
-        *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1));
+        *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                                  dot+1, Py_UCS4_strlen(dot+1));
         if (*p_outputname == NULL) {
             Py_DECREF(result);
             return NULL;
@@ -3166,7 +3206,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name,
             Py_DECREF(item);
             return 0;
         }
-        if (PyUnicode_AS_UNICODE(item)[0] == '*') {
+        if (PyUnicode_READ_CHAR(item, 0) == '*') {
             PyObject *all;
             Py_DECREF(item);
             /* See if the package defines __all__ */
@@ -3304,7 +3344,7 @@ PyImport_ReloadModule(PyObject *m)
     PyObject *modules = PyImport_GetModuleDict();
     PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL;
     PyObject *nameobj, *bufobj, *subnameobj;
-    Py_UNICODE *name, *subname;
+    Py_UCS4 *name = NULL, *subname;
     struct filedescr *fdp;
     FILE *fp = NULL;
     PyObject *newm = NULL;
@@ -3321,7 +3361,7 @@ PyImport_ReloadModule(PyObject *m)
         return NULL;
     }
     nameobj = PyModule_GetNameObject(m);
-    if (nameobj == NULL)
+    if (nameobj == NULL || PyUnicode_READY(nameobj) == -1)
         return NULL;
     if (m != PyDict_GetItem(modules, nameobj)) {
         PyErr_Format(PyExc_ImportError,
@@ -3343,8 +3383,12 @@ PyImport_ReloadModule(PyObject *m)
         return NULL;
     }
 
-    name = PyUnicode_AS_UNICODE(nameobj);
-    subname = Py_UNICODE_strrchr(name, '.');
+    name = PyUnicode_AsUCS4Copy(nameobj);
+    if (!name) {
+        Py_DECREF(nameobj);
+        return NULL;
+    }
+    subname = Py_UCS4_strrchr(name, '.');
     if (subname == NULL) {
         Py_INCREF(nameobj);
         subnameobj = nameobj;
@@ -3353,7 +3397,8 @@ PyImport_ReloadModule(PyObject *m)
         PyObject *parentname, *parent;
         Py_ssize_t len;
         len = subname - name;
-        parentname = PyUnicode_FromUnicode(name, len);
+        parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                               name, len);
         if (parentname == NULL) {
             goto error;
         }
@@ -3370,8 +3415,9 @@ PyImport_ReloadModule(PyObject *m)
         if (path_list == NULL)
             PyErr_Clear();
         subname++;
-        len = PyUnicode_GET_SIZE(nameobj) - (len + 1);
-        subnameobj = PyUnicode_FromUnicode(subname, len);
+        len = PyUnicode_GET_LENGTH(nameobj) - (len + 1);
+        subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
+                                               subname, len);
     }
     if (subnameobj == NULL)
         goto error;
@@ -3403,6 +3449,7 @@ PyImport_ReloadModule(PyObject *m)
 error:
     imp_modules_reloading_clear();
     Py_DECREF(nameobj);
+    PyMem_Free(name);
     return newm;
 }
 
@@ -3910,9 +3957,7 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws)
         return NULL;
     }
 
-    cpathname = make_compiled_pathname(
-        PyUnicode_AS_UNICODE(pathname),
-        debug);
+    cpathname = make_compiled_pathname(pathname, debug);
     Py_DECREF(pathname);
 
     if (cpathname == NULL) {
@@ -4105,7 +4150,7 @@ NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds)
                           &pathobj))
         return -1;
 
-    if (PyUnicode_GET_SIZE(pathobj) == 0) {
+    if (PyUnicode_GET_LENGTH(pathobj) == 0) {
         PyErr_SetString(PyExc_ImportError, "empty pathname");
         return -1;
     }
diff --git a/Python/marshal.c b/Python/marshal.c
index edf0366..31fe66b 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -311,9 +311,7 @@ w_object(PyObject *v, WFILE *p)
     }
     else if (PyUnicode_CheckExact(v)) {
         PyObject *utf8;
-        utf8 = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(v),
-                                    PyUnicode_GET_SIZE(v),
-                                    "surrogatepass");
+        utf8 = PyUnicode_AsEncodedString(v, "utf8", "surrogatepass");
         if (utf8 == NULL) {
             p->depth--;
             p->error = WFERR_UNMARSHALLABLE;
diff --git a/Python/peephole.c b/Python/peephole.c
index 705622f..5d53677 100644
--- a/Python/peephole.c
+++ b/Python/peephole.c
@@ -183,24 +183,6 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts, PyObject **ob
             break;
         case BINARY_SUBSCR:
             newconst = PyObject_GetItem(v, w);
-            /* #5057: if v is unicode, there might be differences between
-               wide and narrow builds in cases like '\U00012345'[0].
-               Wide builds will return a non-BMP char, whereas narrow builds
-               will return a surrogate.  In both the cases skip the
-               optimization in order to produce compatible pycs.
-             */
-            if (newconst != NULL &&
-                PyUnicode_Check(v) && PyUnicode_Check(newconst)) {
-                Py_UNICODE ch = PyUnicode_AS_UNICODE(newconst)[0];
-#ifdef Py_UNICODE_WIDE
-                if (ch > 0xFFFF) {
-#else
-                if (ch >= 0xD800 && ch <= 0xDFFF) {
-#endif
-                    Py_DECREF(newconst);
-                    return 0;
-                }
-            }
             break;
         case BINARY_LSHIFT:
             newconst = PyNumber_Lshift(v, w);
diff --git a/Python/symtable.c b/Python/symtable.c
index a0bedfc..3578b0c 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -1525,10 +1525,10 @@ symtable_visit_alias(struct symtable *st, alias_ty a)
     */
     PyObject *store_name;
     PyObject *name = (a->asname == NULL) ? a->name : a->asname;
-    const Py_UNICODE *base = PyUnicode_AS_UNICODE(name);
-    Py_UNICODE *dot = Py_UNICODE_strchr(base, '.');
-    if (dot) {
-        store_name = PyUnicode_FromUnicode(base, dot - base);
+    Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
+                                        PyUnicode_GET_LENGTH(name), 1);
+    if (dot != -1) {
+        store_name = PyUnicode_Substring(name, 0, dot);
         if (!store_name)
             return 0;
     }
diff --git a/Python/traceback.c b/Python/traceback.c
index e74a147..9a11bf2 100644
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -229,8 +229,8 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent)
     PyObject *lineobj = NULL;
     PyObject *res;
     char buf[MAXPATHLEN+1];
-    Py_UNICODE *u, *p;
-    Py_ssize_t len;
+    int kind;
+    void *data;
 
     /* open the file */
     if (filename == NULL)
@@ -285,13 +285,16 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent)
     }
 
     /* remove the indentation of the line */
-    u = PyUnicode_AS_UNICODE(lineobj);
-    len = PyUnicode_GET_SIZE(lineobj);
-    for (p=u; *p == ' ' || *p == '\t' || *p == '\014'; p++)
-        len--;
-    if (u != p) {
+    kind = PyUnicode_KIND(lineobj);
+    data = PyUnicode_DATA(lineobj);
+    for (i=0; i < PyUnicode_GET_LENGTH(lineobj); i++) {
+        Py_UCS4 ch = PyUnicode_READ(kind, data, i);
+        if (ch != ' ' && ch != '\t' && ch != '\014')
+            break;
+    }
+    if (i) {
         PyObject *truncated;
-        truncated = PyUnicode_FromUnicode(p, len);
+        truncated = PyUnicode_Substring(lineobj, i, PyUnicode_GET_LENGTH(lineobj));
         if (truncated) {
             Py_DECREF(lineobj);
             lineobj = truncated;
@@ -476,13 +479,26 @@ dump_hexadecimal(int width, unsigned long value, int fd)
 static void
 dump_ascii(int fd, PyObject *text)
 {
+    PyASCIIObject *ascii = (PyASCIIObject *)text;
     Py_ssize_t i, size;
     int truncated;
-    Py_UNICODE *u;
-    char c;
-
-    size = PyUnicode_GET_SIZE(text);
-    u = PyUnicode_AS_UNICODE(text);
+    int kind;
+    void *data;
+    Py_UCS4 ch;
+
+    size = ascii->length;
+    kind = ascii->state.kind;
+    if (ascii->state.compact) {
+        if (ascii->state.ascii)
+            data = ((PyASCIIObject*)text) + 1;
+        else
+            data = ((PyCompactUnicodeObject*)text) + 1;
+    }
+    else {
+        data = ((PyUnicodeObject *)text)->data.any;
+        if (data == NULL)
+            return;
+    }
 
     if (MAX_STRING_LENGTH < size) {
         size = MAX_STRING_LENGTH;
@@ -491,27 +507,28 @@ dump_ascii(int fd, PyObject *text)
     else
         truncated = 0;
 
-    for (i=0; i < size; i++, u++) {
-        if (*u < 128) {
-            c = (char)*u;
+    for (i=0; i < size; i++) {
+        ch = PyUnicode_READ(kind, data, i);
+        if (ch < 128) {
+            char c = (char)ch;
             write(fd, &c, 1);
         }
-        else if (*u < 256) {
+        else if (ch < 256) {
             PUTS(fd, "\\x");
-            dump_hexadecimal(2, *u, fd);
+            dump_hexadecimal(2, ch, fd);
         }
         else
 #ifdef Py_UNICODE_WIDE
-        if (*u < 65536)
+        if (ch < 65536)
 #endif
         {
             PUTS(fd, "\\u");
-            dump_hexadecimal(4, *u, fd);
+            dump_hexadecimal(4, ch, fd);
 #ifdef Py_UNICODE_WIDE
         }
         else {
             PUTS(fd, "\\U");
-            dump_hexadecimal(8, *u, fd);
+            dump_hexadecimal(8, ch, fd);
 #endif
         }
     }
@@ -542,7 +559,7 @@ dump_frame(int fd, PyFrameObject *frame)
     }
 
     /* PyFrame_GetLineNumber() was introduced in Python 2.7.0 and 3.2.0 */
-    lineno = PyCode_Addr2Line(frame->f_code, frame->f_lasti);
+    lineno = PyCode_Addr2Line(code, frame->f_lasti);
     PUTS(fd, ", line ");
     dump_decimal(fd, lineno);
     PUTS(fd, " in ");