Issue #13417: speed up utf-8 decoding by around 2x for the non-fully-ASCII case.

This almost catches up with pre-PEP 393 performance, when decoding needed
only one pass.

1 files changed, 156 insertions, 0 deletions

diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
new file mode 100644
index 0000000..148bacb
--- /dev/null
+++ b/Objects/stringlib/codecs.h
@@ -0,0 +1,156 @@
+/* stringlib: codec implementations */
+
+#if STRINGLIB_IS_UNICODE
+
+/* Mask to check or force alignment of a pointer to C 'long' boundaries */
+#define LONG_PTR_MASK (size_t) (SIZEOF_LONG - 1)
+
+/* Mask to quickly check whether a C 'long' contains a
+   non-ASCII, UTF8-encoded char. */
+#if (SIZEOF_LONG == 8)
+# define ASCII_CHAR_MASK 0x8080808080808080L
+#elif (SIZEOF_LONG == 4)
+# define ASCII_CHAR_MASK 0x80808080L
+#else
+# error C 'long' size should be either 4 or 8!
+#endif
+
+Py_LOCAL_INLINE(int)
+STRINGLIB(utf8_try_decode)(const char *start, const char *end,
+                           STRINGLIB_CHAR *dest,
+                           const char **src_pos, Py_ssize_t *dest_index)
+{
+    int ret;
+    Py_ssize_t n;
+    const char *s = start;
+    const char *aligned_end = (const char *) ((size_t) end & ~LONG_PTR_MASK);
+    STRINGLIB_CHAR *p = dest;
+
+    while (s < end) {
+        Py_UCS4 ch = (unsigned char)*s;
+
+        if (ch < 0x80) {
+            /* Fast path for runs of ASCII characters. Given that common UTF-8
+               input will consist of an overwhelming majority of ASCII
+               characters, we try to optimize for this case by checking
+               as many characters as a C 'long' can contain.
+               First, check if we can do an aligned read, as most CPUs have
+               a penalty for unaligned reads.
+            */
+            if (!((size_t) s & LONG_PTR_MASK)) {
+                /* Help register allocation */
+                register const char *_s = s;
+                register STRINGLIB_CHAR *_p = p;
+                while (_s < aligned_end) {
+                    /* Read a whole long at a time (either 4 or 8 bytes),
+                       and do a fast unrolled copy if it only contains ASCII
+                       characters. */
+                    unsigned long value = *(unsigned long *) _s;
+                    if (value & ASCII_CHAR_MASK)
+                        break;
+                    _p[0] = _s[0];
+                    _p[1] = _s[1];
+                    _p[2] = _s[2];
+                    _p[3] = _s[3];
+#if (SIZEOF_LONG == 8)
+                    _p[4] = _s[4];
+                    _p[5] = _s[5];
+                    _p[6] = _s[6];
+                    _p[7] = _s[7];
+#endif
+                    _s += SIZEOF_LONG;
+                    _p += SIZEOF_LONG;
+                }
+                s = _s;
+                p = _p;
+                if (s == end)
+                    break;
+                ch = (unsigned char)*s;
+            }
+        }
+
+        if (ch < 0x80) {
+            s++;
+            *p++ = ch;
+            continue;
+        }
+
+        n = utf8_code_length[ch];
+
+        if (s + n > end) {
+            /* unexpected end of data: the caller will decide whether
+               it's an error or not */
+            goto _error;
+        }
+
+        switch (n) {
+        case 0:
+            /* invalid start byte */
+            goto _error;
+        case 1:
+            /* internal error */
+            goto _error;
+        case 2:
+            if ((s[1] & 0xc0) != 0x80)
+                /* invalid continuation byte */
+                goto _error;
+            ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
+            assert ((ch > 0x007F) && (ch <= 0x07FF));
+            s += 2;
+            *p++ = ch;
+            break;
+
+        case 3:
+            /* Decoding UTF-8 sequences in range \xed\xa0\x80-\xed\xbf\xbf
+               will result in surrogates in range d800-dfff. Surrogates are
+               not valid UTF-8 so they are rejected.
+               See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
+               (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
+            if ((s[1] & 0xc0) != 0x80 ||
+                (s[2] & 0xc0) != 0x80 ||
+                ((unsigned char)s[0] == 0xE0 &&
+                 (unsigned char)s[1] < 0xA0) ||
+                ((unsigned char)s[0] == 0xED &&
+                 (unsigned char)s[1] > 0x9F)) {
+                /* invalid continuation byte */
+                goto _error;
+            }
+            ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
+            assert ((ch > 0x07FF) && (ch <= 0xFFFF));
+            s += 3;
+            *p++ = ch;
+            break;
+
+        case 4:
+            if ((s[1] & 0xc0) != 0x80 ||
+                (s[2] & 0xc0) != 0x80 ||
+                (s[3] & 0xc0) != 0x80 ||
+                ((unsigned char)s[0] == 0xF0 &&
+                 (unsigned char)s[1] < 0x90) ||
+                ((unsigned char)s[0] == 0xF4 &&
+                 (unsigned char)s[1] > 0x8F)) {
+                /* invalid continuation byte */
+                goto _error;
+            }
+            ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
+                 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
+            assert ((ch > 0xFFFF) && (ch <= 0x10ffff));
+            s += 4;
+            *p++ = ch;
+            break;
+        }
+    }
+    ret = 0;
+    goto _ok;
+_error:
+    ret = -1;
+_ok:
+    *src_pos = s;
+    *dest_index = p - dest;
+    return ret;
+}
+
+#undef LONG_PTR_MASK
+#undef ASCII_CHAR_MASK
+
+#endif /* STRINGLIB_IS_UNICODE */