summaryrefslogtreecommitdiffstats
path: root/Objects/stringlib/codecs.h
diff options
context:
space:
mode:
authorSerhiy Storchaka <storchaka@gmail.com>2014-01-04 17:25:37 (GMT)
committerSerhiy Storchaka <storchaka@gmail.com>2014-01-04 17:25:37 (GMT)
commit583a93943c0f090ac2699d7feb33e72bfed7323e (patch)
tree8f0423e5dad407e163eed5082496f53155944266 /Objects/stringlib/codecs.h
parent41adc2670882d49b3120ed2f6803c5a289650648 (diff)
downloadcpython-583a93943c0f090ac2699d7feb33e72bfed7323e.zip
cpython-583a93943c0f090ac2699d7feb33e72bfed7323e.tar.gz
cpython-583a93943c0f090ac2699d7feb33e72bfed7323e.tar.bz2
Issue #15027: Rewrite the UTF-32 encoder. It is now 1.6x to 3.5x faster.
Diffstat (limited to 'Objects/stringlib/codecs.h')
-rw-r--r--Objects/stringlib/codecs.h87
1 files changed, 87 insertions, 0 deletions
diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h
index 2eb2d14..cd5d944 100644
--- a/Objects/stringlib/codecs.h
+++ b/Objects/stringlib/codecs.h
@@ -718,6 +718,93 @@ STRINGLIB(utf16_encode)(const STRINGLIB_CHAR *in,
return len - (end - in + 1);
#endif
}
+
+#if STRINGLIB_SIZEOF_CHAR == 1
+# define SWAB4(CH, tmp) ((CH) << 24) /* high bytes are zero */
+#elif STRINGLIB_SIZEOF_CHAR == 2
+# define SWAB4(CH, tmp) (tmp = (CH), \
+ ((tmp & 0x00FFu) << 24) + ((tmp & 0xFF00u) << 8))
+ /* high bytes are zero */
+#else
+# define SWAB4(CH, tmp) (tmp = (CH), \
+ tmp = ((tmp & 0x00FF00FFu) << 8) + ((tmp >> 8) & 0x00FF00FFu), \
+ ((tmp & 0x0000FFFFu) << 16) + ((tmp >> 16) & 0x0000FFFFu))
+#endif
+Py_LOCAL_INLINE(Py_ssize_t)
+STRINGLIB(utf32_encode)(const STRINGLIB_CHAR *in,
+ Py_ssize_t len,
+ PY_UINT32_T **outptr,
+ int native_ordering)
+{
+ PY_UINT32_T *out = *outptr;
+ const STRINGLIB_CHAR *end = in + len;
+ if (native_ordering) {
+ const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4);
+ while (in < unrolled_end) {
+#if STRINGLIB_SIZEOF_CHAR > 1
+ /* check if any character is a surrogate character */
+ if (((in[0] ^ 0xd800) &
+ (in[1] ^ 0xd800) &
+ (in[2] ^ 0xd800) &
+ (in[3] ^ 0xd800) & 0xf800) == 0)
+ break;
+#endif
+ out[0] = in[0];
+ out[1] = in[1];
+ out[2] = in[2];
+ out[3] = in[3];
+ in += 4; out += 4;
+ }
+ while (in < end) {
+ Py_UCS4 ch;
+ ch = *in++;
+#if STRINGLIB_SIZEOF_CHAR > 1
+ if (Py_UNICODE_IS_SURROGATE(ch)) {
+ /* reject surrogate characters (U+DC800-U+DFFF) */
+ goto fail;
+ }
+#endif
+ *out++ = ch;
+ }
+ } else {
+ const STRINGLIB_CHAR *unrolled_end = in + _Py_SIZE_ROUND_DOWN(len, 4);
+ while (in < unrolled_end) {
+#if STRINGLIB_SIZEOF_CHAR > 1
+ Py_UCS4 ch1, ch2, ch3, ch4;
+ /* check if any character is a surrogate character */
+ if (((in[0] ^ 0xd800) &
+ (in[1] ^ 0xd800) &
+ (in[2] ^ 0xd800) &
+ (in[3] ^ 0xd800) & 0xf800) == 0)
+ break;
+#endif
+ out[0] = SWAB4(in[0], ch1);
+ out[1] = SWAB4(in[1], ch2);
+ out[2] = SWAB4(in[2], ch3);
+ out[3] = SWAB4(in[3], ch4);
+ in += 4; out += 4;
+ }
+ while (in < end) {
+ Py_UCS4 ch = *in++;
+#if STRINGLIB_SIZEOF_CHAR > 1
+ if (Py_UNICODE_IS_SURROGATE(ch)) {
+ /* reject surrogate characters (U+DC800-U+DFFF) */
+ goto fail;
+ }
+#endif
+ *out++ = SWAB4(ch, ch);
+ }
+ }
+ *outptr = out;
+ return len;
+#if STRINGLIB_SIZEOF_CHAR > 1
+ fail:
+ *outptr = out;
+ return len - (end - in + 1);
+#endif
+}
+#undef SWAB4
+
#endif
#endif /* STRINGLIB_IS_UNICODE */