This is my patch:

[ 1181301 ] make float packing copy bytes when they can

which hasn't been reviewed, despite numerous threats to check it in
anyway if noone reviews it.  Please read the diff on the checkin list,
at least!

The basic idea is to examine the bytes of some 'probe values' to see if
the current platform is a IEEE 754-ish platform, and if so
_PyFloat_{Pack,Unpack}{4,8} just copy bytes around.

The rest is hair for testing, and tests.

1 files changed, 509 insertions, 239 deletions

diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index 55f43cb..c95b5c9 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -983,8 +983,139 @@ float_getnewargs(PyFloatObject *v)
 	return Py_BuildValue("(d)", v->ob_fval);
 }
 
+/* this is for the benefit of the pack/unpack routines below */
+
+typedef enum {
+	unknown_format, ieee_big_endian_format, ieee_little_endian_format
+} float_format_type;
+
+static float_format_type double_format, float_format;
+static float_format_type detected_double_format, detected_float_format;
+
+static PyObject *
+float_getformat(PyTypeObject *v, PyObject* arg)
+{
+	char* s;
+	float_format_type r;
+
+	if (!PyString_Check(arg)) {
+		PyErr_Format(PyExc_TypeError,
+	     "__getformat__() argument must be string, not %.500s",
+			     arg->ob_type->tp_name);
+		return NULL;
+	}
+	s = PyString_AS_STRING(arg);
+	if (strcmp(s, "double") == 0) {
+		r = double_format;
+	}
+	else if (strcmp(s, "float") == 0) {
+		r = float_format;
+	}
+	else {
+		PyErr_SetString(PyExc_ValueError,
+				"__getformat__() argument 1 must be "
+				"'double' or 'float'");
+		return NULL;
+	}
+	
+	switch (r) {
+	case unknown_format:
+		return PyString_FromString("unknown");
+	case ieee_little_endian_format:
+		return PyString_FromString("IEEE, little-endian");
+	case ieee_big_endian_format:
+		return PyString_FromString("IEEE, big-endian");
+	default:
+		Py_FatalError("insane float_format or double_format");
+		return NULL;
+	}
+}
+
+PyDoc_STRVAR(float_getformat_doc,
+"float.__getformat__(typestr) -> string\n"
+"\n"
+"You probably don't want to use this function.  It exists mainly to be\n"
+"used in Python's test suite.\n"
+"\n"
+"typestr must be 'double' or 'float'.  This function returns whichever of\n"
+"'unknown', 'IEEE, big-endian' or 'IEEE, little-endian' best describes the\n"
+"format of floating point numbers used by the C type named by typestr.");
+
+static PyObject *
+float_setformat(PyTypeObject *v, PyObject* args)
+{
+	char* typestr;
+	char* format;
+	float_format_type f;
+	float_format_type detected;
+	float_format_type *p;
+
+	if (!PyArg_ParseTuple(args, "ss:__setformat__", &typestr, &format))
+		return NULL;
+
+	if (strcmp(typestr, "double") == 0) {
+		p = &double_format;
+		detected = detected_double_format;
+	}
+	else if (strcmp(typestr, "float") == 0) {
+		p = &float_format;
+		detected = detected_float_format;
+	}
+	else {
+		PyErr_SetString(PyExc_ValueError,
+				"__setformat__() argument 1 must "
+				"be 'double' or 'float'");
+		return NULL;
+	}
+	
+	if (strcmp(format, "unknown") == 0) {
+		f = unknown_format;
+	}
+	else if (strcmp(format, "IEEE, little-endian") == 0) {
+		f = ieee_little_endian_format;
+	}
+	else if (strcmp(format, "IEEE, big-endian") == 0) {
+		f = ieee_big_endian_format;
+	}
+	else {
+		PyErr_SetString(PyExc_ValueError,
+				"__setformat__() argument 2 must be "
+				"'unknown', 'IEEE, little-endian' or "
+				"'IEEE, big-endian'");
+		return NULL;
+
+	}
+
+	if (f != unknown_format && f != detected) {
+		PyErr_Format(PyExc_ValueError,
+			     "can only set %s format to 'unknown' or the "
+			     "detected platform value", typestr);
+		return NULL;
+	}
+
+	*p = f;
+	Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(float_setformat_doc,
+"float.__setformat__(typestr, fmt) -> None\n"
+"\n"
+"You probably don't want to use this function.  It exists mainly to be\n"
+"used in Python's test suite.\n"
+"\n"
+"typestr must be 'double' or 'float'.  fmt must be one of 'unknown',\n"
+"'IEEE, big-endian' or 'IEEE, little-endian', and in addition can only be\n"
+"one of the latter two if it appears to match the underlying C reality.\n"
+"\n"
+"Overrides the automatic determination of C-level floating point type.\n"
+"This affects how floats are converted to and from binary strings.");
+
 static PyMethodDef float_methods[] = {
 	{"__getnewargs__",	(PyCFunction)float_getnewargs,	METH_NOARGS},
+	{"__getformat__",	(PyCFunction)float_getformat,	
+	 METH_O|METH_CLASS,		float_getformat_doc},
+	{"__setformat__",	(PyCFunction)float_setformat,	
+	 METH_VARARGS|METH_CLASS,	float_setformat_doc},
 	{NULL,		NULL}		/* sentinel */
 };
 
@@ -1079,6 +1210,56 @@ PyTypeObject PyFloat_Type = {
 };
 
 void
+_PyFloat_Init(void)
+{
+	/* We attempt to determine if this machine is using IEEE
+	   floating point formats by peering at the bits of some
+	   carefully chosen values.  If it looks like we are on an
+	   IEEE platform, the float packing/unpacking routines can
+	   just copy bits, if not they resort to arithmetic & shifts
+	   and masks.  The shifts & masks approach works on all finite
+	   values, but what happens to infinities, NaNs and signed
+	   zeroes on packing is an accident, and attempting to unpack
+	   a NaN or an infinity will raise an exception.
+
+	   Note that if we're on some whacked-out platform which uses
+	   IEEE formats but isn't strictly little-endian or big-
+	   endian, we will fall back to the portable shifts & masks
+	   method. */
+
+#if SIZEOF_DOUBLE == 8
+	{
+		double x = 9006104071832581.0;
+		if (memcmp(&x, "\x43\x3f\xff\x01\x02\x03\x04\x05", 8) == 0)
+			detected_double_format = ieee_big_endian_format;
+		else if (memcmp(&x, "\x05\x04\x03\x02\x01\xff\x3f\x43", 8) == 0)
+			detected_double_format = ieee_little_endian_format;
+		else 
+			detected_double_format = unknown_format;
+	}
+#else
+	detected_double_format = unknown_format;
+#endif
+
+#if SIZEOF_FLOAT == 4
+	{
+		float y = 16711938.0;
+		if (memcmp(&y, "\x4b\x7f\x01\x02", 4) == 0)
+			detected_float_format = ieee_big_endian_format;
+		else if (memcmp(&y, "\x02\x01\x7f\x4b", 4) == 0)
+			detected_float_format = ieee_little_endian_format;
+		else 
+			detected_float_format = unknown_format;
+	}
+#else
+	detected_float_format = unknown_format;
+#endif
+
+	double_format = detected_double_format;
+	float_format = detected_float_format;
+}
+
+void
 PyFloat_Fini(void)
 {
 	PyFloatObject *p;
@@ -1165,306 +1346,395 @@ PyFloat_Fini(void)
 int
 _PyFloat_Pack4(double x, unsigned char *p, int le)
 {
-	unsigned char sign;
-	int e;
-	double f;
-	unsigned int fbits;
-	int incr = 1;
-
-	if (le) {
-		p += 3;
-		incr = -1;
-	}
-
-	if (x < 0) {
-		sign = 1;
-		x = -x;
-	}
-	else
-		sign = 0;
+	if (float_format == unknown_format) {
+		unsigned char sign;
+		int e;
+		double f;
+		unsigned int fbits;
+		int incr = 1;
+
+		if (le) {
+			p += 3;
+			incr = -1;
+		}
 
-	f = frexp(x, &e);
+		if (x < 0) {
+			sign = 1;
+			x = -x;
+		}
+		else
+			sign = 0;
 
-	/* Normalize f to be in the range [1.0, 2.0) */
-	if (0.5 <= f && f < 1.0) {
-		f *= 2.0;
-		e--;
-	}
-	else if (f == 0.0)
-		e = 0;
-	else {
-		PyErr_SetString(PyExc_SystemError,
-				"frexp() result out of range");
-		return -1;
-	}
+		f = frexp(x, &e);
 
-	if (e >= 128)
-		goto Overflow;
-	else if (e < -126) {
-		/* Gradual underflow */
-		f = ldexp(f, 126 + e);
-		e = 0;
-	}
-	else if (!(e == 0 && f == 0.0)) {
-		e += 127;
-		f -= 1.0; /* Get rid of leading 1 */
-	}
+		/* Normalize f to be in the range [1.0, 2.0) */
+		if (0.5 <= f && f < 1.0) {
+			f *= 2.0;
+			e--;
+		}
+		else if (f == 0.0)
+			e = 0;
+		else {
+			PyErr_SetString(PyExc_SystemError,
+					"frexp() result out of range");
+			return -1;
+		}
 
-	f *= 8388608.0; /* 2**23 */
-	fbits = (unsigned int)(f + 0.5); /* Round */
-	assert(fbits <= 8388608);
-	if (fbits >> 23) {
-		/* The carry propagated out of a string of 23 1 bits. */
-		fbits = 0;
-		++e;
-		if (e >= 255)
+		if (e >= 128)
 			goto Overflow;
-	}
+		else if (e < -126) {
+			/* Gradual underflow */
+			f = ldexp(f, 126 + e);
+			e = 0;
+		}
+		else if (!(e == 0 && f == 0.0)) {
+			e += 127;
+			f -= 1.0; /* Get rid of leading 1 */
+		}
+
+		f *= 8388608.0; /* 2**23 */
+		fbits = (unsigned int)(f + 0.5); /* Round */
+		assert(fbits <= 8388608);
+		if (fbits >> 23) {
+			/* The carry propagated out of a string of 23 1 bits. */
+			fbits = 0;
+			++e;
+			if (e >= 255)
+				goto Overflow;
+		}
 
-	/* First byte */
-	*p = (sign << 7) | (e >> 1);
-	p += incr;
+		/* First byte */
+		*p = (sign << 7) | (e >> 1);
+		p += incr;
 
-	/* Second byte */
-	*p = (char) (((e & 1) << 7) | (fbits >> 16));
-	p += incr;
+		/* Second byte */
+		*p = (char) (((e & 1) << 7) | (fbits >> 16));
+		p += incr;
 
-	/* Third byte */
-	*p = (fbits >> 8) & 0xFF;
-	p += incr;
+		/* Third byte */
+		*p = (fbits >> 8) & 0xFF;
+		p += incr;
 
-	/* Fourth byte */
-	*p = fbits & 0xFF;
+		/* Fourth byte */
+		*p = fbits & 0xFF;
 
-	/* Done */
-	return 0;
+		/* Done */
+		return 0;
 
- Overflow:
-	PyErr_SetString(PyExc_OverflowError,
-			"float too large to pack with f format");
-	return -1;
+	  Overflow:
+		PyErr_SetString(PyExc_OverflowError,
+				"float too large to pack with f format");
+		return -1;
+	}
+	else {
+		float y = x;
+		const char *s = (char*)&y;
+		int i, incr = 1;
+
+		if ((float_format == ieee_little_endian_format && !le)
+		    || (float_format == ieee_big_endian_format && le)) {
+			p += 3;
+			incr = -1;
+		}
+		
+		for (i = 0; i < 4; i++) {
+			*p = *s++;
+			p += incr;
+		}
+		return 0;
+	}
 }
 
 int
 _PyFloat_Pack8(double x, unsigned char *p, int le)
 {
-	unsigned char sign;
-	int e;
-	double f;
-	unsigned int fhi, flo;
-	int incr = 1;
+	if (double_format == unknown_format) {
+		unsigned char sign;
+		int e;
+		double f;
+		unsigned int fhi, flo;
+		int incr = 1;
+
+		if (le) {
+			p += 7;
+			incr = -1;
+		}
 
-	if (le) {
-		p += 7;
-		incr = -1;
-	}
+		if (x < 0) {
+			sign = 1;
+			x = -x;
+		}
+		else
+			sign = 0;
 
-	if (x < 0) {
-		sign = 1;
-		x = -x;
-	}
-	else
-		sign = 0;
+		f = frexp(x, &e);
 
-	f = frexp(x, &e);
+		/* Normalize f to be in the range [1.0, 2.0) */
+		if (0.5 <= f && f < 1.0) {
+			f *= 2.0;
+			e--;
+		}
+		else if (f == 0.0)
+			e = 0;
+		else {
+			PyErr_SetString(PyExc_SystemError,
+					"frexp() result out of range");
+			return -1;
+		}
 
-	/* Normalize f to be in the range [1.0, 2.0) */
-	if (0.5 <= f && f < 1.0) {
-		f *= 2.0;
-		e--;
-	}
-	else if (f == 0.0)
-		e = 0;
-	else {
-		PyErr_SetString(PyExc_SystemError,
-				"frexp() result out of range");
-		return -1;
-	}
+		if (e >= 1024)
+			goto Overflow;
+		else if (e < -1022) {
+			/* Gradual underflow */
+			f = ldexp(f, 1022 + e);
+			e = 0;
+		}
+		else if (!(e == 0 && f == 0.0)) {
+			e += 1023;
+			f -= 1.0; /* Get rid of leading 1 */
+		}
 
-	if (e >= 1024)
-		goto Overflow;
-	else if (e < -1022) {
-		/* Gradual underflow */
-		f = ldexp(f, 1022 + e);
-		e = 0;
-	}
-	else if (!(e == 0 && f == 0.0)) {
-		e += 1023;
-		f -= 1.0; /* Get rid of leading 1 */
-	}
-
-	/* fhi receives the high 28 bits; flo the low 24 bits (== 52 bits) */
-	f *= 268435456.0; /* 2**28 */
-	fhi = (unsigned int)f; /* Truncate */
-	assert(fhi < 268435456);
-
-	f -= (double)fhi;
-	f *= 16777216.0; /* 2**24 */
-	flo = (unsigned int)(f + 0.5); /* Round */
-	assert(flo <= 16777216);
-	if (flo >> 24) {
-		/* The carry propagated out of a string of 24 1 bits. */
-		flo = 0;
-		++fhi;
-		if (fhi >> 28) {
-			/* And it also progagated out of the next 28 bits. */
-			fhi = 0;
-			++e;
-			if (e >= 2047)
-				goto Overflow;
+		/* fhi receives the high 28 bits; flo the low 24 bits (== 52 bits) */
+		f *= 268435456.0; /* 2**28 */
+		fhi = (unsigned int)f; /* Truncate */
+		assert(fhi < 268435456);
+
+		f -= (double)fhi;
+		f *= 16777216.0; /* 2**24 */
+		flo = (unsigned int)(f + 0.5); /* Round */
+		assert(flo <= 16777216);
+		if (flo >> 24) {
+			/* The carry propagated out of a string of 24 1 bits. */
+			flo = 0;
+			++fhi;
+			if (fhi >> 28) {
+				/* And it also progagated out of the next 28 bits. */
+				fhi = 0;
+				++e;
+				if (e >= 2047)
+					goto Overflow;
+			}
 		}
-	}
 
-	/* First byte */
-	*p = (sign << 7) | (e >> 4);
-	p += incr;
+		/* First byte */
+		*p = (sign << 7) | (e >> 4);
+		p += incr;
 
-	/* Second byte */
-	*p = (unsigned char) (((e & 0xF) << 4) | (fhi >> 24));
-	p += incr;
+		/* Second byte */
+		*p = (unsigned char) (((e & 0xF) << 4) | (fhi >> 24));
+		p += incr;
 
-	/* Third byte */
-	*p = (fhi >> 16) & 0xFF;
-	p += incr;
+		/* Third byte */
+		*p = (fhi >> 16) & 0xFF;
+		p += incr;
 
-	/* Fourth byte */
-	*p = (fhi >> 8) & 0xFF;
-	p += incr;
+		/* Fourth byte */
+		*p = (fhi >> 8) & 0xFF;
+		p += incr;
 
-	/* Fifth byte */
-	*p = fhi & 0xFF;
-	p += incr;
+		/* Fifth byte */
+		*p = fhi & 0xFF;
+		p += incr;
 
-	/* Sixth byte */
-	*p = (flo >> 16) & 0xFF;
-	p += incr;
+		/* Sixth byte */
+		*p = (flo >> 16) & 0xFF;
+		p += incr;
 
-	/* Seventh byte */
-	*p = (flo >> 8) & 0xFF;
-	p += incr;
+		/* Seventh byte */
+		*p = (flo >> 8) & 0xFF;
+		p += incr;
 
-	/* Eighth byte */
-	*p = flo & 0xFF;
-	p += incr;
+		/* Eighth byte */
+		*p = flo & 0xFF;
+		p += incr;
 
-	/* Done */
-	return 0;
+		/* Done */
+		return 0;
+
+	  Overflow:
+		PyErr_SetString(PyExc_OverflowError,
+				"float too large to pack with d format");
+		return -1;
+	}
+	else {
+		const char *s = (char*)&x;
+		int i, incr = 1;
 
- Overflow:
-	PyErr_SetString(PyExc_OverflowError,
-			"float too large to pack with d format");
-	return -1;
+		if ((double_format == ieee_little_endian_format && !le)
+		    || (double_format == ieee_big_endian_format && le)) {
+			p += 7;
+			incr = -1;
+		}
+		
+		for (i = 0; i < 8; i++) {
+			*p = *s++;
+			p += incr;
+		}
+		return 0;
+	}
 }
 
 double
 _PyFloat_Unpack4(const unsigned char *p, int le)
 {
-	unsigned char sign;
-	int e;
-	unsigned int f;
-	double x;
-	int incr = 1;
+	if (float_format == unknown_format) {
+		unsigned char sign;
+		int e;
+		unsigned int f;
+		double x;
+		int incr = 1;
+
+		if (le) {
+			p += 3;
+			incr = -1;
+		}
 
-	if (le) {
-		p += 3;
-		incr = -1;
-	}
+		/* First byte */
+		sign = (*p >> 7) & 1;
+		e = (*p & 0x7F) << 1;
+		p += incr;
+
+		/* Second byte */
+		e |= (*p >> 7) & 1;
+		f = (*p & 0x7F) << 16;
+		p += incr;
+
+		if (e == 255) {
+			PyErr_SetString(
+				PyExc_ValueError,
+				"can't unpack IEEE 754 special value "
+				"on non-IEEE platform");
+			return -1;
+		}
 
-	/* First byte */
-	sign = (*p >> 7) & 1;
-	e = (*p & 0x7F) << 1;
-	p += incr;
+		/* Third byte */
+		f |= *p << 8;
+		p += incr;
 
-	/* Second byte */
-	e |= (*p >> 7) & 1;
-	f = (*p & 0x7F) << 16;
-	p += incr;
+		/* Fourth byte */
+		f |= *p;
 
-	/* Third byte */
-	f |= *p << 8;
-	p += incr;
+		x = (double)f / 8388608.0;
 
-	/* Fourth byte */
-	f |= *p;
+		/* XXX This sadly ignores Inf/NaN issues */
+		if (e == 0)
+			e = -126;
+		else {
+			x += 1.0;
+			e -= 127;
+		}
+		x = ldexp(x, e);
 
-	x = (double)f / 8388608.0;
+		if (sign)
+			x = -x;
 
-	/* XXX This sadly ignores Inf/NaN issues */
-	if (e == 0)
-		e = -126;
-	else {
-		x += 1.0;
-		e -= 127;
+		return x;
 	}
-	x = ldexp(x, e);
-
-	if (sign)
-		x = -x;
-
-	return x;
+	else {
+		if ((float_format == ieee_little_endian_format && !le)
+		    || (float_format == ieee_big_endian_format && le)) {
+			char buf[8];
+			char *d = &buf[3];
+			int i;
+
+			for (i = 0; i < 4; i++) {
+				*d-- = *p++;
+			}
+			return *(float*)&buf[0];
+		}
+		else {
+			return *(float*)p;
+		}
+	}		
 }
 
 double
 _PyFloat_Unpack8(const unsigned char *p, int le)
 {
-	unsigned char sign;
-	int e;
-	unsigned int fhi, flo;
-	double x;
-	int incr = 1;
+	if (double_format == unknown_format) {
+		unsigned char sign;
+		int e;
+		unsigned int fhi, flo;
+		double x;
+		int incr = 1;
+
+		if (le) {
+			p += 7;
+			incr = -1;
+		}
 
-	if (le) {
-		p += 7;
-		incr = -1;
-	}
+		/* First byte */
+		sign = (*p >> 7) & 1;
+		e = (*p & 0x7F) << 4;
+		
+		p += incr;
+
+		/* Second byte */
+		e |= (*p >> 4) & 0xF;
+		fhi = (*p & 0xF) << 24;
+		p += incr;
+
+		if (e == 2047) {
+			PyErr_SetString(
+				PyExc_ValueError,
+				"can't unpack IEEE 754 special value "
+				"on non-IEEE platform");
+			return -1.0;
+		}
 
-	/* First byte */
-	sign = (*p >> 7) & 1;
-	e = (*p & 0x7F) << 4;
-	p += incr;
+		/* Third byte */
+		fhi |= *p << 16;
+		p += incr;
 
-	/* Second byte */
-	e |= (*p >> 4) & 0xF;
-	fhi = (*p & 0xF) << 24;
-	p += incr;
+		/* Fourth byte */
+		fhi |= *p  << 8;
+		p += incr;
 
-	/* Third byte */
-	fhi |= *p << 16;
-	p += incr;
+		/* Fifth byte */
+		fhi |= *p;
+		p += incr;
 
-	/* Fourth byte */
-	fhi |= *p  << 8;
-	p += incr;
+		/* Sixth byte */
+		flo = *p << 16;
+		p += incr;
 
-	/* Fifth byte */
-	fhi |= *p;
-	p += incr;
+		/* Seventh byte */
+		flo |= *p << 8;
+		p += incr;
 
-	/* Sixth byte */
-	flo = *p << 16;
-	p += incr;
+		/* Eighth byte */
+		flo |= *p;
 
-	/* Seventh byte */
-	flo |= *p << 8;
-	p += incr;
+		x = (double)fhi + (double)flo / 16777216.0; /* 2**24 */
+		x /= 268435456.0; /* 2**28 */
 
-	/* Eighth byte */
-	flo |= *p;
+		if (e == 0)
+			e = -1022;
+		else {
+			x += 1.0;
+			e -= 1023;
+		}
+		x = ldexp(x, e);
 
-	x = (double)fhi + (double)flo / 16777216.0; /* 2**24 */
-	x /= 268435456.0; /* 2**28 */
+		if (sign)
+			x = -x;
 
-	/* XXX This sadly ignores Inf/NaN */
-	if (e == 0)
-		e = -1022;
+		return x;
+	}
 	else {
-		x += 1.0;
-		e -= 1023;
+		if ((double_format == ieee_little_endian_format && !le)
+		    || (double_format == ieee_big_endian_format && le)) {
+			char buf[8];
+			char *d = &buf[7];
+			int i;
+			
+			for (i = 0; i < 8; i++) {
+				*d-- = *p++;
+			}
+			return *(double*)&buf[0];
+		}
+		else {
+			return *(double*)p;
+		}
 	}
-	x = ldexp(x, e);
-
-	if (sign)
-		x = -x;
-
-	return x;
 }