Merged revisions 63562,63570,63728,63734,63784,63788,63802,63817,63827,63839,63887,63975,63998 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r63562 | martin.v.loewis | 2008-05-23 17:06:50 +0200 (Fri, 23 May 2008) | 2 lines

  Patch #1722225: Support QNX 6.
........
  r63570 | trent.nelson | 2008-05-23 22:33:14 +0200 (Fri, 23 May 2008) | 1 line

  Introduce a user macro named $(externalsDir), which should point to the root directory of where all the external sources should live.  Developers can change this value if their external sources live elsewhere.  The default of '..\..' matches the current status quo.
........
  r63728 | gregory.p.smith | 2008-05-26 23:16:34 +0200 (Mon, 26 May 2008) | 4 lines

  Fix issue2589: there was a potential integer overflow leading to
  memory corruption on esoteric platforms and incorrect behavior on
  normal platforms.
........
  r63734 | gregory.p.smith | 2008-05-27 00:07:28 +0200 (Tue, 27 May 2008) | 3 lines

  Fix issue2588: Do not execute str[size-1] = '\0' when a 0 size is
  passed in.  (The assert won't prevent this in non-debug builds).
........
  r63784 | raymond.hettinger | 2008-05-29 10:38:23 +0200 (Thu, 29 May 2008) | 1 line

  Fix two typos.
........
  r63788 | facundo.batista | 2008-05-29 18:39:26 +0200 (Thu, 29 May 2008) | 6 lines


  Fixed the semantic of timeout for socket.create_connection and
  all the upper level libraries that use it, including urllib2.
  Added and fixed some tests, and changed docs correspondingly.
  Thanks to John J Lee for the patch and the pusing, :)
........
  r63802 | mark.dickinson | 2008-05-30 04:46:53 +0200 (Fri, 30 May 2008) | 2 lines

  Fix typo in testSum
........
  r63817 | raymond.hettinger | 2008-05-30 20:20:50 +0200 (Fri, 30 May 2008) | 8 lines

  * Mark intermedidate computes values (hi, lo, yr) as volatile.
  * Expand comments.
  * Swap variable names in the sum_exact code so that x and y
    are consistently chosen as the larger and smaller magnitude
    values respectively.
........
  r63827 | raymond.hettinger | 2008-05-31 05:24:31 +0200 (Sat, 31 May 2008) | 1 line

  Implement heapq in terms of less-than (to match list.sort()).
........
  r63839 | gerhard.haering | 2008-05-31 23:33:27 +0200 (Sat, 31 May 2008) | 2 lines

  Fixed rowcount for SELECT statements. They're -1 now (again), for better DB-API 2.0 compliance.
........
  r63887 | gregory.p.smith | 2008-06-02 06:05:52 +0200 (Mon, 02 Jun 2008) | 4 lines

  Fix issue 2782: be less strict about the format string type in strftime.
  Accept unicode and anything else ParseTuple "s#" can deal with.  This
  matches the time.strftime behavior.
........
  r63975 | neal.norwitz | 2008-06-06 06:47:01 +0200 (Fri, 06 Jun 2008) | 3 lines

  Aldo Cortesi confirmed this is still needed for OpenBSD 4.2 and 4.3.
  (I didn't regen configure, since I don't have a working autoconf.)
........
  r63998 | raymond.hettinger | 2008-06-06 23:47:51 +0200 (Fri, 06 Jun 2008) | 1 line

  Issue 3501: Make heapq support both __le__ and __lt__.
........

8 files changed, 110 insertions, 62 deletions

diff --git a/Modules/_ctypes/libffi/configure b/Modules/_ctypes/libffi/configure
index ee8f636..aaa7914 100755
--- a/Modules/_ctypes/libffi/configure
+++ b/Modules/_ctypes/libffi/configure
@@ -20406,6 +20406,9 @@ case "$host" in
   i?86-*-solaris2.1[0-9]*)
 	TARGET=X86_64; TARGETDIR=x86
 	;;
+  i*86-*-nto-qnx*)
+        TARGET=X86; TARGETDIR=x86
+        ;;
   i?86-*-*)
 	TARGET=X86; TARGETDIR=x86
 	;;
diff --git a/Modules/_ctypes/libffi/configure.ac b/Modules/_ctypes/libffi/configure.ac
index 0a85146..57c161b 100644
--- a/Modules/_ctypes/libffi/configure.ac
+++ b/Modules/_ctypes/libffi/configure.ac
@@ -86,6 +86,9 @@ case "$host" in
   i?86-*-solaris2.1[[0-9]]*)
 	TARGET=X86_64; TARGETDIR=x86
 	;;
+  i*86-*-nto-qnx*) 
+        TARGET=X86; TARGETDIR=x86
+        ;;
   i?86-*-*)
 	TARGET=X86; TARGETDIR=x86
 	;;
diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c
index 29e9f4c..b7d58c0 100644
--- a/Modules/_ctypes/malloc_closure.c
+++ b/Modules/_ctypes/malloc_closure.c
@@ -44,7 +44,11 @@ static void more_core(void)
 	}
 #else
 	if (!_pagesize) {
+#ifdef _SC_PAGESIZE
+		_pagesize = sysconf(_SC_PAGESIZE);
+#else
 		_pagesize = getpagesize();
+#endif
 	}
 #endif
 
diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c
index 97ccd86..a5a5605 100644
--- a/Modules/_heapqmodule.c
+++ b/Modules/_heapqmodule.c
@@ -8,6 +8,25 @@ annotated by François Pinard, and converted to C by Raymond Hettinger.
 
 #include "Python.h"
 
+/* Older implementations of heapq used Py_LE for comparisons.  Now, it uses
+   Py_LT so it will match min(), sorted(), and bisect().  Unfortunately, some
+   client code (Twisted for example) relied on Py_LE, so this little function
+   restores compatability by trying both.
+*/
+static int
+cmp_lt(PyObject *x, PyObject *y)
+{
+	int cmp;
+	cmp = PyObject_RichCompareBool(x, y, Py_LT);
+	if (cmp == -1 && PyErr_ExceptionMatches(PyExc_AttributeError)) {
+		PyErr_Clear();
+		cmp = PyObject_RichCompareBool(y, x, Py_LE);
+		if (cmp != -1)
+			cmp = 1 - cmp;
+	}
+	return cmp;
+}
+
 static int
 _siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
 {
@@ -28,12 +47,12 @@ _siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
 	while (pos > startpos){
 		parentpos = (pos - 1) >> 1;
 		parent = PyList_GET_ITEM(heap, parentpos);
-		cmp = PyObject_RichCompareBool(parent, newitem, Py_LE);
+		cmp = cmp_lt(newitem, parent);
 		if (cmp == -1) {
 			Py_DECREF(newitem);
 			return -1;
 		}
-		if (cmp == 1)
+		if (cmp == 0)
 			break;
 		Py_INCREF(parent);
 		Py_DECREF(PyList_GET_ITEM(heap, pos));
@@ -68,15 +87,14 @@ _siftup(PyListObject *heap, Py_ssize_t pos)
 		/* Set childpos to index of smaller child.   */
 		rightpos = childpos + 1;
 		if (rightpos < endpos) {
-			cmp = PyObject_RichCompareBool(
-				PyList_GET_ITEM(heap, rightpos),
+			cmp = cmp_lt(
 				PyList_GET_ITEM(heap, childpos),
-				Py_LE);
+				PyList_GET_ITEM(heap, rightpos));
 			if (cmp == -1) {
 				Py_DECREF(newitem);
 				return -1;
 			}
-			if (cmp == 1)
+			if (cmp == 0)
 				childpos = rightpos;
 		}
 		/* Move the smaller child up. */
@@ -214,10 +232,10 @@ heappushpop(PyObject *self, PyObject *args)
 		return item;
 	}
 
-	cmp = PyObject_RichCompareBool(item, PyList_GET_ITEM(heap, 0), Py_LE);
+	cmp = cmp_lt(PyList_GET_ITEM(heap, 0), item);
 	if (cmp == -1)
 		return NULL;
-	if (cmp == 1) {
+	if (cmp == 0) {
 		Py_INCREF(item);
 		return item;
 	}
@@ -270,6 +288,7 @@ nlargest(PyObject *self, PyObject *args)
 {
 	PyObject *heap=NULL, *elem, *iterable, *sol, *it, *oldelem;
 	Py_ssize_t i, n;
+	int cmp;
 
 	if (!PyArg_ParseTuple(args, "nO:nlargest", &n, &iterable))
 		return NULL;
@@ -312,7 +331,12 @@ nlargest(PyObject *self, PyObject *args)
 			else
 				goto sortit;
 		}
-		if (PyObject_RichCompareBool(elem, sol, Py_LE)) {
+		cmp = cmp_lt(sol, elem);
+		if (cmp == -1) {
+			Py_DECREF(elem);
+			goto fail;
+		}
+		if (cmp == 0) {
 			Py_DECREF(elem);
 			continue;
 		}
@@ -362,12 +386,12 @@ _siftdownmax(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos)
 	while (pos > startpos){
 		parentpos = (pos - 1) >> 1;
 		parent = PyList_GET_ITEM(heap, parentpos);
-		cmp = PyObject_RichCompareBool(newitem, parent, Py_LE);
+		cmp = cmp_lt(parent, newitem);
 		if (cmp == -1) {
 			Py_DECREF(newitem);
 			return -1;
 		}
-		if (cmp == 1)
+		if (cmp == 0)
 			break;
 		Py_INCREF(parent);
 		Py_DECREF(PyList_GET_ITEM(heap, pos));
@@ -402,15 +426,14 @@ _siftupmax(PyListObject *heap, Py_ssize_t pos)
 		/* Set childpos to index of smaller child.   */
 		rightpos = childpos + 1;
 		if (rightpos < endpos) {
-			cmp = PyObject_RichCompareBool(
-				PyList_GET_ITEM(heap, childpos),
+			cmp = cmp_lt(
 				PyList_GET_ITEM(heap, rightpos),
-				Py_LE);
+				PyList_GET_ITEM(heap, childpos));
 			if (cmp == -1) {
 				Py_DECREF(newitem);
 				return -1;
 			}
-			if (cmp == 1)
+			if (cmp == 0)
 				childpos = rightpos;
 		}
 		/* Move the smaller child up. */
@@ -434,6 +457,7 @@ nsmallest(PyObject *self, PyObject *args)
 {
 	PyObject *heap=NULL, *elem, *iterable, *los, *it, *oldelem;
 	Py_ssize_t i, n;
+	int cmp;
 
 	if (!PyArg_ParseTuple(args, "nO:nsmallest", &n, &iterable))
 		return NULL;
@@ -477,7 +501,12 @@ nsmallest(PyObject *self, PyObject *args)
 			else
 				goto sortit;
 		}
-		if (PyObject_RichCompareBool(los, elem, Py_LE)) {
+		cmp = cmp_lt(elem, los);
+		if (cmp == -1) {
+			Py_DECREF(elem);
+			goto fail;
+		}
+		if (cmp == 0) {
 			Py_DECREF(elem);
 			continue;
 		}
diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c
index 243b0b6..29d3ff4 100644
--- a/Modules/_sqlite/cursor.c
+++ b/Modules/_sqlite/cursor.c
@@ -101,10 +101,7 @@ int pysqlite_cursor_init(pysqlite_Cursor* self, PyObject* args, PyObject* kwargs
 
     self->arraysize = 1;
 
-    self->rowcount = PyLong_FromLong(-1L);
-    if (!self->rowcount) {
-        return -1;
-    }
+    self->rowcount = -1L;
 
     Py_INCREF(Py_None);
     self->row_factory = Py_None;
@@ -130,7 +127,6 @@ void pysqlite_cursor_dealloc(pysqlite_Cursor* self)
     Py_XDECREF(self->row_cast_map);
     Py_XDECREF(self->description);
     Py_XDECREF(self->lastrowid);
-    Py_XDECREF(self->rowcount);
     Py_XDECREF(self->row_factory);
     Py_XDECREF(self->next_row);
 
@@ -418,12 +414,12 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
     int statement_type;
     PyObject* descriptor;
     PyObject* second_argument = NULL;
-    long rowcount = 0;
     int allow_8bit_chars;
 
     if (!pysqlite_check_thread(self->connection) || !pysqlite_check_connection(self->connection)) {
         return NULL;
     }
+
     /* Make shooting yourself in the foot with not utf-8 decodable 8-bit-strings harder */
     allow_8bit_chars = ((self->connection->text_factory != (PyObject*)&PyUnicode_Type) &&
         (self->connection->text_factory != (PyObject*)&PyUnicode_Type && pysqlite_OptimizedUnicode));
@@ -498,10 +494,11 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
     if (operation == NULL)
         goto error;
 
-    /* reset description */
+    /* reset description and rowcount */
     Py_DECREF(self->description);
     Py_INCREF(Py_None);
     self->description = Py_None;
+    self->rowcount = -1L;
 
     func_args = PyTuple_New(1);
     if (!func_args) {
@@ -723,7 +720,10 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
             case STATEMENT_DELETE:
             case STATEMENT_INSERT:
             case STATEMENT_REPLACE:
-                rowcount += (long)sqlite3_changes(self->connection->db);
+                if (self->rowcount == -1L) {
+                    self->rowcount = 0L;
+                }
+                self->rowcount += (long)sqlite3_changes(self->connection->db);
         }
 
         Py_DECREF(self->lastrowid);
@@ -757,13 +757,9 @@ error:
     Py_XDECREF(parameters_list);
 
     if (PyErr_Occurred()) {
-        Py_DECREF(self->rowcount);
-        self->rowcount = PyLong_FromLong(-1L);
+        self->rowcount = -1L;
         return NULL;
     } else {
-        Py_DECREF(self->rowcount);
-        self->rowcount = PyLong_FromLong(rowcount);
-
         Py_INCREF(self);
         return (PyObject*)self;
     }
@@ -1053,7 +1049,7 @@ static struct PyMemberDef cursor_members[] =
     {"description", T_OBJECT, offsetof(pysqlite_Cursor, description), READONLY},
     {"arraysize", T_INT, offsetof(pysqlite_Cursor, arraysize), 0},
     {"lastrowid", T_OBJECT, offsetof(pysqlite_Cursor, lastrowid), READONLY},
-    {"rowcount", T_OBJECT, offsetof(pysqlite_Cursor, rowcount), READONLY},
+    {"rowcount", T_LONG, offsetof(pysqlite_Cursor, rowcount), READONLY},
     {"row_factory", T_OBJECT, offsetof(pysqlite_Cursor, row_factory), 0},
     {NULL}
 };
diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h
index d916ca5..54d816d 100644
--- a/Modules/_sqlite/cursor.h
+++ b/Modules/_sqlite/cursor.h
@@ -37,7 +37,7 @@ typedef struct
     PyObject* row_cast_map;
     int arraysize;
     PyObject* lastrowid;
-    PyObject* rowcount;
+    long rowcount;
     PyObject* row_factory;
     pysqlite_Statement* statement;
 
diff --git a/Modules/datetimemodule.c b/Modules/datetimemodule.c
index 0ad5b91..f37290c 100644
--- a/Modules/datetimemodule.c
+++ b/Modules/datetimemodule.c
@@ -1201,17 +1201,17 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple,
 	PyObject *Zreplacement = NULL;	/* py string, replacement for %Z */
 	PyObject *freplacement = NULL;	/* py string, replacement for %f */
 
-	const char *pin;/* pointer to next char in input format */
-        Py_ssize_t flen;/* length of input format */
-	char ch;	/* next char in input format */
+	const char *pin;	/* pointer to next char in input format */
+	Py_ssize_t flen;	/* length of input format */
+	char ch;		/* next char in input format */
 
 	PyObject *newfmt = NULL;	/* py string, the output format */
 	char *pnew;	/* pointer to available byte in output format */
-	int totalnew;	/* number bytes total in output format buffer,
-			   exclusive of trailing \0 */
-	int usednew;	/* number bytes used so far in output format buffer */
+	size_t totalnew;	/* number bytes total in output format buffer,
+				   exclusive of trailing \0 */
+	size_t usednew;	/* number bytes used so far in output format buffer */
 
-	const char *ptoappend;/* pointer to string to append to output buffer */
+	const char *ptoappend;	/* ptr to string to append to output buffer */
 	Py_ssize_t ntoappend;	/* # of bytes to append to output buffer */
 
 	assert(object && format && timetuple);
@@ -1335,7 +1335,7 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple,
  		assert(ptoappend != NULL);
  		assert(ntoappend > 0);
  		while (usednew + ntoappend > totalnew) {
- 			int bigger = totalnew << 1;
+ 			size_t bigger = totalnew << 1;
  			if ((bigger >> 1) != totalnew) { /* overflow */
  				PyErr_NoMemory();
  				goto Done;
@@ -2444,8 +2444,8 @@ date_strftime(PyDateTime_Date *self, PyObject *args, PyObject *kw)
 	 * timetuple() method appropriate to self's class.
 	 */
 	PyObject *result;
-	PyObject *format;
 	PyObject *tuple;
+	PyObject *format;
 	static char *keywords[] = {"format", NULL};
 
 	if (! PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords,
@@ -3211,8 +3211,8 @@ static PyObject *
 time_strftime(PyDateTime_Time *self, PyObject *args, PyObject *kw)
 {
 	PyObject *result;
-	PyObject *format;
 	PyObject *tuple;
+	PyObject *format;
 	static char *keywords[] = {"format", NULL};
 
 	if (! PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords,
@@ -3232,7 +3232,8 @@ time_strftime(PyDateTime_Time *self, PyObject *args, PyObject *kw)
 	if (tuple == NULL)
 		return NULL;
 	assert(PyTuple_Size(tuple) == 9);
-	result = wrap_strftime((PyObject *)self, format, tuple, Py_None);
+	result = wrap_strftime((PyObject *)self, format, tuple,
+			       Py_None);
 	Py_DECREF(tuple);
 	return result;
 }
diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c
index 5c5def2..16c829a 100644
--- a/Modules/mathmodule.c
+++ b/Modules/mathmodule.c
@@ -373,14 +373,20 @@ FUNC1(tanh, tanh, 0,
    value semantics across iterations (i.e. handling -Inf + Inf).
 
    Note 2:  No provision is made for intermediate overflow handling;
-   therefore, sum([1e+308, 1e-308, 1e+308]) returns result 1e+308 while
+   therefore, sum([1e+308, 1e-308, 1e+308]) returns 1e+308 while
    sum([1e+308, 1e+308, 1e-308]) raises an OverflowError due to the
    overflow of the first partial sum.
 
-   Note 3: Aggressively optimizing compilers can potentially eliminate the
-   residual values needed for accurate summation. For instance, the statements
-   "hi = x + y; lo = y - (hi - x);" could be mis-transformed to
-   "hi = x + y; lo = 0.0;" which defeats the computation of residuals.
+   Note 3: The itermediate values lo, yr, and hi are declared volatile so
+   aggressive compilers won't algebraicly reduce lo to always be exactly 0.0.
+   Also, the volatile declaration forces the values to be stored in memory as
+   regular doubles instead of extended long precision (80-bit) values.  This
+   prevents double rounding because any addition or substraction of two doubles
+   can be resolved exactly into double-sized hi and lo values.  As long as the 
+   hi value gets forced into a double before yr and lo are computed, the extra
+   bits in downstream extended precision operations (x87 for example) will be
+   exactly zero and therefore can be losslessly stored back into a double,
+   thereby preventing double rounding.
 
    Note 4: A similar implementation is in Modules/cmathmodule.c.
    Be sure to update both when making changes.
@@ -457,7 +463,8 @@ math_sum(PyObject *self, PyObject *seq)
 {
 	PyObject *item, *iter, *sum = NULL;
 	Py_ssize_t i, j, n = 0, m = NUM_PARTIALS;
-	double x, y, hi, lo=0.0, ps[NUM_PARTIALS], *p = ps;
+	double x, y, t, ps[NUM_PARTIALS], *p = ps;
+	volatile double hi, yr, lo;
 
 	iter = PyObject_GetIter(seq);
 	if (iter == NULL)
@@ -483,10 +490,12 @@ math_sum(PyObject *self, PyObject *seq)
 
 		for (i = j = 0; j < n; j++) {       /* for y in partials */
 			y = p[j];
+			if (fabs(x) < fabs(y)) {
+					t = x; x = y; y = t;
+			}
 			hi = x + y;
-			lo = fabs(x) < fabs(y)
-			   ? x - (hi - y)
-			   : y - (hi - x);
+			yr = hi - x;
+			lo = y - yr;
 			if (lo != 0.0)
 				p[i++] = lo;
 			x = hi;
@@ -506,38 +515,41 @@ math_sum(PyObject *self, PyObject *seq)
 		}
 	}
 
+	hi = 0.0;
 	if (n > 0) {
 		hi = p[--n];
 		if (Py_IS_FINITE(hi)) {
 			/* sum_exact(ps, hi) from the top, stop when the sum becomes inexact. */
 			while (n > 0) {
-				x = p[--n];
-				y = hi;
+				x = hi;
+				y = p[--n];
+				assert(fabs(y) < fabs(x));
 				hi = x + y;
-				assert(fabs(x) < fabs(y));
-				lo = x - (hi - y);
+				yr = hi - x;
+				lo = y - yr;
 				if (lo != 0.0)
 					break;
 			}
-			/* Little dance to allow half-even rounding across multiple partials.
-                           Needed so that sum([1e-16, 1, 1e16]) will round-up to two instead
-                           of down to zero (the 1e16 makes the 1 slightly closer to two). */
+			/* Make half-even rounding work across multiple partials.  Needed 
+			   so that sum([1e-16, 1, 1e16]) will round-up the last digit to 
+			   two instead of down to zero (the 1e-16 makes the 1 slightly 
+			   closer to two).  With a potential 1 ULP rounding error fixed-up,
+			   math.sum() can guarantee commutativity. */
 			if (n > 0 && ((lo < 0.0 && p[n-1] < 0.0) ||
 			              (lo > 0.0 && p[n-1] > 0.0))) {
 				y = lo * 2.0;
 				x = hi + y;
-				if (y == (x - hi))
+				yr = x - hi;
+				if (y == yr)
 					hi = x;
 			}
 		}
-		else {  /* raise corresponding error */
+		else {  /* raise exception corresponding to a special value */
 			errno = Py_IS_NAN(hi) ? EDOM : ERANGE;
 			if (is_error(hi))
 				goto _sum_error;
 		}
 	}
-	else  /* default */
-		hi = 0.0;
 	sum = PyFloat_FromDouble(hi);
 
 _sum_error: