Mondo changes to the iterator stuff, without changing how Python code

sees it (test_iter.py is unchanged).

- Added a tp_iternext slot, which calls the iterator's next() method;
  this is much faster for built-in iterators over built-in types
  such as lists and dicts, speeding up pybench's ForLoop with about
  25% compared to Python 2.1.  (Now there's a good argument for
  iterators. ;-)

- Renamed the built-in sequence iterator SeqIter, affecting the C API
  functions for it.  (This frees up the PyIter prefix for generic
  iterator operations.)

- Added PyIter_Check(obj), which checks that obj's type has a
  tp_iternext slot and that the proper feature flag is set.

- Added PyIter_Next(obj) which calls the tp_iternext slot.  It has a
  somewhat complex return condition due to the need for speed: when it
  returns NULL, it may not have set an exception condition, meaning
  the iterator is exhausted; when the exception StopIteration is set
  (or a derived exception class), it means the same thing; any other
  exception means some other error occurred.

5 files changed, 148 insertions, 28 deletions

diff --git a/Objects/abstract.c b/Objects/abstract.c
index 8a6df76..f656747 100644
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -1748,10 +1748,32 @@ PyObject_GetIter(PyObject *o)
 		f = t->tp_iter;
 	if (f == NULL) {
 		if (PySequence_Check(o))
-			return PyIter_New(o);
+			return PySeqIter_New(o);
 		PyErr_SetString(PyExc_TypeError, "iter() of non-sequence");
 		return NULL;
 	}
-	else
-		return (*f)(o);
+	else {
+		PyObject *res = (*f)(o);
+		if (res != NULL && !PyIter_Check(res)) {
+			PyErr_Format(PyExc_TypeError,
+				     "iter() returned non-iterator "
+				     "of type '%.100s'",
+				     res->ob_type->tp_name);
+			Py_DECREF(res);
+			res = NULL;
+		}
+		return res;
+	}
+}
+
+PyObject *
+PyIter_Next(PyObject *iter)
+{
+	if (!PyIter_Check(iter)) {
+		PyErr_Format(PyExc_TypeError,
+			     "'%.100s' object is not an iterator",
+			     iter->ob_type->tp_name);
+		return NULL;
+	}
+	return (*iter->ob_type->tp_iternext)(iter);
 }
diff --git a/Objects/classobject.c b/Objects/classobject.c
index 742e472..80b7ae5 100644
--- a/Objects/classobject.c
+++ b/Objects/classobject.c
@@ -848,7 +848,8 @@ instance_traverse(PyInstanceObject *o, visitproc visit, void *arg)
 	return 0;
 }
 
-static PyObject *getitemstr, *setitemstr, *delitemstr, *lenstr, *iterstr;
+static PyObject *getitemstr, *setitemstr, *delitemstr, *lenstr;
+static PyObject *iterstr, *nextstr;
 
 static int
 instance_length(PyInstanceObject *inst)
@@ -1726,6 +1727,14 @@ instance_getiter(PyInstanceObject *self)
 	if ((func = instance_getattr(self, iterstr)) != NULL) {
 		PyObject *res = PyEval_CallObject(func, (PyObject *)NULL);
 		Py_DECREF(func);
+		if (res != NULL && !PyIter_Check(res)) {
+			PyErr_Format(PyExc_TypeError,
+				     "__iter__ returned non-iterator "
+				     "of type '%.100s'",
+				     res->ob_type->tp_name);
+			Py_DECREF(res);
+			res = NULL;
+		}
 		return res;
 	}
 	PyErr_Clear();
@@ -1734,7 +1743,33 @@ instance_getiter(PyInstanceObject *self)
 		return NULL;
 	}
 	Py_DECREF(func);
-	return PyIter_New((PyObject *)self);
+	return PySeqIter_New((PyObject *)self);
+}
+
+
+/* Call the iterator's next */
+static PyObject *
+instance_iternext(PyInstanceObject *self)
+{
+	PyObject *func;
+
+	if (nextstr == NULL)
+		nextstr = PyString_InternFromString("next");
+
+	if ((func = instance_getattr(self, nextstr)) != NULL) {
+		PyObject *res = PyEval_CallObject(func, (PyObject *)NULL);
+		Py_DECREF(func);
+		if (res != NULL) {
+			return res;
+		}
+		if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
+			PyErr_Clear();
+			return NULL;
+		}
+		return NULL;
+	}
+	PyErr_SetString(PyExc_TypeError, "instance has no next() method");
+	return NULL;
 }
 
 
@@ -1803,6 +1838,7 @@ PyTypeObject PyInstance_Type = {
 	instance_richcompare,			/* tp_richcompare */
  	offsetof(PyInstanceObject, in_weakreflist), /* tp_weaklistoffset */
 	(getiterfunc)instance_getiter,		/* tp_iter */
+	(iternextfunc)instance_iternext,	/* tp_iternext */
 };
 
 
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 17b6a04..9a24109 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -1354,6 +1354,7 @@ PyTypeObject PyDict_Type = {
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	(getiterfunc)dictiter_new,		/* tp_iter */
+	0,					/* tp_iternext */
 };
 
 /* For backward compatibility with old dictionary interface */
@@ -1433,6 +1434,7 @@ static PyObject *
 dictiter_next(dictiterobject *di, PyObject *args)
 {
 	PyObject *key;
+
 	if (di->di_size != di->di_dict->ma_size) {
 		PyErr_SetString(PyExc_RuntimeError,
 				"dictionary changed size during iteration");
@@ -1460,9 +1462,25 @@ static PyMethodDef dictiter_methods[] = {
 };
 
 static PyObject *
-dictiter_getattr(dictiterobject *it, char *name)
+dictiter_getattr(dictiterobject *di, char *name)
+{
+	return Py_FindMethod(dictiter_methods, (PyObject *)di, name);
+}
+
+static PyObject *dictiter_iternext(dictiterobject *di)
 {
-	return Py_FindMethod(dictiter_methods, (PyObject *)it, name);
+	PyObject *key;
+
+	if (di->di_size != di->di_dict->ma_size) {
+		PyErr_SetString(PyExc_RuntimeError,
+				"dictionary changed size during iteration");
+		return NULL;
+	}
+	if (PyDict_Next((PyObject *)(di->di_dict), &di->di_pos, &key, NULL)) {
+		Py_INCREF(key);
+		return key;
+	}
+	return NULL;
 }
 
 PyTypeObject PyDictIter_Type = {
@@ -1494,4 +1512,5 @@ PyTypeObject PyDictIter_Type = {
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	(getiterfunc)dictiter_getiter,		/* tp_iter */
+	(iternextfunc)dictiter_iternext,	/* tp_iternext */
 };
diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index b6cfb92..7083384 100644
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -1340,6 +1340,7 @@ PyTypeObject PyFile_Type = {
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	(getiterfunc)file_getiter,		/* tp_iter */
+	0,					/* tp_iternext */
 };
 
 /* Interface for the 'soft space' between print items. */
diff --git a/Objects/iterobject.c b/Objects/iterobject.c
index 616417a..8601980 100644
--- a/Objects/iterobject.c
+++ b/Objects/iterobject.c
@@ -6,13 +6,13 @@ typedef struct {
 	PyObject_HEAD
 	long      it_index;
 	PyObject *it_seq;
-} iterobject;
+} seqiterobject;
 
 PyObject *
-PyIter_New(PyObject *seq)
+PySeqIter_New(PyObject *seq)
 {
-	iterobject *it;
-	it = PyObject_NEW(iterobject, &PyIter_Type);
+	seqiterobject *it;
+	it = PyObject_NEW(seqiterobject, &PySeqIter_Type);
 	if (it == NULL)
 		return NULL;
 	it->it_index = 0;
@@ -21,21 +21,44 @@ PyIter_New(PyObject *seq)
 	return (PyObject *)it;
 }
 static void
-iter_dealloc(iterobject *it)
+iter_dealloc(seqiterobject *it)
 {
 	Py_DECREF(it->it_seq);
 	PyObject_DEL(it);
 }
 
 static PyObject *
-iter_next(iterobject *it, PyObject *args)
+iter_next(seqiterobject *it, PyObject *args)
 {
 	PyObject *seq = it->it_seq;
+	PyObject *result = PySequence_GetItem(seq, it->it_index++);
+
+	if (result == NULL && PyErr_ExceptionMatches(PyExc_IndexError))
+		PyErr_SetObject(PyExc_StopIteration, Py_None);
+	return result;
+}
+
+static PyObject *
+iter_getiter(PyObject *it)
+{
+	Py_INCREF(it);
+	return it;
+}
+
+/* Return (value, 0) if OK; (NULL, 0) at end; (NULL, -1) if exception */
+static PyObject *
+iter_iternext(PyObject *iterator)
+{
+	seqiterobject *it;
+	PyObject *seq;
+
+	assert(PySeqIter_Check(iterator));
+	it = (seqiterobject *)iterator;
+	seq = it->it_seq;
 
 	if (PyList_Check(seq)) {
 		PyObject *item;
 		if (it->it_index >= PyList_GET_SIZE(seq)) {
-			PyErr_SetObject(PyExc_StopIteration, Py_None);
 			return NULL;
 		}
 		item = PyList_GET_ITEM(seq, it->it_index);
@@ -45,20 +68,20 @@ iter_next(iterobject *it, PyObject *args)
 	}
 	else {
 		PyObject *result = PySequence_GetItem(seq, it->it_index++);
-		if (result == NULL &&
-		    PyErr_ExceptionMatches(PyExc_IndexError))
-			PyErr_SetObject(PyExc_StopIteration, Py_None);
-		return result;
+		if (result != NULL) {
+			return result;
+		}
+		if (PyErr_ExceptionMatches(PyExc_IndexError) ||
+			PyErr_ExceptionMatches(PyExc_StopIteration)) {
+			PyErr_Clear();
+			return NULL;
+		}
+		else {
+			return NULL;
+		}
 	}
 }
 
-static PyObject *
-iter_getiter(PyObject *it)
-{
-	Py_INCREF(it);
-	return it;
-}
-
 static PyMethodDef iter_methods[] = {
 	{"next",	(PyCFunction)iter_next,	METH_VARARGS,
 	 "it.next() -- get the next value, or raise StopIteration"},
@@ -66,16 +89,16 @@ static PyMethodDef iter_methods[] = {
 };
 
 static PyObject *
-iter_getattr(iterobject *it, char *name)
+iter_getattr(seqiterobject *it, char *name)
 {
 	return Py_FindMethod(iter_methods, (PyObject *)it, name);
 }
 
-PyTypeObject PyIter_Type = {
+PyTypeObject PySeqIter_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
 	"iterator",				/* tp_name */
-	sizeof(iterobject),			/* tp_basicsize */
+	sizeof(seqiterobject),			/* tp_basicsize */
 	0,					/* tp_itemsize */
 	/* methods */
 	(destructor)iter_dealloc, 		/* tp_dealloc */
@@ -100,6 +123,7 @@ PyTypeObject PyIter_Type = {
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	(getiterfunc)iter_getiter,		/* tp_iter */
+	(iternextfunc)iter_iternext,		/* tp_iternext */
 };
 
 /* -------------------------------------- */
@@ -130,6 +154,7 @@ calliter_dealloc(calliterobject *it)
 	Py_DECREF(it->it_sentinel);
 	PyObject_DEL(it);
 }
+
 static PyObject *
 calliter_next(calliterobject *it, PyObject *args)
 {
@@ -156,6 +181,22 @@ calliter_getattr(calliterobject *it, char *name)
 	return Py_FindMethod(calliter_methods, (PyObject *)it, name);
 }
 
+static PyObject *
+calliter_iternext(calliterobject *it)
+{
+	PyObject *result = PyObject_CallObject(it->it_callable, NULL);
+	if (result != NULL) {
+		if (PyObject_RichCompareBool(result, it->it_sentinel, Py_EQ)) {
+			Py_DECREF(result);
+			result = NULL;
+		}
+	}
+	else if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
+		PyErr_Clear();
+	}
+	return result;
+}
+
 PyTypeObject PyCallIter_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -185,4 +226,5 @@ PyTypeObject PyCallIter_Type = {
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	(getiterfunc)iter_getiter,		/* tp_iter */
+	(iternextfunc)calliter_iternext,	/* tp_iternext */
 };