Mass checkin of universal newline support.

Highlights: import and friends will understand any of \r, \n and \r\n
as end of line. Python file input will do the same if you use mode 'U'.
Everything can be disabled by configuring with --without-universal-newlines.

See PEP278 for details.

2 files changed, 16 insertions, 4 deletions

diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
index 2a284bc..d25cbd4 100644
--- a/Parser/pgenmain.c
+++ b/Parser/pgenmain.c
@@ -13,6 +13,7 @@
    - check for duplicate definitions of names (instead of fatal err)
 */
 
+#include "Python.h"
 #include "pgenheaders.h"
 #include "grammar.h"
 #include "node.h"
@@ -182,6 +183,16 @@ PyOS_Readline(char *prompt)
 	return PyMem_REALLOC(p, n+1);
 }
 
+#ifdef WITH_UNIVERSAL_NEWLINES
+/* No-nonsense fgets */
+char *
+Py_UniversalNewlineFgets(char *buf, int n, FILE *stream, PyObject *fobj)
+{
+	return fgets(buf, n, stream);
+}
+#endif
+
+
 #include <stdarg.h>
 
 void
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 324d9b6..b4e0fbf 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1,6 +1,7 @@
 
 /* Tokenizer implementation */
 
+#include "Python.h"
 #include "pgenheaders.h"
 
 #include <ctype.h>
@@ -245,8 +246,8 @@ tok_nextc(register struct tok_state *tok)
 					}
 					tok->end = tok->buf + BUFSIZ;
 				}
-				if (fgets(tok->buf, (int)(tok->end - tok->buf),
-					  tok->fp) == NULL) {
+				if (Py_UniversalNewlineFgets(tok->buf, (int)(tok->end - tok->buf),
+					  tok->fp, NULL) == NULL) {
 					tok->done = E_EOF;
 					done = 1;
 				}
@@ -284,9 +285,9 @@ tok_nextc(register struct tok_state *tok)
 				tok->end = tok->buf + newsize;
 				tok->start = curstart < 0 ? NULL :
 					     tok->buf + curstart;
-				if (fgets(tok->inp,
+				if (Py_UniversalNewlineFgets(tok->inp,
 					       (int)(tok->end - tok->inp),
-					       tok->fp) == NULL) {
+					       tok->fp, NULL) == NULL) {
 					/* Last line does not end in \n,
 					   fake one */
 					strcpy(tok->inp, "\n");