Patch #1357836:

Prevent an invalid memory read from test_coding in case the done flag is set.
In that case, the loop isn't entered.  I wonder if rather than setting
the done flag in the cases before the loop, if they should just exit early.

This code looks like it should be refactored.

Backport candidate (also the early break above if decoding_fgets fails)

1 files changed, 11 insertions, 9 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 4a28105..92c2087 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -893,15 +893,17 @@ tok_nextc(register struct tok_state *tok)
 				tok->inp = strchr(tok->inp, '\0');
 				done = tok->inp[-1] == '\n';
 			}
-			tok->cur = tok->buf + cur;
-			tok->line_start = tok->cur;
-			/* replace "\r\n" with "\n" */
-			/* For Mac we leave the \r, giving a syntax error */
-			pt = tok->inp - 2;
-			if (pt >= tok->buf && *pt == '\r') {
-				*pt++ = '\n';
-				*pt = '\0';
-				tok->inp = pt;
+			if (tok->buf != NULL) {
+				tok->cur = tok->buf + cur;
+				tok->line_start = tok->cur;
+				/* replace "\r\n" with "\n" */
+				/* For Mac leave the \r, giving syntax error */
+				pt = tok->inp - 2;
+				if (pt >= tok->buf && *pt == '\r') {
+					*pt++ = '\n';
+					*pt = '\0';
+					tok->inp = pt;
+				}
 			}
 		}
 		if (tok->done != E_OK) {