Patch #1357836:

Prevent an invalid memory read from test_coding in case the done flag is set.
In that case, the loop isn't entered.  I wonder if rather than setting
the done flag in the cases before the loop, if they should just exit early.

This code looks like it should be refactored.

Backport candidate (also the early break above if decoding_fgets fails)

1 files changed, 11 insertions, 9 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 4a281050be..92c2087262 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -893,15 +893,17 @@ tok_nextc(register struct tok_state *tok)
 				tok->inp = strchr(tok->inp, '\0');
 				done = tok->inp[-1] == '\n';
 			}
-			tok->cur = tok->buf + cur;
-			tok->line_start = tok->cur;
-			/* replace "\r\n" with "\n" */
-			/* For Mac we leave the \r, giving a syntax error */
-			pt = tok->inp - 2;
-			if (pt >= tok->buf && *pt == '\r') {
-				*pt++ = '\n';
-				*pt = '\0';
-				tok->inp = pt;
+			if (tok->buf != NULL) {
+				tok->cur = tok->buf + cur;
+				tok->line_start = tok->cur;
+				/* replace "\r\n" with "\n" */
+				/* For Mac leave the \r, giving syntax error */
+				pt = tok->inp - 2;
+				if (pt >= tok->buf && *pt == '\r') {
+					*pt++ = '\n';
+					*pt = '\0';
+					tok->inp = pt;
+				}
 			}
 		}
 		if (tok->done != E_OK) {