Merged revisions 78872 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/branches/py3k

................
  r78872 | victor.stinner | 2010-03-12 15:45:56 +0100 (ven., 12 mars 2010) | 12 lines

  Merged revisions 78826 via svnmerge from
  svn+ssh://pythondev@svn.python.org/python/trunk

  ........
    r78826 | victor.stinner | 2010-03-10 23:30:19 +0100 (mer., 10 mars 2010) | 5 lines

    Issue #3137: Don't ignore errors at startup, especially a keyboard interrupt
    (SIGINT). If an error occurs while importing the site module, the error is
    printed and Python exits. Initialize the GIL before importing the site
    module.
  ........
................

1 files changed, 12 insertions, 5 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 658dc316fb..af99deff38 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1187,21 +1187,28 @@ indenterror(struct tok_state *tok)
 }
 
 #ifdef PGEN
-#define verify_identifier(s,e) 1
+#define verify_identifier(tok) 1
 #else
 /* Verify that the identifier follows PEP 3131. */
 static int
-verify_identifier(char *start, char *end)
+verify_identifier(struct tok_state *tok)
 {
 	PyObject *s;
 	int result;
-	s = PyUnicode_DecodeUTF8(start, end-start, NULL);
+	s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
 	if (s == NULL) {
-		PyErr_Clear();
+		if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
+			PyErr_Clear();
+			tok->done = E_IDENTIFIER;
+		} else {
+			tok->done = E_ERROR;
+		}
 		return 0;
 	}
 	result = PyUnicode_IsIdentifier(s);
 	Py_DECREF(s);
+	if (result == 0)
+		tok->done = E_IDENTIFIER;
 	return result;
 }
 #endif
@@ -1350,7 +1357,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 		}
 		tok_backup(tok, c);
 		if (nonascii &&
-		    !verify_identifier(tok->start, tok->cur)) {
+		    !verify_identifier(tok)) {
 			tok->done = E_IDENTIFIER;
 			return ERRORTOKEN;
 		}