From cbf0afd8a1474d68310331af9218606959d4cc22 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 6 Oct 2022 16:07:17 -0700 Subject: gh-97973: Return all necessary information from the tokenizer (GH-97984) Right now, the tokenizer only returns type and two pointers to the start and end of the token. This PR modifies the tokenizer to return the type and set all of the necessary information, so that the parser does not have to this. --- Python/Python-tokenize.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'Python/Python-tokenize.c') diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index c5124a6942..8daa987725 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -60,9 +60,8 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source) static PyObject * tokenizeriter_next(tokenizeriterobject *it) { - const char *start; - const char *end; - int type = _PyTokenizer_Get(it->tok, &start, &end); + struct token token; + int type = _PyTokenizer_Get(it->tok, &token); if (type == ERRORTOKEN && PyErr_Occurred()) { return NULL; } @@ -71,11 +70,11 @@ tokenizeriter_next(tokenizeriterobject *it) return NULL; } PyObject *str = NULL; - if (start == NULL || end == NULL) { + if (token.start == NULL || token.end == NULL) { str = PyUnicode_FromString(""); } else { - str = PyUnicode_FromStringAndSize(start, end - start); + str = PyUnicode_FromStringAndSize(token.start, token.end - token.start); } if (str == NULL) { return NULL; @@ -92,11 +91,11 @@ tokenizeriter_next(tokenizeriterobject *it) int end_lineno = it->tok->lineno; int col_offset = -1; int end_col_offset = -1; - if (start != NULL && start >= line_start) { - col_offset = (int)(start - line_start); + if (token.start != NULL && token.start >= line_start) { + col_offset = (int)(token.start - line_start); } - if (end != NULL && end >= it->tok->line_start) { - end_col_offset = (int)(end - it->tok->line_start); + if (token.end != NULL && token.end >= it->tok->line_start) { + end_col_offset = (int)(token.end - it->tok->line_start); } return Py_BuildValue("(NiiiiiN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line); -- cgit v1.2.1