gh-97973: Return all necessary information from the tokenizer (GH-97984)

Right now, the tokenizer only returns type and two pointers to the start and end of the token.
This PR modifies the tokenizer to return the type and set all of the necessary information,
so that the parser does not have to this.

1 files changed, 8 insertions, 9 deletions

diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index c5124a6942..8daa987725 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -60,9 +60,8 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source)
 static PyObject *
 tokenizeriter_next(tokenizeriterobject *it)
 {
-    const char *start;
-    const char *end;
-    int type = _PyTokenizer_Get(it->tok, &start, &end);
+    struct token token;
+    int type = _PyTokenizer_Get(it->tok, &token);
     if (type == ERRORTOKEN && PyErr_Occurred()) {
         return NULL;
     }
@@ -71,11 +70,11 @@ tokenizeriter_next(tokenizeriterobject *it)
         return NULL;
     }
     PyObject *str = NULL;
-    if (start == NULL || end == NULL) {
+    if (token.start == NULL || token.end == NULL) {
         str = PyUnicode_FromString("");
     }
     else {
-        str = PyUnicode_FromStringAndSize(start, end - start);
+        str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
     }
     if (str == NULL) {
         return NULL;
@@ -92,11 +91,11 @@ tokenizeriter_next(tokenizeriterobject *it)
     int end_lineno = it->tok->lineno;
     int col_offset = -1;
     int end_col_offset = -1;
-    if (start != NULL && start >= line_start) {
-        col_offset = (int)(start - line_start);
+    if (token.start != NULL && token.start >= line_start) {
+        col_offset = (int)(token.start - line_start);
     }
-    if (end != NULL && end >= it->tok->line_start) {
-        end_col_offset = (int)(end - it->tok->line_start);
+    if (token.end != NULL && token.end >= it->tok->line_start) {
+        end_col_offset = (int)(token.end - it->tok->line_start);
     }
 
     return Py_BuildValue("(NiiiiiN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);