gh-120317: Lock around global state in the tokenize module

Lib/test/test_free_threading/test_tokenize.py

@@ -45,8 +45,9 @@ def next_token(it):
             tokenize.TokenInfo(type=0, string='', start=(2, -1), end=(2, -1), line='  pass'),
         ]
 
-        for token in tokens:
-            self.assertIn(token, expected_tokens)
+        tokens.sort()
+        expected_tokens.sort()
+        self.assertListEqual(tokens, expected_tokens)
 
 
 if __name__ == "__main__":

Python/Python-tokenize.c

@@ -179,6 +179,7 @@ _tokenizer_error(struct tok_state *tok)
 static PyObject *
 _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t size)
 {
+    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
     PyObject *line;
     if (it->tok->lineno != it->last_lineno) {
         // Line has changed since last token, so we fetch the new line and cache it
@@ -187,7 +188,8 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si
         line = PyUnicode_DecodeUTF8(line_start, size, "replace");
         it->last_line = line;
         it->byte_col_offset_diff = 0;
-    } else {
+    }
+    else {
         // Line hasn't changed so we reuse the cached one.
         line = it->last_line;
     }
@@ -199,7 +201,8 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s
                  PyObject *line, Py_ssize_t lineno, Py_ssize_t end_lineno,
                  Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
 {
-    Py_ssize_t byte_offset;
+    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
+    Py_ssize_t byte_offset = -1;
     if (token.start != NULL && token.start >= line_start) {
         byte_offset = token.start - line_start;
         *col_offset = byte_offset - it->byte_col_offset_diff;
@@ -214,7 +217,8 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s
             Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
             *end_col_offset = *col_offset + token_col_offset;
             it->byte_col_offset_diff += token.end - token.start - token_col_offset;
-        } else {
+        }
+        else {
             *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
             it->byte_col_offset_diff += end_byte_offset - *end_col_offset;
         }
@@ -243,11 +247,7 @@ tokenizeriter_next(tokenizeriterobject *it)
     }
     if (it->done || type == ERRORTOKEN) {
         PyErr_SetString(PyExc_StopIteration, "EOF");
-
-        Py_BEGIN_CRITICAL_SECTION(it);
-        it->done = 1;
-        Py_END_CRITICAL_SECTION();
-
+        _Py_atomic_store_int(&it->done, 1);
         goto exit;
     }
     PyObject *str = NULL;
@@ -275,6 +275,7 @@ tokenizeriter_next(tokenizeriterobject *it)
         if (size >= 1 && it->tok->implicit_newline) {
             size -= 1;
         }
+
         Py_BEGIN_CRITICAL_SECTION(it);
         line = _get_current_line(it, line_start, size);
         Py_END_CRITICAL_SECTION();
@@ -288,6 +289,7 @@ tokenizeriter_next(tokenizeriterobject *it)
     Py_ssize_t end_lineno = it->tok->lineno;
     Py_ssize_t col_offset = -1;
     Py_ssize_t end_col_offset = -1;
+
     Py_BEGIN_CRITICAL_SECTION(it);
     _get_col_offsets(it, token, line_start, line, lineno, end_lineno, &col_offset, &end_col_offset);
     Py_END_CRITICAL_SECTION();
@@ -330,9 +332,7 @@ tokenizeriter_next(tokenizeriterobject *it)
 exit:
     _PyToken_Free(&token);
     if (type == ENDMARKER) {
-        Py_BEGIN_CRITICAL_SECTION(it);
-        it->done = 1;
-        Py_END_CRITICAL_SECTION();
+        _Py_atomic_store_int(&it->done, 1);
     }
     return result;
 }