Use Py_BEGIN_CRITICAL_SECTION

Author: lysnikolaou

Python/Python-tokenize.c

@@ -1,10 +1,10 @@
 #include "Python.h"
 #include "errcode.h"
-#include "internal/pycore_lock.h" // PyMutex
+#include "internal/pycore_critical_section.h"   // Py_BEGIN_CRITICAL_SECTION
 #include "../Parser/lexer/state.h"
 #include "../Parser/lexer/lexer.h"
 #include "../Parser/tokenizer/tokenizer.h"
-#include "../Parser/pegen.h"      // _PyPegen_byte_offset_to_character_offset()
+#include "../Parser/pegen.h"                    // _PyPegen_byte_offset_to_character_offset()
 
 static struct PyModuleDef _tokenizemodule;
 
@@ -38,10 +38,6 @@ typedef struct
     PyObject *last_line;
     Py_ssize_t last_lineno;
     Py_ssize_t byte_col_offset_diff;
-
-#ifdef Py_GIL_DISABLED
-    PyMutex mutex;
-#endif
 } tokenizeriterobject;
 
 /*[clinic input]
@@ -79,10 +75,6 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
     }
     self->done = 0;
 
-#ifdef Py_GIL_DISABLED
-    self->mutex = (PyMutex) {_Py_UNLOCKED};
-#endif
-
     self->last_line = NULL;
     self->byte_col_offset_diff = 0;
     self->last_lineno = 0;
@@ -191,13 +183,10 @@ tokenizeriter_next(tokenizeriterobject *it)
     struct token token;
     _PyToken_Init(&token);
 
-#ifdef Py_GIL_DISABLED
-    PyMutex_Lock(&it->mutex);
-#endif
-    int type = _PyTokenizer_Get(it->tok, &token);
-#ifdef Py_GIL_DISABLED
-    PyMutex_Unlock(&it->mutex);
-#endif
+    int type;
+    Py_BEGIN_CRITICAL_SECTION(it);
+    type = _PyTokenizer_Get(it->tok, &token);
+    Py_END_CRITICAL_SECTION();
 
     if (type == ERRORTOKEN) {
         if(!PyErr_Occurred()) {