From 55bb12f7a5190cb79fb490b38610c61e668a71ce Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Sat, 29 Mar 2025 01:02:32 +0100
Subject: [PATCH] PHP 8.1 | Tokenizer/PHP: fix nullable type tokenization for
 readonly properties

The `?` in a type for a `readonly` property declared without visibility, would be tokenized as `T_INLINE_THEN`, not `T_NULLABLE`.

Fixed now.

Includes perfunctory test.
Realistically, a lot more tests are needed for the `T_NULLABLE` vs `T_INLINE_THEN` tokenization, but at least, this is a start and covers the current change.
---
 src/Tokenizers/PHP.php                        |  7 ++
 .../PHP/NullableVsInlineThenTest.inc          | 13 +++
 .../PHP/NullableVsInlineThenTest.php          | 96 +++++++++++++++++++
 3 files changed, 116 insertions(+)
 create mode 100644 tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc
 create mode 100644 tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php

diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
index c6b8e29065..0334ad73bd 100644
--- a/src/Tokenizers/PHP.php
+++ b/src/Tokenizers/PHP.php
@@ -1494,6 +1494,12 @@ protected function tokenize($string)
                     ];
                     $newStackPtr++;
 
+                    // Also modify the original token stack so that
+                    // future checks (like looking for T_NULLABLE) can
+                    // detect the T_READONLY token more easily.
+                    $tokens[$stackPtr][0] = T_READONLY;
+                    $token[0] = T_READONLY;
+
                     if (PHP_CODESNIFFER_VERBOSITY > 1 && $type !== T_READONLY) {
                         echo "\t\t* token $stackPtr changed from $type to T_READONLY".PHP_EOL;
                     }
@@ -2142,6 +2148,7 @@ protected function tokenize($string)
                         || $tokenType === T_FN
                         || isset(Tokens::$methodPrefixes[$tokenType]) === true
                         || $tokenType === T_VAR
+                        || $tokenType === T_READONLY
                     ) {
                         if (PHP_CODESNIFFER_VERBOSITY > 1) {
                             echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
diff --git a/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc
new file mode 100644
index 0000000000..4c018af631
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc
@@ -0,0 +1,13 @@
+<?php
+
+class Nullable
+{
+    /* testNullableReadonlyOnly */
+    readonly ?int $prop;
+}
+
+class InlineThen
+{
+    /* testInlineThenInPropertyDefaultValue */
+    public int $prop = self::SOMECONT ? PHP_CONST ? OTHER_CONST;
+}
diff --git a/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php
new file mode 100644
index 0000000000..fc7ed4b105
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php
@@ -0,0 +1,96 @@
+<?php
+/**
+ * Tests the retokenization of ? to T_NULLABLE or T_INLINE_THEN.
+ *
+ * @copyright 2025 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+
+/**
+ * Tests the retokenization of ? to T_NULLABLE or T_INLINE_THEN.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+ */
+final class NullableVsInlineThenTest extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that the ? at the start of type declarations is retokenized to T_NULLABLE.
+     *
+     * @param string $testMarker The comment which prefaces the target token in the test file.
+     *
+     * @dataProvider dataNullable
+     *
+     * @return void
+     */
+    public function testNullable($testMarker)
+    {
+        $tokens     = $this->phpcsFile->getTokens();
+        $target     = $this->getTargetToken($testMarker, [T_NULLABLE, T_INLINE_THEN]);
+        $tokenArray = $tokens[$target];
+
+        $this->assertSame(T_NULLABLE, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_NULLABLE (code)');
+        $this->assertSame('T_NULLABLE', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_NULLABLE (type)');
+
+    }//end testNullable()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testNullable()
+     *
+     * @return array<string, array<string>>
+     */
+    public static function dataNullable()
+    {
+        return [
+            'property declaration, readonly, no visibility' => ['/* testNullableReadonlyOnly */'],
+        ];
+
+    }//end dataNullable()
+
+
+    /**
+     * Test that "readonly" when not used as the keyword is still tokenized as `T_STRING`.
+     *
+     * @param string $testMarker The comment which prefaces the target token in the test file.
+     *
+     * @dataProvider dataInlineThen
+     *
+     * @return void
+     */
+    public function testInlineThen($testMarker)
+    {
+        $tokens     = $this->phpcsFile->getTokens();
+        $target     = $this->getTargetToken($testMarker, [T_NULLABLE, T_INLINE_THEN]);
+        $tokenArray = $tokens[$target];
+
+        $this->assertSame(T_INLINE_THEN, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_INLINE_THEN (code)');
+        $this->assertSame('T_INLINE_THEN', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_INLINE_THEN (type)');
+
+    }//end testInlineThen()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testInlineThen()
+     *
+     * @return array<string, array<string>>
+     */
+    public static function dataInlineThen()
+    {
+        return [
+            'ternary in property default value' => ['/* testInlineThenInPropertyDefaultValue */'],
+        ];
+
+    }//end dataInlineThen()
+
+
+}//end class