PHP 8.0 | "undo" namespaced names as single token

package.xml

@@ -121,6 +121,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
       <file baseinstalldir="" name="StableCommentWhitespaceTest.php" role="test" />
       <file baseinstalldir="" name="StableCommentWhitespaceWinTest.inc" role="test" />
       <file baseinstalldir="" name="StableCommentWhitespaceWinTest.php" role="test" />
+      <file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.inc" role="test" />
+      <file baseinstalldir="" name="UndoNamespacedNameSingleTokenTest.php" role="test" />
      </dir>
      <file baseinstalldir="" name="AbstractMethodUnitTest.php" role="test" />
      <file baseinstalldir="" name="AllTests.php" role="test" />
@@ -1997,6 +1999,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
    <install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
    <install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
   </filelist>
@@ -2058,6 +2062,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" name="tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" name="tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc" />
    <install as="CodeSniffer/Standards/AllSniffs.php" name="tests/Standards/AllSniffs.php" />
    <install as="CodeSniffer/Standards/AbstractSniffUnitTest.php" name="tests/Standards/AbstractSniffUnitTest.php" />
    <ignore name="bin/phpcs.bat" />

src/Tokenizers/PHP.php

@@ -815,6 +815,81 @@ protected function tokenize($string)
                 continue;
             }//end if
 
+            /*
+                As of PHP 8.0 fully qualified, partially qualified and namespace relative
+                identifier names are tokenized differently.
+                This "undoes" the new tokenization so the tokenization will be the same in
+                in PHP 5, 7 and 8.
+            */
+
+            if (PHP_VERSION_ID >= 80000
+                && $tokenIsArray === true
+                && ($token[0] === T_NAME_QUALIFIED
+                || $token[0] === T_NAME_FULLY_QUALIFIED
+                || $token[0] === T_NAME_RELATIVE)
+            ) {
+                $name = $token[1];
+
+                if ($token[0] === T_NAME_FULLY_QUALIFIED) {
+                    $newToken            = [];
+                    $newToken['code']    = T_NS_SEPARATOR;
+                    $newToken['type']    = 'T_NS_SEPARATOR';
+                    $newToken['content'] = '\\';
+                    $finalTokens[$newStackPtr] = $newToken;
+                    ++$newStackPtr;
+
+                    $name = ltrim($name, '\\');
+                }
+
+                if ($token[0] === T_NAME_RELATIVE) {
+                    $newToken            = [];
+                    $newToken['code']    = T_NAMESPACE;
+                    $newToken['type']    = 'T_NAMESPACE';
+                    $newToken['content'] = substr($name, 0, 9);
+                    $finalTokens[$newStackPtr] = $newToken;
+                    ++$newStackPtr;
+
+                    $newToken            = [];
+                    $newToken['code']    = T_NS_SEPARATOR;
+                    $newToken['type']    = 'T_NS_SEPARATOR';
+                    $newToken['content'] = '\\';
+                    $finalTokens[$newStackPtr] = $newToken;
+                    ++$newStackPtr;
+
+                    $name = substr($name, 10);
+                }
+
+                $parts     = explode('\\', $name);
+                $partCount = count($parts);
+                $lastPart  = ($partCount - 1);
+
+                foreach ($parts as $i => $part) {
+                    $newToken            = [];
+                    $newToken['code']    = T_STRING;
+                    $newToken['type']    = 'T_STRING';
+                    $newToken['content'] = $part;
+                    $finalTokens[$newStackPtr] = $newToken;
+                    ++$newStackPtr;
+
+                    if ($i !== $lastPart) {
+                        $newToken            = [];
+                        $newToken['code']    = T_NS_SEPARATOR;
+                        $newToken['type']    = 'T_NS_SEPARATOR';
+                        $newToken['content'] = '\\';
+                        $finalTokens[$newStackPtr] = $newToken;
+                        ++$newStackPtr;
+                    }
+                }
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    $type    = Util\Tokens::tokenName($token[0]);
+                    $content = Util\Common::prepareForOutput($token[1]);
+                    echo "\t\t* token $stackPtr split into individual tokens; was: $type => $content".PHP_EOL;
+                }
+
+                continue;
+            }//end if
+
             /*
                 Before PHP 7.0, the "yield from" was tokenized as
                 T_YIELD, T_WHITESPACE and T_STRING. So look for
@@ -1131,7 +1206,7 @@ protected function tokenize($string)
                  * Check if the next non-empty token is one of the tokens which can be used
                  * in type declarations. If not, it's definitely a ternary.
                  * At this point, the only token types which need to be taken into consideration
-                 * as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
+                 * as potential type declarations are identifier names, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
                  */
 
                 $lastRelevantNonEmpty = null;
@@ -1148,6 +1223,9 @@ protected function tokenize($string)
                     }
 
                     if ($tokenType === T_STRING
+                        || $tokenType === T_NAME_FULLY_QUALIFIED
+                        || $tokenType === T_NAME_RELATIVE
+                        || $tokenType === T_NAME_QUALIFIED
                         || $tokenType === T_ARRAY
                         || $tokenType === T_NS_SEPARATOR
                     ) {
@@ -1159,7 +1237,10 @@ protected function tokenize($string)
                         && isset($lastRelevantNonEmpty) === false)
                         || ($lastRelevantNonEmpty === T_ARRAY
                         && $tokenType === '(')
-                        || ($lastRelevantNonEmpty === T_STRING
+                        || (($lastRelevantNonEmpty === T_STRING
+                        || $lastRelevantNonEmpty === T_NAME_FULLY_QUALIFIED
+                        || $lastRelevantNonEmpty === T_NAME_RELATIVE
+                        || $lastRelevantNonEmpty === T_NAME_QUALIFIED)
                         && ($tokenType === T_DOUBLE_COLON
                         || $tokenType === '('
                         || $tokenType === ':'))
@@ -1304,6 +1385,10 @@ protected function tokenize($string)
                 tokenized as T_STRING even if it appears to be a different token,
                 such as when writing code like: function default(): foo
                 so go forward and change the token type before it is processed.
+
+                Note: this should not be done for `function Level\Name` within a
+                group use statement for the PHP 8 identifier name tokens as it
+                would interfere with the re-tokenization of those.
             */
 
             if ($tokenIsArray === true
@@ -1321,7 +1406,10 @@ protected function tokenize($string)
                         }
                     }
 
-                    if ($x < $numTokens && is_array($tokens[$x]) === true) {
+                    if ($x < $numTokens
+                        && is_array($tokens[$x]) === true
+                        && $tokens[$x][0] !== T_NAME_QUALIFIED
+                    ) {
                         if (PHP_CODESNIFFER_VERBOSITY > 1) {
                             $oldType = Util\Tokens::tokenName($tokens[$x][0]);
                             echo "\t\t* token $x changed from $oldType to T_STRING".PHP_EOL;
@@ -1377,12 +1465,15 @@ function return types. We want to keep the parenthesis map clean,
                         && $tokens[$x] === ':'
                     ) {
                         $allowed = [
-                            T_STRING       => T_STRING,
-                            T_ARRAY        => T_ARRAY,
-                            T_CALLABLE     => T_CALLABLE,
-                            T_SELF         => T_SELF,
-                            T_PARENT       => T_PARENT,
-                            T_NS_SEPARATOR => T_NS_SEPARATOR,
+                            T_STRING               => T_STRING,
+                            T_NAME_FULLY_QUALIFIED => T_NAME_FULLY_QUALIFIED,
+                            T_NAME_RELATIVE        => T_NAME_RELATIVE,
+                            T_NAME_QUALIFIED       => T_NAME_QUALIFIED,
+                            T_ARRAY                => T_ARRAY,
+                            T_CALLABLE             => T_CALLABLE,
+                            T_SELF                 => T_SELF,
+                            T_PARENT               => T_PARENT,
+                            T_NS_SEPARATOR         => T_NS_SEPARATOR,
                         ];
 
                         $allowed += Util\Tokens::$emptyTokens;

src/Util/Tokens.php

@@ -129,6 +129,18 @@
     define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
 }
 
+if (defined('T_NAME_QUALIFIED') === false) {
+    define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
+}
+
+if (defined('T_NAME_FULLY_QUALIFIED') === false) {
+    define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
+}
+
+if (defined('T_NAME_RELATIVE') === false) {
+    define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
+}
+
 // Tokens used for parsing doc blocks.
 define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
 define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

tests/Core/Tokenizer/UndoNamespacedNameSingleTokenTest.inc

@@ -0,0 +1,147 @@
+<?php
+
+/* testNamespaceDeclaration */
+namespace Package;
+
+/* testNamespaceDeclarationWithLevels */
+namespace Vendor\SubLevel\Domain;
+
+/* testUseStatement */
+use ClassName;
+
+/* testUseStatementWithLevels */
+use Vendor\Level\Domain;
+
+/* testFunctionUseStatement */
+use function function_name;
+
+/* testFunctionUseStatementWithLevels */
+use function Vendor\Level\function_in_ns;
+
+/* testConstantUseStatement */
+use const CONSTANT_NAME;
+
+/* testConstantUseStatementWithLevels */
+use const Vendor\Level\OTHER_CONSTANT;
+
+/* testMultiUseUnqualified */
+use UnqualifiedClassName,
+    /* testMultiUsePartiallyQualified */
+    Sublevel\PartiallyClassName;
+
+/* testGroupUseStatement */
+use Vendor\Level\{
+    AnotherDomain,
+    function function_grouped,
+    const CONSTANT_GROUPED,
+    Sub\YetAnotherDomain,
+    function SubLevelA\function_grouped_too,
+    const SubLevelB\CONSTANT_GROUPED_TOO,
+};
+
+/* testClassName */
+class MyClass
+    /* testExtendedFQN */
+    extends \Vendor\Level\FQN
+    /* testImplementsRelative */
+    implements namespace\Name,
+        /* testImplementsFQN */
+        \Fully\Qualified,
+        /* testImplementsUnqualified */
+        Unqualified,
+        /* testImplementsPartiallyQualified */
+        Sub\Level\Name
+{
+    /* testFunctionName */
+    public function function_name(
+        /* testTypeDeclarationRelative */
+        ?namespace\Name|object $paramA,
+
+        /* testTypeDeclarationFQN */
+        \Fully\Qualified\Name $paramB,
+
+        /* testTypeDeclarationUnqualified */
+        Unqualified|false $paramC,
+
+        /* testTypeDeclarationPartiallyQualified */
+        ?Sublevel\Name $paramD,
+
+    /* testReturnTypeFQN */
+    ) : ?\Name {
+
+        try {
+            /* testFunctionCallRelative */
+            echo NameSpace\function_name();
+
+            /* testFunctionCallFQN */
+            echo \Vendor\Package\function_name();
+
+            /* testFunctionCallUnqualified */
+            echo function_name();
+
+            /* testFunctionPartiallyQualified */
+            echo Level\function_name();
+
+        /* testCatchRelative */
+        } catch (namespace\SubLevel\Exception $e) {
+
+        /* testCatchFQN */
+        } catch (\Exception $e) {
+
+        /* testCatchUnqualified */
+        } catch (Exception $e) {
+
+        /* testCatchPartiallyQualified */
+        } catch (Level\Exception $e) {
+        }
+
+        /* testNewRelative */
+        $obj = new namespace\ClassName();
+
+        /* testNewFQN */
+        $obj = new \Vendor\ClassName();
+
+        /* testNewUnqualified */
+        $obj = new ClassName;
+
+        /* testNewPartiallyQualified */
+        $obj = new Level\ClassName;
+
+        /* testDoubleColonRelative */
+        $value = namespace\ClassName::property;
+
+        /* testDoubleColonFQN */
+        $value = \ClassName::static_function();
+
+        /* testDoubleColonUnqualified */
+        $value = ClassName::CONSTANT_NAME;
+
+        /* testDoubleColonPartiallyQualified */
+        $value = Level\ClassName::CONSTANT_NAME['key'];
+
+        /* testInstanceOfRelative */
+        $is = $obj instanceof namespace\ClassName;
+
+        /* testInstanceOfFQN */
+        if ($obj instanceof \Full\ClassName) {}
+
+        /* testInstanceOfUnqualified */
+        if ($a === $b && $obj instanceof ClassName && true) {}
+
+        /* testInstanceOfPartiallyQualified */
+        $is = $obj instanceof Partially\ClassName;
+    }
+}
+
+/* testInvalidInPHP8Whitespace */
+namespace \ Sublevel
+          \ function_name();
+
+/* testInvalidInPHP8Comments */
+$value = \Fully
+    // phpcs:ignore Stnd.Cat.Sniff -- for reasons
+    \Qualified
+    /* comment */
+    \Name
+    // comment
+    :: function_name();