From 43299a89463ef0d68cd74f9d3ed7a3b5a3cdd069 Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Sun, 6 Apr 2025 05:15:03 +0200
Subject: [PATCH] PHP 7.4 | Tokenizer/PHP: handle PHP tag at end of file
 consistently

Prior to PHP 7.4, a PHP open tag at the end of a file was not tokenized correctly in PHP itself.

From the PHP 7.4 changelog:
> `<?php` at the end of the file (without trailing newline) will now be
> interpreted as an opening PHP tag. Previously it was interpreted either as
> `<? php` and resulted in a syntax error (with short_open_tag=1) or was
> interpreted as a literal `<?php` string (with short_open_tag=0).

This commit makes the tokenization of PHP open tags at the end of a file consistent in all PHP versions.

Includes tests.
Includes adding a few extra test builds which test with `short_open_tag=On` for various PHP versions.

Refs:
* https://www.php.net/manual/en/migration74.incompatible.php#migration74.incompatible.core.php-tag
* https://github.com/php/php-src/blob/30de357fa14480468132bbc22a272aeb91789ba8/UPGRADING#L37-L40
---
 .github/workflows/test.yml                    |  9 +++-
 src/Tokenizers/PHP.php                        | 40 ++++++++++++++
 .../Tokenizers/PHP/PHPOpenTagEOF1Test.inc     |  4 ++
 .../Tokenizers/PHP/PHPOpenTagEOF1Test.php     | 54 +++++++++++++++++++
 .../Tokenizers/PHP/PHPOpenTagEOF2Test.inc     |  4 ++
 .../Tokenizers/PHP/PHPOpenTagEOF2Test.php     | 54 +++++++++++++++++++
 .../Tokenizers/PHP/PHPOpenTagEOF3Test.inc     |  4 ++
 .../Tokenizers/PHP/PHPOpenTagEOF3Test.php     | 54 +++++++++++++++++++
 8 files changed, 222 insertions(+), 1 deletion(-)
 create mode 100644 tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.inc
 create mode 100644 tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php
 create mode 100644 tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.inc
 create mode 100644 tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.php
 create mode 100644 tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.inc
 create mode 100644 tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.php

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6e32587624..83da62422a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -92,6 +92,12 @@ jobs:
           - php: '7.0'
             os: 'ubuntu-latest'
             custom_ini: true
+          - php: '8.0'
+            os: 'ubuntu-latest'
+            custom_ini: true
+          - php: '8.2'
+            os: 'ubuntu-latest'
+            custom_ini: true
 
     # yamllint disable-line rule:line-length
     name: "PHP: ${{ matrix.php }} ${{ matrix.custom_ini && ' with custom ini settings' || '' }}${{ matrix.libxml_minor && format( ' with libxml {0}', matrix.libxml_minor ) || '' }} (${{ matrix.os == 'ubuntu-latest' && 'Linux' || 'Win'  }})"
@@ -177,7 +183,7 @@ jobs:
           # Also turn on error_reporting to ensure all notices are shown.
           if [[ ${{ matrix.custom_ini }} == true && "${{ matrix.php }}" == '5.5' ]]; then
             echo 'PHP_INI=error_reporting=-1, display_errors=On, date.timezone=Australia/Sydney, short_open_tag=On, asp_tags=On' >> "$GITHUB_OUTPUT"
-          elif [[ ${{ matrix.custom_ini }} == true && "${{ matrix.php }}" == '7.0' ]]; then
+          elif [[ ${{ matrix.custom_ini }} == true && "${{ matrix.php }}" != '5.5' ]]; then
             echo 'PHP_INI=error_reporting=-1, display_errors=On, date.timezone=Australia/Sydney, short_open_tag=On' >> "$GITHUB_OUTPUT"
           else
             echo 'PHP_INI=error_reporting=-1, display_errors=On' >> "$GITHUB_OUTPUT"
@@ -233,6 +239,7 @@ jobs:
           PHP_CODESNIFFER_CBF: '1'
 
       - name: 'PHPCS: check code style without cache, no parallel'
+        if: ${{ matrix.custom_ini == false }}
         id: phpcs
         run: >
           php "bin/phpcs" --no-cache --parallel=1
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
index 5c5434d5b1..32fefc9473 100644
--- a/src/Tokenizers/PHP.php
+++ b/src/Tokenizers/PHP.php
@@ -803,6 +803,46 @@ protected function tokenize($string)
                 }
             }//end if
 
+            /*
+                Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file
+                (without a new line), so we need to make sure that the tokenization in PHPCS is consistent
+                cross-version PHP by retokenizing to T_OPEN_TAG.
+            */
+
+            if (PHP_VERSION_ID < 70400
+                && $tokenIsArray === true
+                // PHP < 7.4 with short open tags off.
+                && (($stackPtr === ($numTokens - 1)
+                && $token[0] === T_INLINE_HTML
+                && stripos($token[1], '<?php') === 0)
+                // PHP < 7.4 with short open tags on.
+                || ($stackPtr === ($numTokens - 2)
+                && $token[0] === T_OPEN_TAG
+                && $token[1] === '<?'
+                && is_array($tokens[($stackPtr + 1)]) === true
+                && $tokens[($stackPtr + 1)][0] === T_STRING
+                && strtolower($tokens[($stackPtr + 1)][1]) === 'php'))
+            ) {
+                if ($token[0] === T_INLINE_HTML) {
+                    $finalTokens[$newStackPtr] = [
+                        'code'    => T_OPEN_TAG,
+                        'type'    => 'T_OPEN_TAG',
+                        'content' => $token[1],
+                    ];
+                } else {
+                    $finalTokens[$newStackPtr] = [
+                        'code'    => T_OPEN_TAG,
+                        'type'    => 'T_OPEN_TAG',
+                        'content' => $token[1].$tokens[($stackPtr + 1)][1],
+                    ];
+
+                    $stackPtr++;
+                }
+
+                $newStackPtr++;
+                continue;
+            }//end if
+
             /*
                 Parse doc blocks into something that can be easily iterated over.
             */
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.inc b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.inc
new file mode 100644
index 0000000000..2244b38915
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.inc
@@ -0,0 +1,4 @@
+<?php
+// This test case must be the last (only) test in the file without a new line after it!
+/* testLongOpenTagEndOfFileSpaceNoNewLine */ ?>
+<?php 
\ No newline at end of file
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php
new file mode 100644
index 0000000000..414258e51a
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php
@@ -0,0 +1,54 @@
+<?php
+/**
+ * Tests the tokenization of PHP open tags.
+ *
+ * Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
+ * so we need to make sure that the tokenization in PHPCS is consistent and correct.
+ *
+ * @copyright 2025 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+use PHP_CodeSniffer\Util\Tokens;
+
+/**
+ * Tests the tokenization of PHP open tags.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+ */
+final class PHPOpenTagEOF1Test extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
+     *
+     * @return void
+     */
+    public function testLongOpenTagAtEndOfFile()
+    {
+        $tokens   = $this->phpcsFile->getTokens();
+        $stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileSpaceNoNewLine */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
+
+        $this->assertSame(
+            T_OPEN_TAG,
+            $tokens[$stackPtr]['code'],
+            'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
+        );
+        $this->assertSame(
+            'T_OPEN_TAG',
+            $tokens[$stackPtr]['type'],
+            'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
+        );
+        $this->assertSame('<?php ', $tokens[$stackPtr]['content']);
+
+        // Now make sure that this is the very last token in the file and there are no tokens after it.
+        $this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
+
+    }//end testLongOpenTagAtEndOfFile()
+
+
+}//end class
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.inc b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.inc
new file mode 100644
index 0000000000..191bdae16f
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.inc
@@ -0,0 +1,4 @@
+<?php
+// This test case must be the last (only) test in the file without a new line after it!
+/* testLongOpenTagEndOfFileNoSpaceNoNewLine */ ?>
+<?php
\ No newline at end of file
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.php b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.php
new file mode 100644
index 0000000000..a00e91fc08
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF2Test.php
@@ -0,0 +1,54 @@
+<?php
+/**
+ * Tests the tokenization of PHP open tags.
+ *
+ * Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
+ * so we need to make sure that the tokenization in PHPCS is consistent and correct.
+ *
+ * @copyright 2025 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+use PHP_CodeSniffer\Util\Tokens;
+
+/**
+ * Tests the tokenization of PHP open tags.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+ */
+final class PHPOpenTagEOF2Test extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
+     *
+     * @return void
+     */
+    public function testLongOpenTagAtEndOfFile()
+    {
+        $tokens   = $this->phpcsFile->getTokens();
+        $stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileNoSpaceNoNewLine */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
+
+        $this->assertSame(
+            T_OPEN_TAG,
+            $tokens[$stackPtr]['code'],
+            'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
+        );
+        $this->assertSame(
+            'T_OPEN_TAG',
+            $tokens[$stackPtr]['type'],
+            'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
+        );
+        $this->assertSame('<?php', $tokens[$stackPtr]['content']);
+
+        // Now make sure that this is the very last token in the file and there are no tokens after it.
+        $this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
+
+    }//end testLongOpenTagAtEndOfFile()
+
+
+}//end class
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.inc b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.inc
new file mode 100644
index 0000000000..583314db33
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.inc
@@ -0,0 +1,4 @@
+<?php
+// This test case must be the last (only) test in the file without a new line after it!
+/* testLongOpenTagEndOfFileNoSpaceNoNewLineUppercase */ ?>
+<?PHP
\ No newline at end of file
diff --git a/tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.php b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.php
new file mode 100644
index 0000000000..e442c19118
--- /dev/null
+++ b/tests/Core/Tokenizers/PHP/PHPOpenTagEOF3Test.php
@@ -0,0 +1,54 @@
+<?php
+/**
+ * Tests the tokenization of PHP open tags.
+ *
+ * Prior to PHP 7.4, PHP didn't support stand-alone PHP open tags at the end of a file (without a new line),
+ * so we need to make sure that the tokenization in PHPCS is consistent and correct.
+ *
+ * @copyright 2025 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\PHP;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+use PHP_CodeSniffer\Util\Tokens;
+
+/**
+ * Tests the tokenization of PHP open tags.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+ */
+final class PHPOpenTagEOF3Test extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Test that the tokenization of a long PHP open tag at the very end of a file is correct and consistent.
+     *
+     * @return void
+     */
+    public function testLongOpenTagAtEndOfFile()
+    {
+        $tokens   = $this->phpcsFile->getTokens();
+        $stackPtr = $this->getTargetToken('/* testLongOpenTagEndOfFileNoSpaceNoNewLineUppercase */', [T_OPEN_TAG, T_STRING, T_INLINE_HTML]);
+
+        $this->assertSame(
+            T_OPEN_TAG,
+            $tokens[$stackPtr]['code'],
+            'Token tokenized as '.Tokens::tokenName($tokens[$stackPtr]['code']).', not T_OPEN_TAG (code)'
+        );
+        $this->assertSame(
+            'T_OPEN_TAG',
+            $tokens[$stackPtr]['type'],
+            'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
+        );
+        $this->assertSame('<?PHP', $tokens[$stackPtr]['content']);
+
+        // Now make sure that this is the very last token in the file and there are no tokens after it.
+        $this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
+
+    }//end testLongOpenTagAtEndOfFile()
+
+
+}//end class