Skip to content

Commit e921934

Browse files
jrfnljrfnl
committed
Tokenizer: apply tab replacement to heredoc/nowdoc closers
Since PHP 7.3, heredoc/nowdoc closers may be indented. This indent can use either tabs or spaces and the indent is included in the `T_END_HEREDOC`/`T_END_NOWDOC` token contents as received from the PHP native tokenizer. However, the PHPCS `Tokenizer` did no execute tab replacement on these token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes. This commit adds the `T_END_HEREDOC`/`T_END_NOWDOC` tokens to the array of tokens for which to do tab replacement to make them more consistent with the rest of PHPCS. I also considered splitting the token into a `T_WHITESPACE` token and the `T_END_HEREDOC`/`T_END_NOWDOC` token, but that could potentially break sniffs which expect the `T_END_HEREDOC`/`T_END_NOWDOC` token directly after the last `T_HEREDOC`/`T_NOWDOC` token. The current fix does not contain that risk. Includes unit tests safeguarding this change. The tests will only run on PHP 7.3+ as flexible heredoc/nowdocs don't tokenize correctly in PHP < 7.3.
1 parent f3a8342 commit e921934

File tree

4 files changed

+201
-0
lines changed

4 files changed

+201
-0
lines changed

‎package.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
143143
<file baseinstalldir="" name="FinallyTest.php" role="test" />
144144
<file baseinstalldir="" name="GotoLabelTest.inc" role="test" />
145145
<file baseinstalldir="" name="GotoLabelTest.php" role="test" />
146+
<file baseinstalldir="" name="HeredocNowdocCloserTest.inc" role="test" />
147+
<file baseinstalldir="" name="HeredocNowdocCloserTest.php" role="test" />
146148
<file baseinstalldir="" name="NamedFunctionCallArgumentsTest.inc" role="test" />
147149
<file baseinstalldir="" name="NamedFunctionCallArgumentsTest.php" role="test" />
148150
<file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
@@ -2116,6 +2118,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21162118
<install as="CodeSniffer/Core/Tokenizer/FinallyTest.inc" name="tests/Core/Tokenizer/FinallyTest.inc" />
21172119
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
21182120
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
2121+
<install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.php" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.php" />
2122+
<install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.inc" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.inc" />
21192123
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
21202124
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
21212125
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
@@ -2220,6 +2224,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
22202224
<install as="CodeSniffer/Core/Tokenizer/FinallyTest.inc" name="tests/Core/Tokenizer/FinallyTest.inc" />
22212225
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
22222226
<install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
2227+
<install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.php" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.php" />
2228+
<install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.inc" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.inc" />
22232229
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
22242230
<install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
22252231
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />

‎src/Tokenizers/Tokenizer.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,8 @@ private function createPositionMap()
194194
T_DOUBLE_QUOTED_STRING => true,
195195
T_HEREDOC => true,
196196
T_NOWDOC => true,
197+
T_END_HEREDOC => true,
198+
T_END_NOWDOC => true,
197199
T_INLINE_HTML => true,
198200
];
199201

‎tests/Core/Tokenizer/HeredocNowdocCloserTest.inc

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<?php
2+
3+
/* testHeredocCloserNoIndent */
4+
$heredoc = <<<EOD
5+
some text
6+
some text
7+
some text
8+
EOD;
9+
10+
/* testNowdocCloserNoIndent */
11+
$nowdoc = <<<'EOD'
12+
some text
13+
some text
14+
some text
15+
EOD;
16+
17+
/* testHeredocCloserSpaceIndent */
18+
$heredoc = <<<END
19+
a
20+
b
21+
c
22+
END;
23+
24+
/* testNowdocCloserSpaceIndent */
25+
$nowdoc = <<<'END'
26+
a
27+
b
28+
c
29+
END;
30+
31+
/* testHeredocCloserTabIndent */
32+
$heredoc = <<<"END"
33+
a
34+
b
35+
c
36+
END;
37+
38+
/* testNowdocCloserTabIndent */
39+
$nowdoc = <<<'END'
40+
a
41+
b
42+
c
43+
END;

‎tests/Core/Tokenizer/HeredocNowdocCloserTest.php

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of goto declarations and statements.
4+
*
5+
* @author Juliette Reinders Folmer <[email protected]>
6+
* @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
11+
12+
use PHP_CodeSniffer\Config;
13+
use PHP_CodeSniffer\Ruleset;
14+
use PHP_CodeSniffer\Files\DummyFile;
15+
use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
16+
17+
/**
18+
* Heredoc/nowdoc closer token test.
19+
*
20+
* @requires PHP 7.3
21+
*/
22+
class HeredocNowdocCloserTest extends AbstractMethodUnitTest
23+
{
24+
25+
26+
/**
27+
* Initialize & tokenize \PHP_CodeSniffer\Files\File with code from the test case file.
28+
*
29+
* {@internal This is a near duplicate of the original method. Only difference is that
30+
* tab replacement is enabled for this test.}
31+
*
32+
* @return void
33+
*/
34+
public static function setUpBeforeClass()
35+
{
36+
$config = new Config();
37+
$config->standards = ['PSR1'];
38+
$config->tabWidth = 4;
39+
40+
$ruleset = new Ruleset($config);
41+
42+
// Default to a file with the same name as the test class. Extension is property based.
43+
$relativeCN = str_replace(__NAMESPACE__, '', get_called_class());
44+
$relativePath = str_replace('\\', DIRECTORY_SEPARATOR, $relativeCN);
45+
$pathToTestFile = realpath(__DIR__).$relativePath.'.'.static::$fileExtension;
46+
47+
// Make sure the file gets parsed correctly based on the file type.
48+
$contents = 'phpcs_input_file: '.$pathToTestFile.PHP_EOL;
49+
$contents .= file_get_contents($pathToTestFile);
50+
51+
self::$phpcsFile = new DummyFile($contents, $ruleset, $config);
52+
self::$phpcsFile->process();
53+
54+
}//end setUpBeforeClass()
55+
56+
57+
/**
58+
* Verify that leading (indent) whitespace in a heredoc/nowdoc closer token get the tab replacement treatment.
59+
*
60+
* @param string $testMarker The comment prefacing the target token.
61+
* @param array $expected Expectations for the token array.
62+
*
63+
* @dataProvider dataHeredocNowdocCloserTabReplacement
64+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
65+
*
66+
* @return void
67+
*/
68+
public function testHeredocNowdocCloserTabReplacement($testMarker, $expected)
69+
{
70+
$tokens = self::$phpcsFile->getTokens();
71+
72+
$closer = $this->getTargetToken($testMarker, [T_END_HEREDOC, T_END_NOWDOC]);
73+
74+
foreach ($expected as $key => $value) {
75+
if ($key === 'orig_content' && $value === null) {
76+
$this->assertArrayNotHasKey($key, $tokens[$closer], "Unexpected 'orig_content' key found in the token array.");
77+
continue;
78+
}
79+
80+
$this->assertArrayHasKey($key, $tokens[$closer], "Key $key not found in the token array.");
81+
$this->assertSame($value, $tokens[$closer][$key], "Value for key $key does not match expectation.");
82+
}
83+
84+
}//end testHeredocNowdocCloserTabReplacement()
85+
86+
87+
/**
88+
* Data provider.
89+
*
90+
* @see testHeredocNowdocCloserTabReplacement()
91+
*
92+
* @return array
93+
*/
94+
public function dataHeredocNowdocCloserTabReplacement()
95+
{
96+
return [
97+
[
98+
'testMarker' => '/* testHeredocCloserNoIndent */',
99+
'expected' => [
100+
'length' => 3,
101+
'content' => 'EOD',
102+
'orig_content' => null,
103+
],
104+
],
105+
[
106+
'testMarker' => '/* testNowdocCloserNoIndent */',
107+
'expected' => [
108+
'length' => 3,
109+
'content' => 'EOD',
110+
'orig_content' => null,
111+
],
112+
],
113+
[
114+
'testMarker' => '/* testHeredocCloserSpaceIndent */',
115+
'expected' => [
116+
'length' => 7,
117+
'content' => ' END',
118+
'orig_content' => null,
119+
],
120+
],
121+
[
122+
'testMarker' => '/* testNowdocCloserSpaceIndent */',
123+
'expected' => [
124+
'length' => 8,
125+
'content' => ' END',
126+
'orig_content' => null,
127+
],
128+
],
129+
[
130+
'testMarker' => '/* testHeredocCloserTabIndent */',
131+
'expected' => [
132+
'length' => 8,
133+
'content' => ' END',
134+
'orig_content' => ' END',
135+
],
136+
],
137+
[
138+
'testMarker' => '/* testNowdocCloserTabIndent */',
139+
'expected' => [
140+
'length' => 7,
141+
'content' => ' END',
142+
'orig_content' => ' END',
143+
],
144+
],
145+
];
146+
147+
}//end dataHeredocNowdocCloserTabReplacement()
148+
149+
150+
}//end class

0 commit comments

Comments
 (0)