Fix parsing of comment without space (#347)

VincentLanglet · Jan 16, 2025 · 0ab7a81 · 0ab7a81
1 parent 3d17f1e
commit 0ab7a81
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 5 deletions.
diff --git a/src/Token/Tokenizer.php b/src/Token/Tokenizer.php
@@ -478,17 +478,16 @@ private function lexData(int $limit = 0): void
             $this->lexEOL($match[0]);
         } elseif (1 === preg_match('/\S+/', $this->code, $match, 0, $this->cursor)) {
             $value = $match[0];
+            // Stop if cursor reaches the next expression starter.
+            if ($limit > $this->cursor) {
+                $value = substr($value, 0, $limit - $this->cursor);
+            }
 
             if (self::STATE_COMMENT === $this->getState()) {
                 $this->pushToken(Token::COMMENT_TEXT_TYPE, $value);
             } elseif (self::STATE_INLINE_COMMENT === $this->getState()) {
                 $this->pushToken(Token::INLINE_COMMENT_TEXT_TYPE, $value);
             } else {
-                // Stop if cursor reaches the next expression starter.
-                if (0 !== $limit) {
-                    $value = substr($value, 0, $limit - $this->cursor);
-                }
-
                 $this->pushToken(Token::TEXT_TYPE, $value);
             }
         } else {

diff --git a/tests/Token/Tokenizer/Fixtures/test18.twig b/tests/Token/Tokenizer/Fixtures/test18.twig
@@ -0,0 +1 @@
+{#comment without space#}
diff --git a/tests/Token/Tokenizer/TokenizerTest.php b/tests/Token/Tokenizer/TokenizerTest.php
@@ -836,6 +836,21 @@ public static function tokenizeDataProvider(): iterable
                 14 => Token::EOF_TYPE,
             ],
         ];
+
+        yield [
+            __DIR__.'/Fixtures/test18.twig',
+            [
+                0 => Token::COMMENT_START_TYPE,
+                1 => Token::COMMENT_TEXT_TYPE,
+                2 => Token::COMMENT_WHITESPACE_TYPE,
+                3 => Token::COMMENT_TEXT_TYPE,
+                4 => Token::COMMENT_WHITESPACE_TYPE,
+                5 => Token::COMMENT_TEXT_TYPE,
+                6 => Token::COMMENT_END_TYPE,
+                7 => Token::EOL_TYPE,
+                8 => Token::EOF_TYPE,
+            ],
+        ];
     }
 
     /**