Do not assume "SET" as top level when preceded with "CHARACTER" (#150)

* Do not assume "SET" as top level when preceded with "CHARACTER" * test PHP 8.4
doctrine · Jan 24, 2025 · d6d00ab · d6d00ab
1 parent 1e04414
commit d6d00ab
Show file tree

Hide file tree

Showing 8 changed files with 20 additions and 2 deletions.
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -13,6 +13,6 @@ jobs:
     name: "PHPUnit"
     uses: "doctrine/.github/.github/workflows/[email protected]"
     with:
-      php-versions: '["8.1", "8.2", "8.3"]'
+      php-versions: '["8.1", "8.2", "8.3", "8.4"]'
     secrets:
       CODECOV_TOKEN: "${{ secrets.CODECOV_TOKEN }}"
diff --git a/src/Tokenizer.php b/src/Tokenizer.php
@@ -868,7 +868,7 @@ private function makeTokenizeRegexes(): array
             Token::TOKEN_TYPE_BOUNDARY => $regexBoundaries,
             // A reserved word cannot be preceded by a '.'
             // this makes it so in "mytable.from", "from" is not considered a reserved word
-            Token::TOKEN_TYPE_RESERVED_TOPLEVEL => '(?<!\.)' . $regexReservedToplevel . '(?=$|\s|' . $regexBoundaries . ')',
+            Token::TOKEN_TYPE_RESERVED_TOPLEVEL => '(?<!\.|\sCHARACTER\s(?=SET\s))' . $regexReservedToplevel . '(?=$|\s|' . $regexBoundaries . ')',
             Token::TOKEN_TYPE_RESERVED_NEWLINE => '(?<!\.)' . $regexReservedNewline . '(?=$|\s|' . $regexBoundaries . ')',
             Token::TOKEN_TYPE_RESERVED => '(?<!\.)' . $regexReserved . '(?=$|\s|' . $regexBoundaries . ')'
                 // A function must be succeeded by '('

diff --git a/tests/clihighlight.txt b/tests/clihighlight.txt
@@ -1201,3 +1201,7 @@ MY_NON_TOP_LEVEL_KEYWORD_FX_3[0m();[0m
   t[0m
 [37mWHERE[0m
   t[0m.[0mstart[0m =[0m t[0m.[0mend[0m
+---
+[37mCREATE[0m [37mTABLE[0m t[0m (
+  c[0m [37mVARCHAR[0m([32;1m20[0m)
+) [37mDEFAULT[0m [37mCHARACTER[0m [37mSET[0m utf8mb4[0m [37mENGINE[0m =[0m InnoDB[0m
diff --git a/tests/compress.txt b/tests/compress.txt
@@ -109,3 +109,5 @@ BEGIN FOR i IN 1..5 LOOP DBMS_OUTPUT.PUT_LINE(i); END LOOP; END;
 SELECT a FROM test STRAIGHT_JOIN test2 ON test.id = test2.id
 ---
 SELECT t.id, t.start, t.end, t.end AS e2, t.limit, t.begin, t.case, t.when, t.then, t.else FROM t WHERE t.start = t.end
+---
+CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
diff --git a/tests/format-highlight.html b/tests/format-highlight.html
@@ -1201,3 +1201,7 @@
   <span style="color: #333;">t</span>
 <span style="font-weight:bold;">WHERE</span>
   <span style="color: #333;">t</span><span >.</span><span style="color: #333;">start</span> <span >=</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">end</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">CREATE</span> <span style="font-weight:bold;">TABLE</span> <span style="color: #333;">t</span> (
+  <span style="color: #333;">c</span> <span style="font-weight:bold;">VARCHAR</span>(<span style="color: green;">20</span>)
+) <span style="font-weight:bold;">DEFAULT</span> <span style="font-weight:bold;">CHARACTER</span> <span style="font-weight:bold;">SET</span> <span style="color: #333;">utf8mb4</span> <span style="font-weight:bold;">ENGINE</span> <span >=</span> <span style="color: #333;">InnoDB</span></pre>
diff --git a/tests/format.txt b/tests/format.txt
@@ -1199,3 +1199,7 @@ FROM
   t
 WHERE
   t.start = t.end
+---
+CREATE TABLE t (
+  c VARCHAR(20)
+) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
diff --git a/tests/highlight.html b/tests/highlight.html
@@ -423,3 +423,5 @@
   <span style="color: #333;">t</span><span >.</span><span style="color: #333;">else</span>
 <span style="font-weight:bold;">FROM</span> <span style="color: #333;">t</span>
 <span style="font-weight:bold;">WHERE</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">start</span> <span >=</span> <span style="color: #333;">t</span><span >.</span><span style="color: #333;">end</span></pre>
+---
+<pre style="color: black; background-color: white;"><span style="font-weight:bold;">CREATE</span> <span style="font-weight:bold;">TABLE</span> <span style="color: #333;">t</span> (<span style="color: #333;">c</span> <span style="font-weight:bold;">VARCHAR</span>(<span style="color: green;">20</span>)) <span style="font-weight:bold;">DEFAULT</span> <span style="font-weight:bold;">CHARACTER</span> <span style="font-weight:bold;">SET</span> <span style="color: #333;">utf8mb4</span> <span style="font-weight:bold;">ENGINE</span> <span >=</span> <span style="color: #333;">InnoDB</span></pre>
diff --git a/tests/sql.sql b/tests/sql.sql
@@ -423,3 +423,5 @@ SELECT
   t.else
 FROM t
 WHERE t.start = t.end
+---
+CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB
-Original file line number
+Diff line change
@@ Expand Up / @@ -423,3 +423,5 @@ SELECT @@
       t.else
     FROM t
     WHERE t.start = t.end
+    ---
+    CREATE TABLE t (c VARCHAR(20)) DEFAULT CHARACTER SET utf8mb4 ENGINE = InnoDB