From 2a0baeb6af2cd4b696de08e254860f7b152c3e77 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Tue, 9 Jul 2024 08:29:03 +0200 Subject: [PATCH 01/27] Conditionals support full python expressions Fixes a bug where 'not' isn't highlighted properly --- syntaxes/renpy.tmLanguage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index a1cbcb4..37d914c 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -884,7 +884,7 @@ "match": "(?<=^[ \\t]+)(if|elif|else)\\b(.*?)(:)", "captures": { "1": { "name": "keyword.control.conditional.renpy" }, - "2": { "patterns": [{ "include": "#simple-expression" }] }, + "2": { "patterns": [{ "include": "source.renpy.python#expression" }] }, "3": { "name": "punctuation.section.condition.begin.renpy" } } }, From 3e3b7b14eb5ab39b8310868739890debb9e02447 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 09:26:07 +0200 Subject: [PATCH 02/27] Fix crash on syntax error --- src/tokenizer/tokenizer.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tokenizer/tokenizer.ts b/src/tokenizer/tokenizer.ts index 5ddfe69..40050c8 100644 --- a/src/tokenizer/tokenizer.ts +++ b/src/tokenizer/tokenizer.ts @@ -307,6 +307,10 @@ class DocumentTokenizer { * @param caret The reader head position within the document */ private applyCaptures(captures: TokenPatternCapture, match: RegExpExecArray, source: string, parentNode: TreeNode) { + if (match.indices === undefined) { + return; // syntax error + } + let rootNode = parentNode; if (captures[0] !== undefined) { From 8d01b41d456812d3914ba287168a8798d88fd276 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:58:09 +0200 Subject: [PATCH 03/27] Add contributing tips --- Contributing.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Contributing.md b/Contributing.md index 7352adc..3be1b3e 100644 --- a/Contributing.md +++ b/Contributing.md @@ -32,6 +32,28 @@ If you want to add new features, please make sure to discuss it in an issue. 5. Make your changes and submit a pull request with `develop` as your target branch 6. Happy coding! 🚀 +Tips: + +If you're working on syntax features, add the following textmate rule to your vscode `settings.json` file: + +```json +"editor.tokenColorCustomizations": { + "textMateRules": [ + { + "scope": "debug.invalid.illegal.unmatched.renpy", + "settings": { + "foreground": "#f00" + } + }, +} +``` + +This will make any unmatched tokens red. + +Use ctrl+alt+shift+i to display the vscode build-in token debug information. + +https://regexr.com will be your new best friend. + ## Additional Information ### Extension Versions From ff6145a984d2477670405b89db535ae3bddbf6dd Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:58:38 +0200 Subject: [PATCH 04/27] Add more edge cases in unit test --- examples/game/script.rpy | 2 +- examples/unit_test.rpy | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/examples/game/script.rpy b/examples/game/script.rpy index 5732dc3..f4e71ca 100644 --- a/examples/game/script.rpy +++ b/examples/game/script.rpy @@ -44,7 +44,7 @@ label start: # These display lines of dialogue. - e "You've created a new Ren'Py game." + character.e "You've created a new Ren'Py game." # call a label diff --git a/examples/unit_test.rpy b/examples/unit_test.rpy index 96d4cef..1d68dd4 100644 --- a/examples/unit_test.rpy +++ b/examples/unit_test.rpy @@ -326,6 +326,7 @@ call .subroutine(2) from test #lol + call screen start #lol call expression "sub" + "routine" pass (count=3) #lol call expression "sub" + "routine" pass (count=3) from _class test #lol @@ -547,11 +548,11 @@ show bg washington with None - show eileen happy at left + show eileen happy at left #test show lucy mad at right with dissolve - show eileen happy at left with dissolve + show eileen happy with dissolve at left show lucy mad at right with dissolve with None @@ -625,6 +626,8 @@ pass "Fly above." if drank_tea: pass + "Dig below." if len(my_var) >= 2: + pass default menuset = set() @@ -1566,6 +1569,19 @@ @renpy.atl_warper def linear(t): return t + + init python: + @dataclass + class LatLonPair: + lon: float, + lat: float + + @dataclass + class SimpleWeatherInfo: + id: int, + main: str, + description: str, + icon: str label start: show eileen happy at a, b, c From 5a91b13fbdba665e441f7a265964cd8b5bde1dbd Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:00:28 +0200 Subject: [PATCH 05/27] Image also supports `with` --- syntaxes/renpy.tmLanguage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index 37d914c..28ef425 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -976,7 +976,7 @@ "beginCaptures": { "1": { "name": "keyword.image.renpy" } }, - "end": "(?=\\b(at)\\b|#|=)|$", + "end": "(?=\\b(at|with)\\b|#|=|:)|$", "patterns": [ { "include": "#strings" }, { From a57a738d7a3bcf510916e7ae57d99d3070d1daa2 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:02:39 +0200 Subject: [PATCH 06/27] Add missing keywords in screen language --- syntaxes/renpy.screen.tmLanguage.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/syntaxes/renpy.screen.tmLanguage.json b/syntaxes/renpy.screen.tmLanguage.json index 42dda6c..dff514c 100644 --- a/syntaxes/renpy.screen.tmLanguage.json +++ b/syntaxes/renpy.screen.tmLanguage.json @@ -39,7 +39,7 @@ }, { "comment": "Special manipulation keywords", - "match": "\\b(? Date: Wed, 10 Jul 2024 11:03:16 +0200 Subject: [PATCH 07/27] Fix screen atl missing the variable matcher --- syntaxes/renpy.screen.tmLanguage.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/syntaxes/renpy.screen.tmLanguage.json b/syntaxes/renpy.screen.tmLanguage.json index dff514c..8ded516 100644 --- a/syntaxes/renpy.screen.tmLanguage.json +++ b/syntaxes/renpy.screen.tmLanguage.json @@ -158,7 +158,13 @@ { "include": "source.renpy.python#special-variables" }, { "include": "source.renpy.python#ellipsis" }, { "include": "source.renpy.python#punctuation" }, - { "include": "source.renpy.python#line-continuation" } + { "include": "source.renpy.python#line-continuation" }, + { + "comment": "Tokenize identifiers", + "name": "variable.name.python", + "match": "\\b([[:alpha:]_]\\w*)\\b" + }, + { "include": "source.renpy#whitespace" } ] }, "screen-keywords": { From 9c335bd119b62b2a8280e9d6012f12c8ccf46ae7 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:10:09 +0200 Subject: [PATCH 08/27] Fix at,as,behind,onlayer,with not working if more then one is used --- syntaxes/renpy.tmLanguage.json | 116 +++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index 28ef425..132a099 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -891,69 +891,87 @@ "at-statement": { "name": "meta.at.statement.renpy", - "match": "\\b(?!\\.)(at)\\b(.+)?", - "captures": { - "1": { "name": "keyword.control.flow.at.renpy" }, - "2": { - "name": "meta.at.parameters.renpy", - "patterns": [{ "include": "#simple-expression" }] - } - } + "contentName": "meta.at.parameters.renpy", + "begin": "\\b(?!\\.)(at)\\b", + "beginCaptures": { + "1": { "name": "keyword.control.flow.at.renpy" } + }, + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|=|:)|$", + + "patterns": [ + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] }, "as-statement": { "name": "meta.as.statement.renpy", - "match": "\\b(?!\\.)(as)\\b[ \\t]*(.+)?", - "captures": { - "1": { "name": "keyword.control.flow.as.renpy" }, - "2": { - "name": "meta.as.parameters.renpy", - "patterns": [{ "include": "#simple-expression" }] - } - } + "contentName": "meta.as.parameters.renpy", + "begin": "\\b(?!\\.)(as)\\b", + "beginCaptures": { + "1": { "name": "keyword.control.flow.as.renpy" } + }, + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|=|:)|$", + + "patterns": [ + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] }, "with-statement": { "name": "meta.with.statement.renpy", - "match": "\\b(?!\\.)(with)\\b(.+)?", - "captures": { - "1": { "name": "keyword.control.flow.with.renpy" }, - "2": { - "name": "meta.with.parameters.renpy", - "patterns": [{ "include": "#simple-expression" }] - } - } + "contentName": "meta.with.parameters.renpy", + "begin": "\\b(?!\\.)(with)\\b", + "beginCaptures": { + "1": { "name": "keyword.control.flow.with.renpy" } + }, + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|=|:)|$", + + "patterns": [ + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] }, "behind": { "name": "meta.behind.statement.renpy", - "match": "\\b(?!\\.)(behind)\\b(.+)?", - "captures": { - "1": { "name": "keyword.control.flow.behind.renpy" }, - "2": { - "name": "meta.behind.parameters.renpy", - "patterns": [{ "include": "#simple-expression" }] - } - } + "contentName": "meta.behind.parameters.renpy", + "begin": "\\b(?!\\.)(behind)\\b", + "beginCaptures": { + "1": { "name": "keyword.control.flow.behind.renpy" } + }, + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|=|:)|$", + + "patterns": [ + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] }, "onlayer": { "name": "meta.onlayer.statement.renpy", - "match": "\\b(?!\\.)(onlayer)\\b(.+)?", - "captures": { - "1": { "name": "keyword.control.flow.onlayer.renpy" }, - "2": { - "name": "meta.onlayer.parameters.renpy", - "patterns": [{ "include": "#simple-expression" }] - } - } + "contentName": "meta.onlayer.parameters.renpy", + "begin": "\\b(?!\\.)(onlayer)\\b", + "beginCaptures": { + "1": { "name": "keyword.control.flow.onlayer.renpy" } + }, + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|=|:)|$", + + "patterns": [ + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] }, "zorder": { "name": "meta.zorder.statement.renpy", - "match": "\\b(?!\\.)(zorder)\\b(.+)?", - "captures": { - "1": { "name": "keyword.control.flow.zorder.renpy" }, - "2": { - "name": "meta.zorder.parameters.renpy", - "patterns": [{ "include": "#simple-expression" }] - } - } + "contentName": "meta.zorder.parameters.renpy", + "begin": "\\b(?!\\.)(zorder)\\b", + "beginCaptures": { + "1": { "name": "keyword.control.flow.zorder.renpy" } + }, + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|=|:)|$", + + "patterns": [ + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] }, "image": { "patterns": [ @@ -1057,8 +1075,6 @@ } ] }, - { "include": "#at-statement" }, - { "include": "#as-statement" }, { "include": "#with-statement" }, { "include": "#behind" }, { "include": "#onlayer" }, From 90b72165a809dcda3769bb344d1f95ef3390c509 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:14:48 +0200 Subject: [PATCH 09/27] Add hide statement --- syntaxes/renpy.tmLanguage.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index 132a099..06ec227 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -1045,6 +1045,28 @@ { "include": "#zorder" } ] }, + "hide": { + "patterns": [ + { + "name": "meta.hide.statement.renpy", + "begin": "(?<=^[ \\t]*)(hide)\\b[ \\t]*", + "beginCaptures": { + "1": { "name": "keyword.hide.renpy" } + }, + "end": "(?=\\b(behind|expression|with|zorder)\\b|#)|$", + "patterns": [ + { "include": "#strings" }, + { + "match": "\\b(?:[a-zA-Z_0-9]+)\\b[ \\t]*", + "name": "entity.name.type.image.renpy" + } + ] + }, + { "include": "#with-statement" }, + { "include": "#behind" }, + { "include": "#zorder" } + ] + }, "scene": { "patterns": [ { @@ -1510,6 +1532,7 @@ { "include": "#scene" }, { "include": "#camera" }, { "include": "#show" }, + { "include": "#hide" }, { "include": "#with-statement" }, { "include": "#use-old" }, { "include": "source.renpy.screen#screen" }, From b102be754a896593e9d5fb1b87f8eb991bcfbe33 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:27:00 +0200 Subject: [PATCH 10/27] Add window show/hide --- syntaxes/renpy.tmLanguage.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index 06ec227..225d4af 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -1045,6 +1045,30 @@ { "include": "#zorder" } ] }, + "window": { + "patterns": [ + { + "name": "meta.show.statement.renpy", + "begin": "(?<=^[ \\t]*)(window)", + "beginCaptures": { + "1": { "name": "keyword.window.renpy" } + }, + "end": "#|$", + "patterns": [ + { + "name": "keyword.show.renpy", + "match": "(show)" + }, + { + "name": "keyword.hide.renpy", + "match": "(hide)" + }, + { "include": "#whitespace" }, + { "include": "#simple-expression" } + ] + } + ] + }, "hide": { "patterns": [ { @@ -1531,6 +1555,7 @@ { "include": "source.renpy.atl#transform" }, { "include": "#scene" }, { "include": "#camera" }, + { "include": "#window" }, { "include": "#show" }, { "include": "#hide" }, { "include": "#with-statement" }, From 16e55c010a32391e634bac19e578d5c2374cb1b3 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:59:40 +0200 Subject: [PATCH 11/27] Add block testers to simplify the code --- syntaxes/renpy.atl.tmLanguage.json | 76 ++++++++++++++----------- syntaxes/renpy.screen.tmLanguage.json | 62 +++++++++++--------- syntaxes/renpy.tmLanguage.json | 82 ++++++++------------------- 3 files changed, 101 insertions(+), 119 deletions(-) diff --git a/syntaxes/renpy.atl.tmLanguage.json b/syntaxes/renpy.atl.tmLanguage.json index c6fceb3..2644dfd 100644 --- a/syntaxes/renpy.atl.tmLanguage.json +++ b/syntaxes/renpy.atl.tmLanguage.json @@ -285,25 +285,26 @@ }, "atl-on": { "comment": "https://www.renpy.org/doc/html/atl.html#on-statement", - "contentName": "meta.atl.on.renpy", - "begin": "^([ \\t]+)?(on)\\b[ \\t]*(.+)?(:)", + + "name": "meta.atl.on.statement.renpy", + "contentName": "meta.atl.on.parameters.renpy", + "begin": "(?<=^[ \\t]*)(on)\\b[ \\t]*", "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.block.renpy" }, - "2": { "name": "keyword.on.renpy" }, - "3": { - "patterns": [ - { "include": "#atl-event-name" }, - { - "match": "[ \\t]*,[ \\t]*", - "name": "punctuation.separator.parameters.renpy" - }, - { "include": "source.renpy#comments" } - ] - }, - "4": { "name": "punctuation.section.block.begin.renpy" } + "1": { "name": "keyword.on.renpy" } }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#atl" }] + "end": "(?=#|:)|$", + + "patterns": [ + { "include": "#atl-event-name" }, + { + "match": "[ \\t]*,[ \\t]*", + "name": "punctuation.separator.parameters.renpy" + }, + { + "comment": "TODO: Is this really needed here? I think we need fallback patterns instead", + "include": "source.renpy#comments" + } + ] }, "atl-function": { "comment": "https://www.renpy.org/doc/html/atl.html#function-statement", @@ -365,24 +366,35 @@ "patterns": [{ "include": "source.renpy#base-patterns" }] }, + "atl-block-tester": { + "patterns": [ + { + "contentName": "meta.atl-block.renpy", + "begin": "(?<=(^[ \\t]*)(?:camera|image|show|scene|transform|on)\\b.*?)(:)", + "beginCaptures": { + "2": { "name": "punctuation.section.atl.begin.renpy" } + }, + "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", + "patterns": [{ "include": "source.renpy.atl" }] + } + ] + }, + "transform": { - "contentName": "meta.transform.block.renpy", - "begin": "^([ \\t]+)?(transform)\\b[ \\t]*(.*)?(:)", + "name": "meta.transform.statement.renpy", + "contentName": "meta.transform.parameters.renpy", + "begin": "(?<=^[ \\t]*)(transform)\\b[ \\t]*", "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.renpy" }, - "2": { "name": "keyword.transform.renpy" }, - "3": { - "patterns": [ - { - "name": "variable.name.transform.renpy", - "match": "\\b([[:alpha:]_]\\w*)\\b" - } - ] - }, - "4": { "name": "punctuation.section.atl.begin.renpy" } + "1": { "name": "keyword.transform.renpy" } }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#atl" }] + "end": "(?=#|:)|$", + + "patterns": [ + { + "name": "variable.name.transform.renpy", + "match": "\\b([[:alpha:]_]\\w*)\\b" + } + ] } } } diff --git a/syntaxes/renpy.screen.tmLanguage.json b/syntaxes/renpy.screen.tmLanguage.json index 8ded516..8dda7c8 100644 --- a/syntaxes/renpy.screen.tmLanguage.json +++ b/syntaxes/renpy.screen.tmLanguage.json @@ -193,27 +193,33 @@ } ] }, + "screen-block-tester": { + "patterns": [ + { + "contentName": "meta.screen-block.renpy", + "begin": "(?<=(^[ \\t]*)(?:frame|window|text|vbox|hbox)\\b.*?)(:)", + "beginCaptures": { + "2": { "name": "punctuation.section.screen.begin.renpy" } + }, + "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", + "patterns": [{ "include": "#screen-language" }] + } + ] + }, + "screen-frame": { - "contentName": "meta.screen.frame.renpy", - "begin": "^([ \\t]+)?(frame)\\b[ \\t]*(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.block.renpy" }, - "2": { "name": "keyword.frame.renpy" }, - "3": { "name": "punctuation.section.block.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#screen-language" }] + "name": "meta.screen.frame.statement.renpy", + "match": "(?<=^[ \\t]*)(frame)\\b", + "captures": { + "1": { "name": "keyword.frame.renpy" } + } }, "screen-window": { - "contentName": "meta.screen.window.renpy", - "begin": "^([ \\t]+)?(window)\\b[ \\t]*(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.block.renpy" }, - "2": { "name": "keyword.window.renpy" }, - "3": { "name": "punctuation.section.block.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#screen-language" }] + "name": "meta.screen.window.statement.renpy", + "match": "(?<=^[ \\t]*)(window)\\b", + "captures": { + "1": { "name": "keyword.window.renpy" } + } }, "screen-text": { "contentName": "meta.screen.text.renpy", @@ -258,16 +264,15 @@ ] }, "screen-box": { - "contentName": "meta.box.block.renpy", - "begin": "^([ \\t]+)?(?:(vbox)|(hbox))\\b[ \\t]*(?:.*?)(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.block.renpy" }, - "2": { "name": "keyword.vbox.renpy" }, - "3": { "name": "keyword.hbox.renpy" }, - "4": { "name": "punctuation.section.block.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#screen-language" }] + "patterns": [ + { + "name": "meta.screen.$1.statement.renpy", + "match": "(?<=^[ \\t]*)(vbox|hbox)\\b", + "captures": { + "1": { "name": "keyword.$1.renpy" } + } + } + ] }, "screen": { "patterns": [ @@ -298,6 +303,7 @@ "comment": "https://www.renpy.org/doc/html/screens.html#screen-language", "patterns": [ { "include": "source.renpy#conditionals" }, + { "include": "#screen-block-tester" }, { "include": "#screen-frame" }, { "include": "#screen-window" }, { "include": "#screen-box" }, diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index 225d4af..ecd36fb 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -150,7 +150,7 @@ } }, - "parenthesised-python": { + "parenthesized-python": { "patterns": [ { "begin": "\\(", @@ -217,7 +217,7 @@ { "include": "source.renpy.python#member-access" }, { "include": "source.renpy.python#function-call" }, { "include": "#identifier" }, - { "include": "#parenthesised-python" }, + { "include": "#parenthesized-python" }, { "include": "#whitespace" } ] }, @@ -742,7 +742,22 @@ }, { "include": "#define" }, { "include": "#default-statement" }, - { "include": "#one-line-python" } + { "include": "#one-line-python" }, + { "include": "#python-block-tester" } + ] + }, + "python-block-tester": { + "patterns": [ + { + "comment": "Check for assignments that support python expressions", + "contentName": "meta.embedded.expression.python", + "begin": "(?<=(^[ \\t]*)(?:image)\\b.*?)(=)", + "beginCaptures": { + "2": { "name": "keyword.operator.assignment.renpy" } + }, + "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", + "patterns": [{ "include": "source.renpy.python" }] + } ] }, @@ -975,19 +990,6 @@ }, "image": { "patterns": [ - { - "name": "meta.image.statement.renpy", - "contentName": "meta.atl-block.renpy", - "begin": "^([ \\t]+)?(image)\\b[ \\t]*([a-zA-Z_0-9 ]*)(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.renpy" }, - "2": { "name": "keyword.image.renpy" }, - "3": { "name": "entity.name.type.image.renpy" }, - "4": { "name": "punctuation.section.atl.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "source.renpy.atl" }] - }, { "name": "meta.image.statement.renpy", "begin": "(?<=^[ \\t]*)(image)\\b[ \\t]*", @@ -1009,26 +1011,13 @@ }, "show": { "patterns": [ - { - "name": "meta.show.statement.renpy", - "contentName": "meta.atl-block.renpy", - "begin": "^([ \\t]+)?(show)\\b[ \\t]*([a-zA-Z_0-9 ]*)(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.renpy" }, - "2": { "name": "keyword.show.renpy" }, - "3": { "name": "entity.name.type.image.renpy" }, - "4": { "name": "punctuation.section.atl.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "source.renpy.atl" }] - }, { "name": "meta.show.statement.renpy", "begin": "(?<=^[ \\t]*)(show)\\b[ \\t]*", "beginCaptures": { "1": { "name": "keyword.show.renpy" } }, - "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#)|$", + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|:)|$", "patterns": [ { "include": "#strings" }, { @@ -1093,26 +1082,13 @@ }, "scene": { "patterns": [ - { - "name": "meta.scene.statement.renpy", - "contentName": "meta.atl-block.renpy", - "begin": "^([ \\t]+)?(scene)\\b[ \\t]*([a-zA-Z_0-9 ]*)(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.renpy" }, - "2": { "name": "keyword.scene.renpy" }, - "3": { "name": "entity.name.type.image.renpy" }, - "4": { "name": "punctuation.section.atl.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "source.renpy.atl" }] - }, { "name": "meta.scene.statement.renpy", "begin": "(?<=^[ \\t]*)(scene)\\b[ \\t]*", "beginCaptures": { "1": { "name": "keyword.scene.renpy" } }, - "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#)|$", + "end": "(?=\\b(at|as|behind|onlayer|expression|with|zorder)\\b|#|:)|$", "patterns": [ { "include": "#strings" }, { @@ -1129,26 +1105,13 @@ }, "camera": { "patterns": [ - { - "name": "meta.camera.statement.renpy", - "contentName": "meta.atl-block.renpy", - "begin": "^([ \\t]+)?(camera)\\b[ \\t]*(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.renpy" }, - "2": { "name": "keyword.camera.renpy" }, - "3": { "name": "entity.name.type.image.renpy" }, - "4": { "name": "punctuation.section.atl.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "source.renpy.atl" }] - }, { "name": "meta.camera.statement.renpy", "begin": "(?<=^[ \\t]*)(camera)\\b[ \\t]*", "beginCaptures": { "1": { "name": "keyword.camera.renpy" } }, - "end": "(?=\\b(at|with)\\b|#)|$", + "end": "(?=\\b(at|with)\\b|#|:)|$", "patterns": [ { "match": "\\b(?:[a-zA-Z_0-9]+)\\b[ \\t]*", @@ -1160,7 +1123,7 @@ { "include": "#with-statement" } ] }, - + "style-property": { "patterns": [ { @@ -1553,6 +1516,7 @@ { "include": "#pause" }, { "include": "#style" }, { "include": "source.renpy.atl#transform" }, + { "include": "source.renpy.atl#atl-block-tester" }, { "include": "#scene" }, { "include": "#camera" }, { "include": "#window" }, From 0eabab798752211ce252367e76f4a67dc2dbe808 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 12:12:51 +0200 Subject: [PATCH 12/27] Fix ATL on statement being tested too late --- syntaxes/renpy.atl.tmLanguage.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syntaxes/renpy.atl.tmLanguage.json b/syntaxes/renpy.atl.tmLanguage.json index 2644dfd..72bd4f8 100644 --- a/syntaxes/renpy.atl.tmLanguage.json +++ b/syntaxes/renpy.atl.tmLanguage.json @@ -12,9 +12,9 @@ { "include": "#atl-keywords" }, { "include": "#atl-warper" }, { "include": "#atl-blocks" }, + { "include": "#atl-on" }, { "include": "#atl-simple-expression" }, { "include": "#atl-event" }, - { "include": "#atl-on" }, { "include": "#atl-function" }, { "include": "#atl-fallback" } ] From 39fb78cf6e93f90a7ded5c035ab1443b6169d217 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:14:12 +0200 Subject: [PATCH 13/27] Remove some useless whitespace testing --- syntaxes/renpy.tmLanguage.json | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index ecd36fb..3f97d0a 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -706,9 +706,8 @@ { "comment": "Renpy python block", "contentName": "meta.embedded.block.python", - "begin": "^([ \\t]+)?(?:(init)(?:[ \\t]+(-)?(\\d+))?[ \\t]+)?(python)[ \\t]*(.*)?(:)", + "begin": "(?<=(^[ \\t]*))(?:(init)(?:[ \\t]+(-)?(\\d+))?[ \\t]+)?(python)[ \\t]*(.*)?(:)", "beginCaptures": { - "1": { "name": "punctuation.whitespace.embedded.leading.renpy" }, "2": { "name": "keyword.init.renpy" }, "3": { "name": "keyword.operator.arithmetic.minus.renpy" }, "4": { "name": "constant.numeric.integer.renpy" }, @@ -763,7 +762,7 @@ "define": { "comment": "Match begin and end of python one line statements", - "begin": "(?<=^[ \\t]*)(define)(?=[ \\t])", + "begin": "(?<=^[ \\t]*)(define)\\b", "beginCaptures": { "1": { "name": "keyword.define.renpy" } }, @@ -782,7 +781,7 @@ }, "default-statement": { "comment": "Match begin and end of python one line statements", - "begin": "(?<=^[ \\t]*)(default)(?=[ \\t])", + "begin": "(?<=^[ \\t]*)(default)\\b", "beginCaptures": { "1": { "name": "keyword.default.renpy" } }, @@ -800,7 +799,7 @@ }, "one-line-python": { "comment": "Match begin and end of python one line statements", - "begin": "(?<=^[ \\t]*)(\\$)(?=[ \\t])", + "begin": "(?<=^[ \\t]*)(\\$)(?=[ \\t]|$)", "beginCaptures": { "1": { "name": "keyword.dollar.sign.renpy" } }, @@ -1453,15 +1452,20 @@ "patterns": [ { "name": "meta.play.audio.statement.renpy", - "begin": "(?<=^[ \\t]*)(play)\\b[ \\t]+\\b([a-zA-Z_0-9]*)\\b[ \\t]*", + "begin": "(?<=^[ \\t]*)(play)\\b[ \\t]+\\b([a-zA-Z_0-9]*)\\b", "beginCaptures": { "1": { "name": "keyword.play.renpy" }, "2": { "patterns": [{ "include": "#audio-channel" }] } }, - "end": "(?=[ \\t]*#)|$", - "patterns": [{ "include": "#strings" }, { "include": "#audio-params" }, { "include": "source.renpy.python#expression" }] + "end": "(?=#)|$", + "patterns": [ + { "include": "#strings" }, + { "include": "#audio-params" }, + { "include": "source.renpy.python#expression" }, + { "include": "#whitespace" } + ] } ] }, @@ -1469,15 +1473,20 @@ "patterns": [ { "name": "meta.queue.audio.statement.renpy", - "begin": "(?<=^[ \\t]*)(queue)\\b[ \\t]+\\b([a-zA-Z_0-9]*)\\b[ \\t]*", + "begin": "(?<=^[ \\t]*)(queue)\\b[ \\t]+\\b([a-zA-Z_0-9]*)\\b", "beginCaptures": { "1": { "name": "keyword.queue.renpy" }, "2": { "patterns": [{ "include": "#audio-channel" }] } }, - "end": "(?=[ \\t]*#)|$", - "patterns": [{ "include": "#strings" }, { "include": "#audio-params" }, { "include": "source.renpy.python#expression" }] + "end": "(?=#)|$", + "patterns": [ + { "include": "#strings" }, + { "include": "#audio-params" }, + { "include": "source.renpy.python#expression" }, + { "include": "#whitespace" } + ] } ] }, @@ -1485,20 +1494,21 @@ "patterns": [ { "name": "meta.stop.audio.statement.renpy", - "begin": "(?<=^[ \\t]*)(stop)\\b[ \\t]+\\b([a-zA-Z_0-9]*)\\b[ \\t]*", + "begin": "(?<=^[ \\t]*)(stop)\\b[ \\t]+\\b([a-zA-Z_0-9]*)\\b", "beginCaptures": { "1": { "name": "keyword.stop.renpy" }, "2": { "patterns": [{ "include": "#audio-channel" }] } }, - "end": "(?=[ \\t]*#)|$", + "end": "(?=#)|$", "patterns": [ { "name": "keyword.fadeout.renpy", "match": "\\b(? Date: Wed, 10 Jul 2024 13:15:08 +0200 Subject: [PATCH 14/27] Merge atl blocks into statements patterns --- syntaxes/renpy.atl.tmLanguage.json | 66 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/syntaxes/renpy.atl.tmLanguage.json b/syntaxes/renpy.atl.tmLanguage.json index 72bd4f8..584cf51 100644 --- a/syntaxes/renpy.atl.tmLanguage.json +++ b/syntaxes/renpy.atl.tmLanguage.json @@ -11,8 +11,7 @@ { "include": "source.renpy#conditionals" }, { "include": "#atl-keywords" }, { "include": "#atl-warper" }, - { "include": "#atl-blocks" }, - { "include": "#atl-on" }, + { "include": "#atl-statements" }, { "include": "#atl-simple-expression" }, { "include": "#atl-event" }, { "include": "#atl-function" }, @@ -218,37 +217,6 @@ } ] }, - "atl-blocks": { - "patterns": [ - { - "comment": "See https://www.renpy.org/doc/html/atl.html", - "contentName": "meta.atl.block.renpy", - "begin": "^([ \\t]+)?(?:(block)|(parallel)|(contains))\\b[ \\t]*(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.block.renpy" }, - "2": { "name": "keyword.block.renpy" }, - "3": { "name": "keyword.parallel.renpy" }, - "4": { "name": "keyword.contains.renpy" }, - "5": { "name": "punctuation.section.block.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#atl" }] - }, - { - "comment": "https://www.renpy.org/doc/html/atl.html#choice-statement", - "contentName": "meta.atl.choice.block.renpy", - "begin": "^([ \\t]+)?(choice)\\b[ \\t]*(.+)?(:)", - "beginCaptures": { - "1": { "name": "punctuation.whitespace.leading.block.renpy" }, - "2": { "name": "keyword.choice.renpy" }, - "3": { "patterns": [{ "include": "#atl-simple-expression" }] }, - "4": { "name": "punctuation.section.block.begin.renpy" } - }, - "end": "^(?=(?!\\1)[ \\t]*[^\\s#]|\\1[^\\s#])", - "patterns": [{ "include": "#atl" }] - } - ] - }, "atl-build-in-events": { "comment": "Pre-defined events (https://www.renpy.org/doc/html/atl.html#external-events)", "name": "support.function.event.renpy", @@ -366,11 +334,41 @@ "patterns": [{ "include": "source.renpy#base-patterns" }] }, + "atl-choice": { + "comment": "https://www.renpy.org/doc/html/atl.html#choice-statement", + "name": "meta.atl.choice.statement.renpy", + "contentName": "meta.atl.choice.parameters.renpy", + "begin": "(?<=^[ \\t]*)(choice)\\b[ \\t]*", + "beginCaptures": { + "1": { "name": "keyword.choice.renpy" } + }, + "end": "(?=#|:)|$", + + "patterns": [ + { "include": "#atl-simple-expression" } + ] + }, + + "atl-statements":{ + "patterns": [ + { + "comment": "Statements without parameters", + "name": "meta.atl.$2.statement.renpy", + "match": "(?<=^[ \\t]*)(block|parallel|contains)\\b[ \\t]*", + "captures": { + "1": { "name": "keyword.$2.renpy" } + } + }, + { "include": "#atl-choice" }, + { "include": "#atl-on" } + ] + }, + "atl-block-tester": { "patterns": [ { "contentName": "meta.atl-block.renpy", - "begin": "(?<=(^[ \\t]*)(?:camera|image|show|scene|transform|on)\\b.*?)(:)", + "begin": "(?<=(^[ \\t]*)(?:camera|image|show|scene|transform|on|block|parallel|contains|choice)\\b.*?)(:)", "beginCaptures": { "2": { "name": "punctuation.section.atl.begin.renpy" } }, From e071ae0cb762c0d117ff5e4ec0f309ad0ecf93ac Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Sat, 10 Jun 2023 23:20:01 +0200 Subject: [PATCH 15/27] Refactor Stack to be based on Vector --- src/utilities/stack.ts | 80 ++++++++++++----------------------------- src/utilities/vector.ts | 18 ++++++++++ 2 files changed, 41 insertions(+), 57 deletions(-) diff --git a/src/utilities/stack.ts b/src/utilities/stack.ts index 7a0847d..88e6959 100644 --- a/src/utilities/stack.ts +++ b/src/utilities/stack.ts @@ -1,17 +1,23 @@ -export class Stack implements Iterable { - private buffer: Array = []; - - private headPtr = -1; - private itemCount = 0; +import { Vector } from "./vector"; - private headRef: T | null = null; +export class Stack implements Iterable { + private buffer: Vector; /** * Construct a new empty stack * @param capacity The amount of items this stack should be able to hold */ constructor(capacity = 0) { - if (capacity > 0) this.buffer = new Array(capacity).fill(null); + this.buffer = new Vector(capacity); + } + + /** + * Create a copy of the current Stack so we can later restart from there. + */ + public clone(): Stack { + const newStack = new Stack(this.capacity); + newStack.buffer = this.buffer.clone(); + return newStack; } [Symbol.iterator](): Iterator { @@ -21,7 +27,7 @@ export class Stack implements Iterable { if (index >= this.size) { return { done: true, value: null }; } - return { done: false, value: this.buffer[index++] as T }; + return { done: false, value: this.buffer.at(index++) as T }; }, }; } @@ -31,13 +37,7 @@ export class Stack implements Iterable { * @param item The item to add */ public push(item: T) { - if (this.size === this.capacity) this._grow(); - - ++this.itemCount; - ++this.headPtr; - - this.buffer[this.headPtr] = item; - this.headRef = this.buffer[this.headPtr]; + this.buffer.pushBack(item); } /** @@ -45,21 +45,11 @@ export class Stack implements Iterable { * @returns The item at the top */ public pop() { - if (this.headPtr === -1) throw new RangeError("Stack was already empty. Can't pop another item."); - - const result = this.buffer[this.headPtr]; - - this.buffer[this.headPtr] = null; - --this.itemCount; - --this.headPtr; - - if (this.isEmpty()) { - this.headRef = null; - } else { - this.headRef = this.buffer[this.headPtr]; + if (this.buffer.isEmpty()) { + throw new RangeError("Stack was already empty. Can't pop another item."); } - return result; + return this.buffer.popBack(); } /** @@ -67,7 +57,7 @@ export class Stack implements Iterable { * @returns The item at the top */ public peek() { - return this.headRef; + return this.buffer.back(); } /** @@ -75,7 +65,7 @@ export class Stack implements Iterable { * @returns The amount items that could fit in the internal memory buffer */ get capacity() { - return this.buffer.length; + return this.buffer.capacity; } /** @@ -83,7 +73,7 @@ export class Stack implements Iterable { * @returns The number of items */ get size() { - return this.itemCount; + return this.buffer.size; } /** @@ -91,7 +81,7 @@ export class Stack implements Iterable { * @returns True if the stack currently contains no items */ public isEmpty() { - return this.itemCount === 0; + return this.buffer.isEmpty(); } /** @@ -99,30 +89,6 @@ export class Stack implements Iterable { * @param shrink If true it will also shrink the internal memory buffer to zero */ public clear(shrink = false) { - for (let i = 0; i < this.itemCount; ++i) { - this.buffer[i] = null; - } - - this.itemCount = 0; - this.headPtr = -1; - this.headRef = null; - - if (shrink) this.shrink(); - } - - /** - * Shrink the internal memory buffer to match the size of the stack - */ - public shrink() { - this.buffer.length = this.size; - } - - /** - * Grow the buffer to double the current capacity - */ - private _grow() { - const currentCapacity = Math.max(this.capacity, 1); - - this.buffer = this.buffer.concat(new Array(currentCapacity).fill(null)); + this.buffer.clear(shrink); } } diff --git a/src/utilities/vector.ts b/src/utilities/vector.ts index 7182e8e..ad7d0c7 100644 --- a/src/utilities/vector.ts +++ b/src/utilities/vector.ts @@ -18,6 +18,16 @@ export class Vector implements Iterable { if (capacity > 0) this.buffer = new Array(capacity).fill(null); } + public clone(): Vector { + const newVector = new Vector(this.capacity); + newVector.buffer = this.buffer.slice(); + newVector.headPtr = this.headPtr; + newVector.itemCount = this.itemCount; + newVector.tailRef = this.tailRef; + newVector.headRef = this.headRef; + return newVector; + } + [Symbol.iterator](): Iterator { let index = 0; return { @@ -116,6 +126,14 @@ export class Vector implements Iterable { this.eraseAt(index); } + /** + * Checks if the vector contains the specified item + * @param item The item to check for + */ + public contains(item: T) { + return this.indexOf(item) !== -1; + } + /** * Remove the item at the specified index * @param index The index of the item to remove From 40308bea1cefbdc9a8c058693ed0fce7986afbed Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:43:05 +0200 Subject: [PATCH 16/27] Allow unused variables --- .eslintrc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.eslintrc.json b/.eslintrc.json index dd1a49f..6d386dd 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -25,6 +25,7 @@ "@typescript-eslint/no-var-requires": "warn", "no-param-reassign": "warn", // Disable reassign, since this basically means you override the reference from the caller function with a new local version. (It doesn't do what you expect) "@typescript-eslint/no-namespace": "off", - "@typescript-eslint/ban-ts-comment": "off" + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/no-unused-vars": "off" } } From 32970d00f5854cee9337a44196f6307e1fcc3e54 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:49:01 +0200 Subject: [PATCH 17/27] Rename `tokenType` to `type` --- src/color.ts | 2 +- src/tokenizer/debug-decorator.ts | 6 +++--- src/tokenizer/token-definitions.ts | 22 +++++++++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/color.ts b/src/color.ts index a8f3427..ff8c19d 100644 --- a/src/color.ts +++ b/src/color.ts @@ -133,7 +133,7 @@ export async function injectCustomColorStyles(document: TextDocument) { // Disabled until filter is added to the tree class const documentTokens = await Tokenizer.tokenizeDocument(document); // TODO: Should probably make sure this constant is actually part of a tag, but for now this is fine. - const colorTags = documentTokens.filter((x) => x.token?.tokenType === LiteralTokenType.Color); + const colorTags = documentTokens.filter((x) => x.token?.type === LiteralTokenType.Color); const colorRules = new ValueEqualsSet(); // Build the new rules for this file diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts index 6a95a49..a590159 100644 --- a/src/tokenizer/debug-decorator.ts +++ b/src/tokenizer/debug-decorator.ts @@ -242,7 +242,7 @@ async function updateDecorations() { range: range, hoverMessage: { language: "text", - value: `Token: ${tokenTypeToStringMap[token.tokenType]}(id: ${token.tokenType}) + value: `Token: ${tokenTypeToStringMap[token.type]}(id: ${token.type}) Start: {Line: ${range.start.line + 1}, Char: ${range.start.character + 1}} End: {Line: ${range.end.line + 1}, Char: ${range.end.character + 1}} Content: {${content?.replaceAll("\n", "\\n")}}`, @@ -285,7 +285,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` ); } - switch (token.tokenType) { + switch (token.type) { case KeywordTokenType.Init: // Python statement keywords case KeywordTokenType.Offset: case KeywordTokenType.Python: @@ -639,7 +639,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` break; default: - throw new Error(`Unhandled token case: ${tokenTypeToStringMap[token.tokenType]}(id: ${token.tokenType})`); + throw new Error(`Unhandled token case: ${tokenTypeToStringMap[token.type]}(id: ${token.type})`); } }); diff --git a/src/tokenizer/token-definitions.ts b/src/tokenizer/token-definitions.ts index 1ac71cc..b9b710c 100644 --- a/src/tokenizer/token-definitions.ts +++ b/src/tokenizer/token-definitions.ts @@ -70,12 +70,12 @@ export class TokenPosition { } export class Token { - readonly tokenType: TokenType; + readonly type: TokenType; readonly startPos: TokenPosition; readonly endPos: TokenPosition; constructor(tokenType: TokenType, startPos: TokenPosition, endPos: TokenPosition) { - this.tokenType = tokenType; + this.type = tokenType; this.startPos = startPos; this.endPos = endPos; } @@ -96,39 +96,39 @@ export class Token { } public isKeyword() { - return this.tokenType >= TokenTypeIndex.KeywordStart && this.tokenType < TokenTypeIndex.EntityStart; + return this.type >= TokenTypeIndex.KeywordStart && this.type < TokenTypeIndex.EntityStart; } public isEntity() { - return this.tokenType >= TokenTypeIndex.EntityStart && this.tokenType < TokenTypeIndex.ConstantStart; + return this.type >= TokenTypeIndex.EntityStart && this.type < TokenTypeIndex.ConstantStart; } public isConstant() { - return this.tokenType >= TokenTypeIndex.ConstantStart && this.tokenType < TokenTypeIndex.OperatorsStart; + return this.type >= TokenTypeIndex.ConstantStart && this.type < TokenTypeIndex.OperatorsStart; } public isOperator() { - return this.tokenType >= TokenTypeIndex.OperatorsStart && this.tokenType < TokenTypeIndex.CharactersStart; + return this.type >= TokenTypeIndex.OperatorsStart && this.type < TokenTypeIndex.CharactersStart; } public isCharacter() { - return this.tokenType >= TokenTypeIndex.CharactersStart && this.tokenType < TokenTypeIndex.EscapedCharacterStart; + return this.type >= TokenTypeIndex.CharactersStart && this.type < TokenTypeIndex.EscapedCharacterStart; } public isEscapedCharacter() { - return this.tokenType >= TokenTypeIndex.EscapedCharacterStart && this.tokenType < TokenTypeIndex.MetaStart; + return this.type >= TokenTypeIndex.EscapedCharacterStart && this.type < TokenTypeIndex.MetaStart; } public isMetaToken() { - return this.tokenType >= TokenTypeIndex.MetaStart && this.tokenType < TokenTypeIndex.UnknownCharacterID; + return this.type >= TokenTypeIndex.MetaStart && this.type < TokenTypeIndex.UnknownCharacterID; } public isUnknownCharacter() { - return this.tokenType === CharacterTokenType.Unknown; + return this.type === CharacterTokenType.Unknown; } public isInvalid() { - return this.tokenType === MetaTokenType.Invalid; + return this.type === MetaTokenType.Invalid; } } From 54450209552523f57ff263a36599f987aea5c2a2 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:02:57 +0200 Subject: [PATCH 18/27] Update vector code --- src/utilities/vector.ts | 72 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/src/utilities/vector.ts b/src/utilities/vector.ts index ad7d0c7..3b60949 100644 --- a/src/utilities/vector.ts +++ b/src/utilities/vector.ts @@ -95,7 +95,9 @@ export class Vector implements Iterable { } public at(index: number): T { - if (index < 0 || index >= this.itemCount) throw new RangeError("Index out of range"); + if (index < 0 || index >= this.itemCount) { + throw new RangeError("Index out of range"); + } // Convert to T since we only added the null type to create a storage buffer return this.buffer[index] as T; } @@ -114,8 +116,24 @@ export class Vector implements Iterable { * @param index The index to insert the item at */ public insert(item: T, index: number) { - throw new Error("Not yet implemented"); + if (index < 0 || index >= this.itemCount) { + throw new RangeError("Index out of range"); + } + + if (this.size === this.capacity) { + this._grow(); + } + + // Move all items after the index to the right + for (let i = this.itemCount; i > index; --i) { + this.buffer[i] = this.buffer[i - 1]; + } + + this.buffer[index] = item; + ++this.itemCount; + ++this.headPtr; } + /** * Remove the first occurrence of the item from the vector, if it exists @@ -140,7 +158,9 @@ export class Vector implements Iterable { * @throws RangeError if the index is out of range */ public eraseAt(index: number) { - if (index < 0 || index >= this.itemCount) throw new RangeError("Index out of range"); + if (index < 0 || index >= this.itemCount) { + throw new RangeError("Index out of range"); + } // Move all items after the index to the left for (let i = index; i < this.itemCount - 1; ++i) { @@ -156,8 +176,12 @@ export class Vector implements Iterable { * @throws RangeError if either index is out of range */ public swapElements(elementIndexA: number, elementIndexB: number) { - if (elementIndexA < 0 || elementIndexA >= this.itemCount) throw new RangeError("ElementIndexA out of range"); - if (elementIndexB < 0 || elementIndexB >= this.itemCount) throw new RangeError("ElementIndexB out of range"); + if (elementIndexA < 0 || elementIndexA >= this.itemCount) { + throw new RangeError("ElementIndexA out of range"); + } + if (elementIndexB < 0 || elementIndexB >= this.itemCount) { + throw new RangeError("ElementIndexB out of range"); + } const temp = this.buffer[elementIndexB]; this.buffer[elementIndexB] = this.buffer[elementIndexA]; @@ -170,13 +194,34 @@ export class Vector implements Iterable { } } + public any(callback: (item: T) => boolean) { + for (let i = 0; i < this.itemCount; ++i) { + if (callback(this.buffer[i] as T)) { + return true; + } + } + return false; + } + + public map(callback: (item: T) => U) { + const result = new Vector(this.itemCount); + for (let i = 0; i < this.itemCount; ++i) { + result.pushBack(callback(this.buffer[i] as T)); + } + return result; + } + + + /** * Swap the item at the specified index with the item at the back of the vector * @param index The index of the item to swap to the back * @throws RangeError if the index is out of range */ public swapToBack(index: number) { - if (index < 0 || index >= this.itemCount) throw new RangeError("Index out of range"); + if (index < 0 || index >= this.itemCount) { + throw new RangeError("Index out of range"); + } const temp = this.back(); this.buffer[this.headPtr] = this.buffer[index]; @@ -190,7 +235,9 @@ export class Vector implements Iterable { */ public indexOf(item: T, fromIndex?: number) { const index = this.buffer.indexOf(item, fromIndex); - if (index >= this.itemCount) return -1; + if (index >= this.itemCount) { + return -1; + } return index; } @@ -228,6 +275,13 @@ export class Vector implements Iterable { return this.buffer.slice(0, this.itemCount) as T[]; } + /** + * Calls to string on each item in the vector and joins the results + */ + public toString() { + return `[${this.toArray().toString()}]`; + } + /** * Remove all items from the vector * @param shrink If true it will also shrink the internal memory buffer to zero @@ -242,7 +296,9 @@ export class Vector implements Iterable { this.headRef = null; this.tailRef = null; - if (shrink) this.shrink(); + if (shrink) { + this.shrink(); + } } /** From 82bad45d0ded8283cdaf252505a8944bdfe807c5 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:04:25 +0200 Subject: [PATCH 19/27] Token enum changes Rename `VariableName` to `Identifier` Rename `Period` to `Dot` Add `PythonExpression` --- src/tokenizer/atl-token-patterns.g.ts | 4 +-- src/tokenizer/debug-decorator.ts | 5 +-- src/tokenizer/python-token-patterns.g.ts | 40 ++++++++++++------------ src/tokenizer/renpy-token-patterns.g.ts | 18 +++++------ src/tokenizer/renpy-tokens.ts | 5 +-- src/tokenizer/screen-token-patterns.g.ts | 2 +- src/tokenizer/token-definitions.ts | 5 +-- syntax-to-token-pattern.py | 8 ++--- 8 files changed, 45 insertions(+), 42 deletions(-) diff --git a/src/tokenizer/atl-token-patterns.g.ts b/src/tokenizer/atl-token-patterns.g.ts index a5d1b6c..5e80841 100644 --- a/src/tokenizer/atl-token-patterns.g.ts +++ b/src/tokenizer/atl-token-patterns.g.ts @@ -161,7 +161,7 @@ export const atlSimpleExpression: TokenPattern = { debugName: "atlSimpleExpression.patterns![21]", // Tokenize identifiers - token: EntityTokenType.VariableName, /*variable.name.python*/ + token: EntityTokenType.Identifier, /*variable.name.python*/ match: /\b([a-zA-Z_]\w*)\b/g, }, ] @@ -437,7 +437,7 @@ export const transform: TokenPattern = { { debugName: "transform.beginCaptures![3].patterns![0]", - token: EntityTokenType.VariableName, /*variable.name.transform.renpy*/ + token: EntityTokenType.Identifier, /*variable.name.transform.renpy*/ match: /\b([a-zA-Z_]\w*)\b/g, }, ] diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts index a590159..c1bf830 100644 --- a/src/tokenizer/debug-decorator.ts +++ b/src/tokenizer/debug-decorator.ts @@ -395,7 +395,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` break; // Variables - case EntityTokenType.VariableName: + case EntityTokenType.Identifier: case EntityTokenType.StyleName: case EntityTokenType.ImageName: case EntityTokenType.TextName: @@ -430,6 +430,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` case MetaTokenType.CodeBlock: case MetaTokenType.PythonLine: case MetaTokenType.PythonBlock: + case MetaTokenType.PythonExpression: case MetaTokenType.Arguments: case MetaTokenType.EmptyString: case MetaTokenType.StringTag: @@ -593,7 +594,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` case CharacterTokenType.Whitespace: case CharacterTokenType.NewLine: - case CharacterTokenType.Period: + case CharacterTokenType.Dot: case CharacterTokenType.Colon: case CharacterTokenType.Semicolon: case CharacterTokenType.Comma: diff --git a/src/tokenizer/python-token-patterns.g.ts b/src/tokenizer/python-token-patterns.g.ts index 0d65382..ec5e445 100644 --- a/src/tokenizer/python-token-patterns.g.ts +++ b/src/tokenizer/python-token-patterns.g.ts @@ -259,7 +259,7 @@ export const expression: TokenPattern = { debugName: "expression.patterns![2]", // Tokenize identifiers to help linters - token: EntityTokenType.VariableName, /*variable.name.python*/ + token: EntityTokenType.Identifier, /*variable.name.python*/ match: /\b([a-zA-Z_]\w*)\b/g, }, ] @@ -269,10 +269,10 @@ export const memberAccess: TokenPattern = { debugName: "memberAccess", token: MetaTokenType.MemberAccess, /*meta.member.access.python*/ - contentToken: EntityTokenType.VariableName, /*variable.name.python*/ + contentToken: EntityTokenType.Identifier, /*variable.name.python*/ begin: /(\.)\s*(?!\.)/dg, beginCaptures: { - 1: { token: CharacterTokenType.Period, /*punctuation.separator.period.python*/ }, + 1: { token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ }, }, end: /(?<=\S)(?=\W)|(^|(?<=\s))(?=[^\\\w\s])|$/gm, patterns: [ @@ -674,7 +674,7 @@ export const importStatement: TokenPattern = { { debugName: "importStatement.patterns![0].patterns![0]", - token: CharacterTokenType.Period, /*punctuation.separator.period.python*/ + token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ match: /\.+/g, }, expression, @@ -776,7 +776,7 @@ export const classKwarg: TokenPattern = { match: /\b([a-zA-Z_]\w*)\s*(=)(?!=)/dg, captures: { - 1: { token: EntityTokenType.VariableName, /*entity.other.inherited-class.python variable.parameter.class.python*/ }, + 1: { token: EntityTokenType.Identifier, /*entity.other.inherited-class.python variable.parameter.class.python*/ }, 2: { token: OperatorTokenType.Assignment, /*keyword.operator.assignment.python*/ }, }, }; @@ -796,7 +796,7 @@ export const memberAccessClass: TokenPattern = { token: MetaTokenType.MemberAccess, /*meta.member.access.python*/ begin: /(\.)\s*(?!\.)/dg, beginCaptures: { - 1: { token: CharacterTokenType.Period, /*punctuation.separator.period.python*/ }, + 1: { token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ }, }, end: /(?<=\S)(?=\W)|$/gm, patterns: [ @@ -854,7 +854,7 @@ export const lambda: TokenPattern = { match: /([a-zA-Z_]\w*)\s*(?:(,)|(?=:|$))/dgm, captures: { - 1: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.python*/ }, + 1: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.python*/ }, 2: { token: CharacterTokenType.Comma, /*punctuation.separator.parameters.python*/ }, }, }, @@ -884,7 +884,7 @@ export const lambdaParameterWithDefault: TokenPattern = { begin: /\b([a-zA-Z_]\w*)\s*(=)/dg, beginCaptures: { - 1: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.python*/ }, + 1: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.python*/ }, 2: { token: MetaTokenType.Operator, /*keyword.operator.python*/ }, }, end: /(,)|(?=:|$)/dgm, @@ -967,7 +967,7 @@ export const parameters: TokenPattern = { match: /([a-zA-Z_]\w*)\s*(?:(,)|(?=[)#\n=]))/dg, captures: { - 1: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.python*/ }, + 1: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.python*/ }, 2: { token: CharacterTokenType.Comma, /*punctuation.separator.parameters.python*/ }, }, }, @@ -980,9 +980,9 @@ export const parameterSpecial: TokenPattern = { match: /\b((self)|(cls))\b\s*(?:(,)|(?=\)))/dg, captures: { - 1: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.python*/ }, - 2: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.special.self.python*/ }, - 3: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.special.cls.python*/ }, + 1: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.python*/ }, + 2: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.special.self.python*/ }, + 3: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.special.cls.python*/ }, 4: { token: CharacterTokenType.Comma, /*punctuation.separator.parameters.python*/ }, }, }; @@ -1006,7 +1006,7 @@ export const annotatedParameter: TokenPattern = { begin: /\b([a-zA-Z_]\w*)\s*(:)/dg, beginCaptures: { - 1: { token: EntityTokenType.VariableName, /*variable.parameter.function.language.python*/ }, + 1: { token: EntityTokenType.Identifier, /*variable.parameter.function.language.python*/ }, 2: { token: CharacterTokenType.Colon, /*punctuation.separator.annotation.python*/ }, }, end: /(,)|(?=\))/dg, @@ -1109,7 +1109,7 @@ export const decoratorName: TokenPattern = { token: EntityTokenType.FunctionName, /*entity.name.function.decorator.python*/ match: /([a-zA-Z_]\w*)|(\.)/dg, captures: { - 2: { token: CharacterTokenType.Period, /*punctuation.separator.period.python*/ }, + 2: { token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ }, }, }, lineContinuation, @@ -1202,7 +1202,7 @@ export const functionArguments: TokenPattern = { match: /\b([a-zA-Z_]\w*)\s*(=)(?!=)/dg, captures: { - 1: { token: EntityTokenType.VariableName, /*variable.parameter.function-call.python*/ }, + 1: { token: EntityTokenType.Identifier, /*variable.parameter.function-call.python*/ }, 2: { token: OperatorTokenType.Assignment, /*keyword.operator.assignment.python*/ }, }, }, @@ -1252,7 +1252,7 @@ export const builtinFunctions: TokenPattern = { { debugName: "builtinFunctions.patterns![1]", - token: EntityTokenType.VariableName, /*variable.legacy.builtin.python*/ + token: EntityTokenType.Identifier, /*variable.legacy.builtin.python*/ match: /(? = { NamespaceName: { name: "NamespaceName", value: EntityTokenType.NamespaceName }, FunctionName: { name: "FunctionName", value: EntityTokenType.FunctionName }, TagName: { name: "TagName", value: EntityTokenType.TagName }, - VariableName: { name: "VariableName", value: EntityTokenType.VariableName }, + Identifier: { name: "Identifier", value: EntityTokenType.Identifier }, StyleName: { name: "StyleName", value: EntityTokenType.StyleName }, ImageName: { name: "ImageName", value: EntityTokenType.ImageName }, @@ -409,7 +409,7 @@ const tokenTypeDefinitions: EnumToString = { Whitespace: { name: "Whitespace", value: CharacterTokenType.Whitespace }, NewLine: { name: "NewLine", value: CharacterTokenType.NewLine }, - Period: { name: "Period", value: CharacterTokenType.Period }, + Dot: { name: "Dot", value: CharacterTokenType.Dot }, Colon: { name: "Colon", value: CharacterTokenType.Colon }, Semicolon: { name: "Semicolon", value: CharacterTokenType.Semicolon }, Comma: { name: "Comma", value: CharacterTokenType.Comma }, @@ -457,6 +457,7 @@ const tokenTypeDefinitions: EnumToString = { CodeBlock: { name: "CodeBlock", value: MetaTokenType.CodeBlock }, PythonLine: { name: "PythonLine", value: MetaTokenType.PythonLine }, PythonBlock: { name: "PythonBlock", value: MetaTokenType.PythonBlock }, + PythonExpression: { name: "PythonExpression", value: MetaTokenType.PythonExpression }, Arguments: { name: "Arguments", value: MetaTokenType.Arguments }, EmptyString: { name: "EmptyString", value: MetaTokenType.EmptyString }, diff --git a/syntax-to-token-pattern.py b/syntax-to-token-pattern.py index c2ffce5..f2e9ddb 100644 --- a/syntax-to-token-pattern.py +++ b/syntax-to-token-pattern.py @@ -45,7 +45,7 @@ def get_parts(range: slice) -> str: return "LiteralTokenType.String" elif get_part(0) == "variable": - return "EntityTokenType.VariableName" + return "EntityTokenType.Identifier" elif get_part(0) == "storage": if get_part(1) == "type": @@ -219,7 +219,7 @@ def get_parts(range: slice) -> str: elif get_part(2) == "function": return "EntityTokenType.FunctionName" elif get_part(2) == "variable": - return "EntityTokenType.VariableName" + return "EntityTokenType.Identifier" elif get_part(2) == "namespace": return "EntityTokenType.NamespaceName" elif get_part(2) == "metaclass": @@ -230,7 +230,7 @@ def get_parts(range: slice) -> str: return "MetaTokenType.BuiltinType" elif get_part(1) == "variable": - return "EntityTokenType.VariableName" + return "EntityTokenType.Identifier" elif get_part(1) == "function": if get_part(2) == "event": @@ -241,7 +241,7 @@ def get_parts(range: slice) -> str: elif get_part(1) == "other": if get_part(2) == "match": if get_part(3) == "any": - return "CharacterTokenType.Period" + return "CharacterTokenType.Dot" elif get_part(3) == "begin": return "CharacterTokenType.Caret" elif get_part(3) == "end": From 5944957eec2426a15634bdaaa27856ef46b0b098 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:05:05 +0200 Subject: [PATCH 20/27] Add new API for Tokenizer --- src/tokenizer/token-definitions.ts | 184 ++++++++++++++++++++++++++++- 1 file changed, 182 insertions(+), 2 deletions(-) diff --git a/src/tokenizer/token-definitions.ts b/src/tokenizer/token-definitions.ts index 7a32cd9..cc86bd8 100644 --- a/src/tokenizer/token-definitions.ts +++ b/src/tokenizer/token-definitions.ts @@ -1,4 +1,5 @@ -import { LogLevel, Position, Range as VSRange } from "vscode"; +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { LogLevel, Position, TextDocument, Range as VSRange } from "vscode"; import { CharacterTokenType, EntityTokenType, EscapedCharacterTokenType, KeywordTokenType, LiteralTokenType, MetaTokenType, OperatorTokenType, TokenType, TokenTypeIndex, TypeOfTokenType } from "./renpy-tokens"; import { TokenPattern, TokenRangePattern, TokenMatchPattern, TokenRepoPattern } from "./token-pattern-types"; import { Vector } from "../utilities/vector"; @@ -67,10 +68,15 @@ export class TokenPosition { this.character = newValue.character; this.charStartOffset = newValue.charStartOffset; } + + public toString() { + return `L${this.line + 1}:C${this.character + 1}`; + } } export class Token { readonly type: TokenType; + readonly metaTokens: Vector; readonly startPos: TokenPosition; readonly endPos: TokenPosition; @@ -78,6 +84,7 @@ export class Token { this.type = tokenType; this.startPos = startPos; this.endPos = endPos; + this.metaTokens = new Vector(); } public getVSCodeRange() { @@ -128,7 +135,35 @@ export class Token { } public isInvalid() { - return this.type === MetaTokenType.Invalid; + return this.hasMetaToken(MetaTokenType.Invalid); + } + + public addMetaToken() { + this.metaTokens.pushBack(this.type); + } + + public removeMetaToken(metaToken: MetaTokenType) { + this.metaTokens.erase(metaToken); + } + + public hasMetaToken(metaToken: TokenType) { + return this.metaTokens.contains(metaToken); + } + + public getValue(document: TextDocument) { + return document.getText(this.getVSCodeRange()); + } + + public toString() { + let metaTokenString = ""; + + if (!this.metaTokens.isEmpty()) { + this.metaTokens.forEach((metaToken) => { + metaTokenString += `, ${tokenTypeToString(metaToken)}`; + }); + } + + return `${tokenTypeToString(this.type)}${metaTokenString}: (${this.startPos}) -> (${this.endPos})`; } } @@ -147,13 +182,16 @@ export function isRepoPattern(p: TokenPattern): p is TokenRepoPattern { export class TreeNode { public token: Token | null; public children: Vector; + public parent: TreeNode | null; constructor(token: Token | null = null) { this.token = token; this.children = new Vector(); + this.parent = null; } public addChild(child: TreeNode): void { + child.parent = this; this.children.pushBack(child); } @@ -191,6 +229,56 @@ export class TreeNode { }); return count; } + + /** + * Flatten the node by building a Vector of Token's. + * Each child token should be added to the vector, in the order they appear in the source code (aka. the token.startPos). + * The current token.type should be added to it's children, using the metaTokens field. + * We should ensure the entire range of the current token is covered. + * If it's not covered by all children, we should create a new token filling the gaps and assigning the current token type + */ + public flatten(): Vector { + const tokens = new Vector(); + + // Step 2: Recursively flatten each child node and add its tokens to the vector + this.children.forEach((child) => { + const childTokens = child.flatten(); + childTokens.forEach((token) => { + tokens.pushBack(token); + }); + }); + + // Sort the tokens by their start position + tokens.sort((a, b) => a.startPos.charStartOffset - b.startPos.charStartOffset); + + if (!this.token) { + return tokens; + } + + // Step 3: Add the current token's type to its children using the metaTokens field + tokens.forEach((token) => { + token.metaTokens.pushBack(this.token!.type); + }); + + // Step 4: Ensure that the entire range of the current token is covered by its children + let currentEnd = this.token.startPos; + for (const token of tokens) { + const start = token.startPos; + if (start.charStartOffset > currentEnd.charStartOffset) { + // There is a gap between the current end position and the start of the next token range + const gapToken = new Token(this.token.type, currentEnd, start); + tokens.pushBack(gapToken); + } + currentEnd = currentEnd.charStartOffset > token.endPos.charStartOffset ? currentEnd : token.endPos; + } + if (currentEnd.charStartOffset < this.token.endPos.charStartOffset) { + // The last token range does not extend to the end of the match + const gapToken = new Token(this.token.type, currentEnd, this.token.endPos); + tokens.pushBack(gapToken); + } + + return tokens; + } } export class TokenTree { @@ -215,6 +303,98 @@ export class TokenTree { public count(): number { return this.root.count(); } + + public getIterator(): TokenListIterator { + return new TokenListIterator(this.flatten()); + } + + public flatten(): Vector { + return this.root.flatten(); + } +} + +/** + * Special iterator type, where the iterator can be manipulated while iterating. + * This will allow us to advance based on conditions + */ +export class TokenListIterator { + private readonly _tokens: Vector; + private _index = 0; + private _blacklist: Set = new Set(); + + constructor(tokens: Vector) { + this._tokens = tokens; + } + + /** + * Advances the iterator to the next node that has a valid token that is not blacklisted + */ + public next() { + if (!this.hasNext()) { + throw new Error("next() was called on an iterator that has no more nodes to visit."); + } + + // Move to the next node + this._index++; + + // We should skip any nodes with blacklisted tokens + while (this.isBlacklisted() && this.hasNext()) { + this._index++; + } + } + + public clone() { + const newIterator = new TokenListIterator(this._tokens); + newIterator._index = this._index; + newIterator._blacklist = new Set(this._blacklist); + return newIterator; + } + + public get token() { + return this._tokens.at(this._index); + } + + public get tokenType() { + return this.token.type; + } + + public get metaTokens() { + return this.token.metaTokens; + } + + /** + * Check is the current token is blacklisted + */ + private isBlacklisted() { + if (this._blacklist.has(this.tokenType)) { + return true; + } + + return this.metaTokens.any((token) => { + return this._blacklist.has(token); + }); + } + + /** + * Add a filter to the iterator. This will prevent the iterator from visiting nodes that match the filter. + */ + public setFilter(blacklist: Set) { + this._blacklist = blacklist; + } + + /** + * Returns the current filter + */ + public getFilter() { + return this._blacklist; + } + + /** + * Returns true if there are more nodes to visit + */ + public hasNext() { + return this._index < this._tokens.size - 1; + } } const tokenTypeDefinitions: EnumToString = { From cdffb018732fbc9249591baf6b24099127554251 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:07:24 +0200 Subject: [PATCH 21/27] Import changes from parser branch --- syntaxes/renpy.tmLanguage.json | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/syntaxes/renpy.tmLanguage.json b/syntaxes/renpy.tmLanguage.json index 3f97d0a..6a7136e 100644 --- a/syntaxes/renpy.tmLanguage.json +++ b/syntaxes/renpy.tmLanguage.json @@ -761,44 +761,51 @@ }, "define": { - "comment": "Match begin and end of python one line statements", "begin": "(?<=^[ \\t]*)(define)\\b", "beginCaptures": { "1": { "name": "keyword.define.renpy" } }, "end": "$", "patterns": [ - { - "comment": "Type the first name as a variable (Probably not needed, but python doesn't seem to catch it)", - "match": "(? Date: Wed, 10 Jul 2024 15:27:41 +0200 Subject: [PATCH 22/27] Hide crashes during `Initializing Ren'Py static data` phase Attempts to fix #389 and others --- src/extension.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 860f277..b310ac2 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -64,11 +64,16 @@ export async function activate(context: ExtensionContext): Promise { if (!NavigationData.isImporting) { updateStatusBar("$(sync~spin) Initializing Ren'Py static data..."); - const uri = Uri.file(document.fileName); - const filename = stripWorkspaceFromFile(uri.path); - NavigationData.clearScannedDataForFile(filename); - NavigationData.scanDocumentForClasses(filename, document); - updateStatusBar(getStatusBarText()); + try { + const uri = Uri.file(document.fileName); + const filename = stripWorkspaceFromFile(uri.path); + NavigationData.clearScannedDataForFile(filename); + NavigationData.scanDocumentForClasses(filename, document); + updateStatusBar(getStatusBarText()); + } catch (error) { + updateStatusBar("Failed to load Ren'Py static data..."); + logMessage(LogLevel.Error, error as string); + } } }) ); @@ -201,8 +206,13 @@ export async function activate(context: ExtensionContext): Promise { // Detect file system change to the navigation.json file and trigger a refresh updateStatusBar("$(sync~spin) Initializing Ren'Py static data..."); - await NavigationData.init(context.extensionPath); - updateStatusBar(getStatusBarText()); + try { + await NavigationData.init(context.extensionPath); + updateStatusBar(getStatusBarText()); + } catch (error) { + updateStatusBar("Failed to load Ren'Py static data..."); + logMessage(LogLevel.Error, error as string); + } try { fs.watch(getNavigationJsonFilepath(), async (event, filename) => { From 03988dee8817dfd94fb44621e7f3031317bbdd41 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:28:55 +0200 Subject: [PATCH 23/27] Update generated token patterns and fixes for them --- src/tokenizer/atl-token-patterns.g.ts | 161 +++++++----- src/tokenizer/python-token-patterns.g.ts | 64 ++--- src/tokenizer/renpy-token-patterns.g.ts | 314 +++++++++++++---------- src/tokenizer/renpy-tokens.ts | 2 + src/tokenizer/screen-token-patterns.g.ts | 94 ++++--- src/tokenizer/token-definitions.ts | 2 + src/tokenizer/token-patterns.g.ts | 59 +++-- syntax-to-token-pattern.py | 6 +- syntaxes/renpy.atl.tmLanguage.json | 21 +- syntaxes/renpy.python.tmLanguage.json | 62 ++--- syntaxes/renpy.screen.tmLanguage.json | 15 +- syntaxes/renpy.tmLanguage.json | 8 +- 12 files changed, 463 insertions(+), 345 deletions(-) diff --git a/src/tokenizer/atl-token-patterns.g.ts b/src/tokenizer/atl-token-patterns.g.ts index 5e80841..820163d 100644 --- a/src/tokenizer/atl-token-patterns.g.ts +++ b/src/tokenizer/atl-token-patterns.g.ts @@ -6,7 +6,7 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 08/07/2024 17:35:30 (UTC+0) +// Last generated: 10/07/2024 12:47:55 (UTC+0) import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens"; import { TokenPattern } from "./token-pattern-types"; @@ -244,42 +244,6 @@ export const atlKeywords: TokenPattern = { ] }; -export const atlBlocks: TokenPattern = { - patterns: [ - { - debugName: "atlBlocks.patterns![0]", - - // See https://www.renpy.org/doc/html/atl.html - contentToken: MetaTokenType.ATLBlock, /*meta.atl.block.renpy*/ - begin: /^([ \t]+)?(?:(block)|(parallel)|(contains))\b[ \t]*(:)/dgm, - beginCaptures: { - 1: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.leading.block.renpy*/ }, - 2: { token: KeywordTokenType.Block, /*keyword.block.renpy*/ }, - 3: { token: KeywordTokenType.Parallel, /*keyword.parallel.renpy*/ }, - 4: { token: KeywordTokenType.Contains, /*keyword.contains.renpy*/ }, - 5: { token: CharacterTokenType.Colon, /*punctuation.section.block.begin.renpy*/ }, - }, - end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm, - patterns: [atl] - }, - { - debugName: "atlBlocks.patterns![1]", - - // https://www.renpy.org/doc/html/atl.html#choice-statement - contentToken: MetaTokenType.ATLChoiceBlock, /*meta.atl.choice.block.renpy*/ - begin: /^([ \t]+)?(choice)\b[ \t]*(.+)?(:)/dgm, - beginCaptures: { - 1: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.leading.block.renpy*/ }, - 2: { token: KeywordTokenType.Choice, /*keyword.choice.renpy*/ }, - 3: { patterns: [atlSimpleExpression] }, - 4: { token: CharacterTokenType.Colon, /*punctuation.section.block.begin.renpy*/ }, - }, - end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm, - patterns: [atl] - }, - ] -}; - export const atlBuildInEvents: TokenPattern = { debugName: "atlBuildInEvents", @@ -328,26 +292,22 @@ export const atlOn: TokenPattern = { debugName: "atlOn", // https://www.renpy.org/doc/html/atl.html#on-statement - contentToken: MetaTokenType.ATLOn, /*meta.atl.on.renpy*/ - begin: /^([ \t]+)?(on)\b[ \t]*(.+)?(:)/dgm, + token: MetaTokenType.ATLOnStatement, /*meta.atl.on.statement.renpy*/ + contentToken: MetaTokenType.ATLOnParameters, /*meta.atl.on.parameters.renpy*/ + begin: /(?<=^[ \t]*)(on)\b[ \t]*/dgm, beginCaptures: { - 1: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.leading.block.renpy*/ }, - 2: { token: KeywordTokenType.On, /*keyword.on.renpy*/ }, - 3: { - patterns: [ - atlEventName, - { - debugName: "atlOn.beginCaptures![3].patterns![1]", + 1: { token: KeywordTokenType.On, /*keyword.on.renpy*/ }, + }, + end: /(?=#|:)|$/gm, + patterns: [ + atlEventName, + { + debugName: "atlOn.patterns![1]", - token: CharacterTokenType.Comma, /*punctuation.separator.parameters.renpy*/ - match: /[ \t]*,[ \t]*/g, - }, - ] + token: CharacterTokenType.Comma, /*punctuation.separator.parameters.renpy*/ + match: /[ \t]*,[ \t]*/g, }, - 4: { token: CharacterTokenType.Colon, /*punctuation.section.block.begin.renpy*/ }, - }, - end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm, - patterns: [atl] + ] }; export const atlFunction: TokenPattern = { @@ -424,29 +384,88 @@ export const atlFallback: TokenPattern = { ] }; +export const atlChoice: TokenPattern = { + debugName: "atlChoice", + + // https://www.renpy.org/doc/html/atl.html#choice-statement + token: MetaTokenType.ATLChoiceStatement, /*meta.atl.choice.statement.renpy*/ + contentToken: MetaTokenType.ATLChoiceParameters, /*meta.atl.choice.parameters.renpy*/ + begin: /(?<=^[ \t]*)(choice)\b[ \t]*/dgm, + beginCaptures: { + 1: { token: KeywordTokenType.Choice, /*keyword.choice.renpy*/ }, + }, + end: /(?=#|:)|$/gm, + patterns: [atlSimpleExpression] +}; + +export const atlStatements: TokenPattern = { + patterns: [ + { + debugName: "atlStatements.patterns![0]", + + token: MetaTokenType.ATLBlockStatement, /*meta.atl.block.statement.renpy*/ + match: /(?<=^[ \t]*)(block)\b[ \t]*/dgm, + captures: { + 1: { token: KeywordTokenType.Block, /*keyword.block.renpy*/ }, + }, + }, + { + debugName: "atlStatements.patterns![1]", + + token: MetaTokenType.ATLParallelStatement, /*meta.atl.parallel.statement.renpy*/ + match: /(?<=^[ \t]*)(parallel)\b[ \t]*/dgm, + captures: { + 1: { token: KeywordTokenType.Parallel, /*keyword.parallel.renpy*/ }, + }, + }, + { + debugName: "atlStatements.patterns![2]", + + token: MetaTokenType.ATLContainsStatement, /*meta.atl.contains.statement.renpy*/ + match: /(?<=^[ \t]*)(contains)\b[ \t]*/dgm, + captures: { + 1: { token: KeywordTokenType.Contains, /*keyword.contains.renpy*/ }, + }, + }, + atlChoice, + atlOn, + ] +}; + +export const atlBlockTester: TokenPattern = { + patterns: [ + { + debugName: "atlBlockTester.patterns![0]", + + contentToken: MetaTokenType.ATLBlock, /*meta.atl-block.renpy*/ + begin: /(?<=(^[ \t]*)(?:camera|image|show|scene|transform|on|block|parallel|contains|choice)\b.*?)(:)/dgm, + beginCaptures: { + 2: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ }, + }, + end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm, + patterns: [] + }] +}; + export const transform: TokenPattern = { debugName: "transform", - contentToken: MetaTokenType.TransformBlock, /*meta.transform.block.renpy*/ - begin: /^([ \t]+)?(transform)\b[ \t]*(.*)?(:)/dgm, + token: MetaTokenType.TransformStatement, /*meta.transform.statement.renpy*/ + contentToken: MetaTokenType.TransformParameters, /*meta.transform.parameters.renpy*/ + begin: /(?<=^[ \t]*)(transform)\b[ \t]*/dgm, beginCaptures: { - 1: { token: CharacterTokenType.Whitespace, /*punctuation.whitespace.leading.renpy*/ }, - 2: { token: KeywordTokenType.Transform, /*keyword.transform.renpy*/ }, - 3: { - patterns: [ - { - debugName: "transform.beginCaptures![3].patterns![0]", + 1: { token: KeywordTokenType.Transform, /*keyword.transform.renpy*/ }, + }, + end: /(?=#|:)|$/gm, + patterns: [ + { + debugName: "transform.patterns![0]", - token: EntityTokenType.Identifier, /*variable.name.transform.renpy*/ - match: /\b([a-zA-Z_]\w*)\b/g, - }, - ] + token: EntityTokenType.Identifier, /*variable.name.transform.renpy*/ + match: /\b([a-zA-Z_]\w*)\b/g, }, - 4: { token: CharacterTokenType.Colon, /*punctuation.section.atl.begin.renpy*/ }, - }, - end: /^(?=(?!\1)[ \t]*[^\s#]|\1[^\s#])/gm, - patterns: [atl] + ] }; // Push pattern references that were not defined on include -atl.patterns!.push(atlKeywords, atlWarper, atlBlocks, atlSimpleExpression, atlEvent, atlOn, atlFunction, atlFallback); \ No newline at end of file +atl.patterns!.push(atlKeywords, atlWarper, atlStatements, atlSimpleExpression, atlEvent, atlFunction, atlFallback); \ No newline at end of file diff --git a/src/tokenizer/python-token-patterns.g.ts b/src/tokenizer/python-token-patterns.g.ts index ec5e445..8fc46a8 100644 --- a/src/tokenizer/python-token-patterns.g.ts +++ b/src/tokenizer/python-token-patterns.g.ts @@ -6,7 +6,7 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 08/07/2024 17:35:30 (UTC+0) +// Last generated: 10/07/2024 12:47:55 (UTC+0) import { MetaTokenType, KeywordTokenType, EntityTokenType, CharacterTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens"; import { TokenPattern } from "./token-pattern-types"; @@ -272,7 +272,7 @@ export const memberAccess: TokenPattern = { contentToken: EntityTokenType.Identifier, /*variable.name.python*/ begin: /(\.)\s*(?!\.)/dg, beginCaptures: { - 1: { token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ }, + 1: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ }, }, end: /(?<=\S)(?=\W)|(^|(?<=\s))(?=[^\\\w\s])|$/gm, patterns: [ @@ -674,7 +674,7 @@ export const importStatement: TokenPattern = { { debugName: "importStatement.patterns![0].patterns![0]", - token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ + token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ match: /\.+/g, }, expression, @@ -796,7 +796,7 @@ export const memberAccessClass: TokenPattern = { token: MetaTokenType.MemberAccess, /*meta.member.access.python*/ begin: /(\.)\s*(?!\.)/dg, beginCaptures: { - 1: { token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ }, + 1: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ }, }, end: /(?<=\S)(?=\W)|$/gm, patterns: [ @@ -1109,7 +1109,7 @@ export const decoratorName: TokenPattern = { token: EntityTokenType.FunctionName, /*entity.name.function.decorator.python*/ match: /([a-zA-Z_]\w*)|(\.)/dg, captures: { - 2: { token: CharacterTokenType.Dot, /*punctuation.separator.period.python*/ }, + 2: { token: CharacterTokenType.Dot, /*punctuation.separator.dot.python*/ }, }, }, lineContinuation, @@ -2384,12 +2384,12 @@ export const regexpSingleOneLine: TokenPattern = { debugName: "regexpSingleOneLine", token: LiteralTokenType.String, /*string.regexp.quoted.single.python*/ - begin: /\b(([uU]r)|([bB]r)|(r[bB]?))(\')/dg, + begin: /\b(?:([uU]r)|([bB]r)|(r[bB]?))(\')/dg, beginCaptures: { - 2: { token: MetaTokenType.Deprecated, /*invalid.deprecated.prefix.python*/ }, + 1: { token: MetaTokenType.Deprecated, /*invalid.deprecated.prefix.python*/ }, + 2: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ }, 3: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ }, - 4: { token: MetaTokenType.StringStorageType, /*storage.type.string.python*/ }, - 5: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ }, + 4: { token: MetaTokenType.StringBegin, /*punctuation.definition.string.begin.python*/ }, }, end: /(\')|(? = { ScreenWindow: { name: "ScreenWindow", value: MetaTokenType.ScreenWindow }, ScreenText: { name: "ScreenText", value: MetaTokenType.ScreenText }, ScreenBlock: { name: "ScreenBlock", value: MetaTokenType.ScreenBlock }, + ScreenVboxStatement: { name: "ScreenVboxStatement", value: MetaTokenType.ScreenVboxStatement }, + ScreenHboxStatement: { name: "ScreenHboxStatement", value: MetaTokenType.ScreenHboxStatement }, StyleStatement: { name: "StyleStatement", value: MetaTokenType.StyleStatement }, StyleBlock: { name: "StyleBlock", value: MetaTokenType.StyleBlock }, diff --git a/src/tokenizer/token-patterns.g.ts b/src/tokenizer/token-patterns.g.ts index 5014c74..18e65fd 100644 --- a/src/tokenizer/token-patterns.g.ts +++ b/src/tokenizer/token-patterns.g.ts @@ -4,33 +4,33 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 08/07/2024 17:35:30 (UTC+0) +// Last generated: 10/07/2024 12:47:55 (UTC+0) -import * as ScreenPatterns from "./screen-token-patterns.g"; import * as RenpyPatterns from "./renpy-token-patterns.g"; -import * as AtlPatterns from "./atl-token-patterns.g"; import * as PythonPatterns from "./python-token-patterns.g"; +import * as AtlPatterns from "./atl-token-patterns.g"; +import * as ScreenPatterns from "./screen-token-patterns.g"; // Push all RenpyPatterns external includes RenpyPatterns.literal.patterns!.push(PythonPatterns.literal); -RenpyPatterns.parenthesisedPython.patterns![0].patterns!.push(PythonPatterns.expression); -RenpyPatterns.parenthesisedPython.patterns![1].patterns!.push(PythonPatterns.expression); -RenpyPatterns.parenthesisedPython.patterns![2].patterns!.push(PythonPatterns.expression); +RenpyPatterns.parenthesizedPython.patterns![0].patterns!.push(PythonPatterns.expression); +RenpyPatterns.parenthesizedPython.patterns![1].patterns!.push(PythonPatterns.expression); +RenpyPatterns.parenthesizedPython.patterns![2].patterns!.push(PythonPatterns.expression); RenpyPatterns.simpleExpression.patterns!.splice(0, 0, PythonPatterns.string); RenpyPatterns.simpleExpression.patterns!.splice(3, 0, PythonPatterns.memberAccess); RenpyPatterns.simpleExpression.patterns!.splice(4, 0, PythonPatterns.functionCall); RenpyPatterns.keywords.patterns![1].patterns!.push(PythonPatterns.expression); RenpyPatterns.constantPlaceholder.captures![2].patterns!.push(PythonPatterns.expression); RenpyPatterns.pythonStatements.patterns![1].patterns!.push(PythonPatterns.python); -RenpyPatterns.define.patterns!.splice(3, 0, PythonPatterns.expression); -RenpyPatterns.defaultStatement.patterns!.splice(2, 0, PythonPatterns.expression); +RenpyPatterns.pythonBlockTester.patterns![0].patterns!.push(PythonPatterns.python); +RenpyPatterns.define.patterns!.splice(2, 0, PythonPatterns.expression); +RenpyPatterns.define.patterns![3].patterns!.splice(0, 0, PythonPatterns.expression); +RenpyPatterns.defaultStatement.patterns!.splice(1, 0, PythonPatterns.expression); +RenpyPatterns.defaultStatement.patterns![2].patterns!.splice(0, 0, PythonPatterns.expression); RenpyPatterns.oneLinePython.patterns!.splice(2, 0, PythonPatterns.expression); RenpyPatterns.sayStatements.patterns![0].endCaptures![3].patterns!.push(PythonPatterns.functionArguments); RenpyPatterns.sayStatements.patterns![1].endCaptures![3].patterns!.push(PythonPatterns.functionArguments); -RenpyPatterns.image.patterns![0].patterns!.push(AtlPatterns.atl); -RenpyPatterns.show.patterns![0].patterns!.push(AtlPatterns.atl); -RenpyPatterns.scene.patterns![0].patterns!.push(AtlPatterns.atl); -RenpyPatterns.camera.patterns![0].patterns!.push(AtlPatterns.atl); +RenpyPatterns.conditionals.captures![2].patterns!.push(PythonPatterns.expression); RenpyPatterns.labelName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); RenpyPatterns.labelCall.patterns!.splice(0, 0, PythonPatterns.specialVariables); RenpyPatterns.labelCall.patterns!.push(PythonPatterns.functionArguments); @@ -44,11 +44,12 @@ RenpyPatterns.menuOption.beginCaptures![3].patterns![1].captures![2].patterns!.p RenpyPatterns.menuSet.captures![2].patterns!.push(PythonPatterns.expressionBare); RenpyPatterns.menu.beginCaptures![3].patterns!.push(PythonPatterns.functionArguments); RenpyPatterns.audioParams.patterns!.push(PythonPatterns.number); -RenpyPatterns.play.patterns![0].patterns!.push(PythonPatterns.expression); -RenpyPatterns.queue.patterns![0].patterns!.push(PythonPatterns.expression); -RenpyPatterns.stop.patterns![0].patterns!.push(PythonPatterns.number); +RenpyPatterns.play.patterns![0].patterns!.splice(2, 0, PythonPatterns.expression); +RenpyPatterns.queue.patterns![0].patterns!.splice(2, 0, PythonPatterns.expression); +RenpyPatterns.stop.patterns![0].patterns!.splice(1, 0, PythonPatterns.number); RenpyPatterns.renpyStatements.patterns!.splice(6, 0, AtlPatterns.transform); -RenpyPatterns.renpyStatements.patterns!.splice(12, 0, ScreenPatterns.screen); +RenpyPatterns.renpyStatements.patterns!.splice(7, 0, AtlPatterns.atlBlockTester); +RenpyPatterns.renpyStatements.patterns!.splice(15, 0, ScreenPatterns.screen); // Push all AtlPatterns external includes AtlPatterns.atl.patterns!.splice(0, 0, RenpyPatterns.conditionals); @@ -77,15 +78,35 @@ AtlPatterns.atlExpression.patterns!.push(PythonPatterns.expression); AtlPatterns.atlKeywords.patterns!.push(RenpyPatterns.keywords); AtlPatterns.atlEventName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); AtlPatterns.atlEventDefName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); -AtlPatterns.atlOn.beginCaptures![3].patterns!.push(RenpyPatterns.comments); +AtlPatterns.atlOn.patterns!.push(RenpyPatterns.comments); AtlPatterns.atlFunction.captures![2].patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); AtlPatterns.atlWarperName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); AtlPatterns.atlFallback.patterns!.push(RenpyPatterns.basePatterns); +AtlPatterns.atlBlockTester.patterns![0].patterns!.push(AtlPatterns.atl); // Push all ScreenPatterns external includes ScreenPatterns.screenDefName.patterns!.splice(0, 0, PythonPatterns.builtinPossibleCallables); ScreenPatterns.screenSimpleExpression.patterns!.splice(0, 0, RenpyPatterns.renpyOnlyExpressions); -ScreenPatterns.screenSimpleExpression.patterns!.push(PythonPatterns.memberAccess, PythonPatterns.illegalOperator, PythonPatterns.operator, PythonPatterns.curlyBraces, PythonPatterns.itemAccess, PythonPatterns.list, PythonPatterns.oddFunctionCall, PythonPatterns.roundBraces, PythonPatterns.functionCall, PythonPatterns.builtinFunctions, PythonPatterns.builtinTypes, PythonPatterns.builtinExceptions, PythonPatterns.magicNames, PythonPatterns.specialNames, PythonPatterns.illegalNames, PythonPatterns.specialVariables, PythonPatterns.ellipsis, PythonPatterns.punctuation, PythonPatterns.lineContinuation); +ScreenPatterns.screenSimpleExpression.patterns!.splice(2, 0, PythonPatterns.memberAccess); +ScreenPatterns.screenSimpleExpression.patterns!.splice(3, 0, PythonPatterns.illegalOperator); +ScreenPatterns.screenSimpleExpression.patterns!.splice(4, 0, PythonPatterns.operator); +ScreenPatterns.screenSimpleExpression.patterns!.splice(5, 0, PythonPatterns.curlyBraces); +ScreenPatterns.screenSimpleExpression.patterns!.splice(6, 0, PythonPatterns.itemAccess); +ScreenPatterns.screenSimpleExpression.patterns!.splice(7, 0, PythonPatterns.list); +ScreenPatterns.screenSimpleExpression.patterns!.splice(8, 0, PythonPatterns.oddFunctionCall); +ScreenPatterns.screenSimpleExpression.patterns!.splice(9, 0, PythonPatterns.roundBraces); +ScreenPatterns.screenSimpleExpression.patterns!.splice(10, 0, PythonPatterns.functionCall); +ScreenPatterns.screenSimpleExpression.patterns!.splice(11, 0, PythonPatterns.builtinFunctions); +ScreenPatterns.screenSimpleExpression.patterns!.splice(12, 0, PythonPatterns.builtinTypes); +ScreenPatterns.screenSimpleExpression.patterns!.splice(13, 0, PythonPatterns.builtinExceptions); +ScreenPatterns.screenSimpleExpression.patterns!.splice(14, 0, PythonPatterns.magicNames); +ScreenPatterns.screenSimpleExpression.patterns!.splice(15, 0, PythonPatterns.specialNames); +ScreenPatterns.screenSimpleExpression.patterns!.splice(16, 0, PythonPatterns.illegalNames); +ScreenPatterns.screenSimpleExpression.patterns!.splice(17, 0, PythonPatterns.specialVariables); +ScreenPatterns.screenSimpleExpression.patterns!.splice(18, 0, PythonPatterns.ellipsis); +ScreenPatterns.screenSimpleExpression.patterns!.splice(19, 0, PythonPatterns.punctuation); +ScreenPatterns.screenSimpleExpression.patterns!.splice(20, 0, PythonPatterns.lineContinuation); +ScreenPatterns.screenSimpleExpression.patterns!.push(RenpyPatterns.whitespace); ScreenPatterns.screenText.patterns!.push(RenpyPatterns.atStatement); ScreenPatterns.screenText.patterns![0].patterns!.push(AtlPatterns.atl); ScreenPatterns.screenText.patterns![1].patterns!.splice(0, 0, RenpyPatterns.strings); @@ -97,4 +118,4 @@ ScreenPatterns.screenLanguage.patterns!.splice(0, 0, RenpyPatterns.conditionals) PythonPatterns.stringUnicodeGuts.patterns!.splice(0, 0, RenpyPatterns.stringsInterior); -export { ScreenPatterns, RenpyPatterns, AtlPatterns, PythonPatterns }; \ No newline at end of file +export { RenpyPatterns, PythonPatterns, AtlPatterns, ScreenPatterns }; \ No newline at end of file diff --git a/syntax-to-token-pattern.py b/syntax-to-token-pattern.py index f2e9ddb..8821a19 100644 --- a/syntax-to-token-pattern.py +++ b/syntax-to-token-pattern.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone import json import re from typing import Any @@ -566,7 +566,7 @@ def generate_file(state: GeneratorState, source_file: str, output_file: str): contents += "// THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR\n" contents += "// DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.\n" contents += "// ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.\n" - contents += f"// Last generated: {datetime.utcnow().strftime('%d/%m/%Y %H:%M:%S')} (UTC+0)\n" + contents += f"// Last generated: {datetime.now(timezone.utc).strftime('%d/%m/%Y %H:%M:%S')} (UTC+0)\n" contents += "\n" contents += f"import {{ {', '.join(state.used_token_types)} }} from \"./renpy-tokens\";\n" @@ -604,7 +604,7 @@ def generate_token_patterns(): contents += "// THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR\n" contents += "// DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.\n" contents += "// ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.\n" - contents += f"// Last generated: {datetime.utcnow().strftime('%d/%m/%Y %H:%M:%S')} (UTC+0)\n" + contents += f"// Last generated: {datetime.now(timezone.utc).strftime('%d/%m/%Y %H:%M:%S')} (UTC+0)\n" contents += "\n" # Add all source import from all states, but only the unique ones diff --git a/syntaxes/renpy.atl.tmLanguage.json b/syntaxes/renpy.atl.tmLanguage.json index 584cf51..89e3783 100644 --- a/syntaxes/renpy.atl.tmLanguage.json +++ b/syntaxes/renpy.atl.tmLanguage.json @@ -352,11 +352,24 @@ "atl-statements":{ "patterns": [ { - "comment": "Statements without parameters", - "name": "meta.atl.$2.statement.renpy", - "match": "(?<=^[ \\t]*)(block|parallel|contains)\\b[ \\t]*", + "name": "meta.atl.block.statement.renpy", + "match": "(?<=^[ \\t]*)(block)\\b[ \\t]*", "captures": { - "1": { "name": "keyword.$2.renpy" } + "1": { "name": "keyword.block.renpy" } + } + }, + { + "name": "meta.atl.parallel.statement.renpy", + "match": "(?<=^[ \\t]*)(parallel)\\b[ \\t]*", + "captures": { + "1": { "name": "keyword.parallel.renpy" } + } + }, + { + "name": "meta.atl.contains.statement.renpy", + "match": "(?<=^[ \\t]*)(contains)\\b[ \\t]*", + "captures": { + "1": { "name": "keyword.contains.renpy" } } }, { "include": "#atl-choice" }, diff --git a/syntaxes/renpy.python.tmLanguage.json b/syntaxes/renpy.python.tmLanguage.json index 1ee8af6..499909a 100644 --- a/syntaxes/renpy.python.tmLanguage.json +++ b/syntaxes/renpy.python.tmLanguage.json @@ -263,7 +263,7 @@ "contentName": "variable.name.python", "begin": "(\\.)\\s*(?!\\.)", "beginCaptures": { - "1": { "name": "punctuation.separator.period.python" } + "1": { "name": "punctuation.separator.dot.python" } }, "end": "(?<=\\S)(?=\\W)|(^|(?<=\\s))(?=[^\\\\\\w\\s])|$", "patterns": [{ "include": "#function-call" }, { "include": "#member-access-base" }, { "include": "#member-access-attribute" }] @@ -571,7 +571,7 @@ }, "patterns": [ { - "name": "punctuation.separator.period.python", + "name": "punctuation.separator.dot.python", "match": "\\.+" }, { "include": "#expression" } @@ -672,7 +672,7 @@ "begin": "(\\.)\\s*(?!\\.)", "end": "(?<=\\S)(?=\\W)|$", "beginCaptures": { - "1": { "name": "punctuation.separator.period.python" } + "1": { "name": "punctuation.separator.dot.python" } }, "patterns": [{ "include": "#call-wrapper-inheritance" }, { "include": "#member-access-base" }, { "include": "#inheritance-identifier" }] }, @@ -926,7 +926,7 @@ "name": "entity.name.function.decorator.python", "match": "([[:alpha:]_]\\w*)|(\\.)", "captures": { - "2": { "name": "punctuation.separator.period.python" } + "2": { "name": "punctuation.separator.dot.python" } } }, { "include": "#line-continuation" }, @@ -1905,13 +1905,13 @@ }, "regexp-single-one-line": { "name": "string.regexp.quoted.single.python", - "begin": "\\b(([uU]r)|([bB]r)|(r[bB]?))(\\')", + "begin": "\\b(?:([uU]r)|([bB]r)|(r[bB]?))(\\')", "end": "(\\')|(? Date: Wed, 10 Jul 2024 15:29:24 +0200 Subject: [PATCH 24/27] Ignore error for now Will need to update this file at some point anyway --- src/navigation-data.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/navigation-data.ts b/src/navigation-data.ts index f73bc29..dbb2669 100644 --- a/src/navigation-data.ts +++ b/src/navigation-data.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { commands, CompletionItem, CompletionItemKind, CompletionItemTag, LogLevel, MarkdownString, Position, SnippetString, TextDocument, window, workspace } from "vscode"; import { getDefinitionFromFile } from "./hover"; import { DataType, getBaseTypeFromDefine, getNamedParameter, getPyDocsFromTextDocumentAtLine, Navigation, splitParameters, stripQuotes } from "./navigation"; From 102fe3a5671163d82d2120457b90cc18549457ab Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:36:56 +0200 Subject: [PATCH 25/27] Fix some typos --- .vscode/settings.json | 10 +++++++++- src/extension.ts | 14 +++++++------- src/logger.ts | 2 +- src/{taskprovider.ts => task-provider.ts} | 0 4 files changed, 17 insertions(+), 9 deletions(-) rename src/{taskprovider.ts => task-provider.ts} (100%) diff --git a/.vscode/settings.json b/.vscode/settings.json index 1b09718..7fa792a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,5 +27,13 @@ "editor.defaultFormatter": "esbenp.prettier-vscode" }, - "eslint.validate": ["javascript", "typescript"] + "eslint.validate": [ + "javascript", + "typescript" + ], + "cSpell.words": [ + "Ren'Py", + "rpyc", + "Uncompiled" + ] } diff --git a/src/extension.ts b/src/extension.ts index b310ac2..43a050c 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -18,13 +18,13 @@ import { referencesProvider } from "./references"; import { registerDebugDecorator, unregisterDebugDecorator } from "./tokenizer/debug-decorator"; import { Tokenizer } from "./tokenizer/tokenizer"; import { signatureProvider } from "./signature"; -import { intializeLoggingSystems, logMessage, logToast, updateStatusBar } from "./logger"; +import { initializeLoggingSystems, logMessage, logToast, updateStatusBar } from "./logger"; import { Configuration } from "./configuration"; import { RenpyAdapterDescriptorFactory, RenpyConfigurationProvider } from "./debugger"; -import { RenpyTaskProvider } from "./taskprovider"; +import { RenpyTaskProvider } from "./task-provider"; export async function activate(context: ExtensionContext): Promise { - intializeLoggingSystems(context); + initializeLoggingSystems(context); updateStatusBar("$(sync~spin) Loading Ren'Py extension..."); Configuration.initialize(context); @@ -334,9 +334,9 @@ function RunWorkspaceFolder(): boolean { if (isValidExecutable(rpyPath)) { const renpyPath = cleanUpPath(Uri.file(rpyPath).path); const cwd = renpyPath.substring(0, renpyPath.lastIndexOf("/")); - const workfolder = getWorkspaceFolder(); - const args: string[] = [`${workfolder}`, "run"]; - if (workfolder.endsWith("/game")) { + const workFolder = getWorkspaceFolder(); + const args: string[] = [`${workFolder}`, "run"]; + if (workFolder.endsWith("/game")) { try { updateStatusBar("$(sync~spin) Running Ren'Py..."); const result = cp.spawnSync(rpyPath, args, { @@ -361,7 +361,7 @@ function RunWorkspaceFolder(): boolean { } return false; } else { - logMessage(LogLevel.Warning, "config for rennpy does not exist"); + logMessage(LogLevel.Warning, "config for renpy does not exist"); return false; } } diff --git a/src/logger.ts b/src/logger.ts index e7e15ab..c1a0f9e 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -6,7 +6,7 @@ const statusBar = window.createStatusBarItem(StatusBarAlignment.Right, 100); let extensionMode: ExtensionMode = null!; -export function intializeLoggingSystems(context: ExtensionContext) { +export function initializeLoggingSystems(context: ExtensionContext) { extensionMode = context.extensionMode; context.subscriptions.push(outputChannel); diff --git a/src/taskprovider.ts b/src/task-provider.ts similarity index 100% rename from src/taskprovider.ts rename to src/task-provider.ts From c5ff34c899397e06da0155567dd7c591d4950ff9 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:50:48 +0200 Subject: [PATCH 26/27] Fix `OddFunctionCall` meta token getting generated --- src/tokenizer/python-token-patterns.g.ts | 8 ++++---- syntaxes/renpy.python.tmLanguage.json | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tokenizer/python-token-patterns.g.ts b/src/tokenizer/python-token-patterns.g.ts index 67fa3b0..19d2218 100644 --- a/src/tokenizer/python-token-patterns.g.ts +++ b/src/tokenizer/python-token-patterns.g.ts @@ -6,7 +6,7 @@ // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT. // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED. -// Last generated: 10/07/2024 12:47:55 (UTC+0) +// Last generated: 10/07/2024 13:49:44 (UTC+0) import { MetaTokenType, KeywordTokenType, EntityTokenType, CharacterTokenType, OperatorTokenType, LiteralTokenType } from "./renpy-tokens"; import { TokenPattern } from "./token-pattern-types"; @@ -338,8 +338,8 @@ export const list: TokenPattern = { export const oddFunctionCall: TokenPattern = { debugName: "oddFunctionCall", - // A bit obscured function call where there may have been anarbitrary number of other operations to get the function.E.g. "arr[idx](args)" - token: MetaTokenType.FunctionCall, /*meta.odd-function-call.python*/ + // A bit obscured function call where there may have been an arbitrary number of other operations to get the function.E.g. "arr[idx](args)" + token: MetaTokenType.FunctionCall, /*meta.function-call.python*/ begin: /(?<=\]|\))\s*(?=\()/g, end: /(\))/dg, endCaptures: { @@ -3169,4 +3169,4 @@ fstringFnormQuotedMultiLine.patterns!.push(fstringMultiBrace, fstringMultiCore); fstringNormfQuotedMultiLine.patterns!.push(fstringMultiBrace, fstringMultiCore); fstringRawQuotedMultiLine.patterns!.push(fstringMultiBrace, fstringRawMultiCore); fstringMultiBrace.patterns!.splice(0, 0, fstringTerminatorMulti); -fstringTerminatorMulti.patterns!.push(fstringTerminatorMultiTail); +fstringTerminatorMulti.patterns!.push(fstringTerminatorMultiTail); \ No newline at end of file diff --git a/syntaxes/renpy.python.tmLanguage.json b/syntaxes/renpy.python.tmLanguage.json index 499909a..7e37867 100644 --- a/syntaxes/renpy.python.tmLanguage.json +++ b/syntaxes/renpy.python.tmLanguage.json @@ -309,8 +309,8 @@ "patterns": [{ "include": "#expression" }] }, "odd-function-call": { - "comment": "A bit obscured function call where there may have been anarbitrary number of other operations to get the function.E.g. \"arr[idx](args)\"", - "name": "meta.odd-function-call.python", + "comment": "A bit obscured function call where there may have been an arbitrary number of other operations to get the function.E.g. \"arr[idx](args)\"", + "name": "meta.function-call.python", "begin": "(?<=\\]|\\))\\s*(?=\\()", "end": "(\\))", "endCaptures": { From 7b9b08d38969134f0959ac9473ccd81903a5c647 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:56:11 +0200 Subject: [PATCH 27/27] Fix formatting issues --- src/completion.ts | 2 +- src/configuration.ts | 2 +- src/decorator.ts | 2 +- src/diagnostics.ts | 4 ++-- src/extension.ts | 8 ++++---- src/navigation-data.ts | 2 +- src/semantics.ts | 2 +- src/signature.ts | 2 +- src/tokenizer/debug-decorator.ts | 12 ++++++------ src/tokenizer/token-definitions.ts | 2 +- src/tokenizer/tokenizer.ts | 4 ++-- src/utilities/vector.ts | 3 --- 12 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/completion.ts b/src/completion.ts index 0f89047..c063934 100644 --- a/src/completion.ts +++ b/src/completion.ts @@ -22,7 +22,7 @@ export const completionProvider = languages.registerCompletionItemProvider( " ", "@", "-", - "(" + "(", ); /** diff --git a/src/configuration.ts b/src/configuration.ts index c053958..19dd0db 100644 --- a/src/configuration.ts +++ b/src/configuration.ts @@ -16,7 +16,7 @@ export class Configuration { this.excludeCompiledFilesConfig(); } } - }) + }), ); } diff --git a/src/decorator.ts b/src/decorator.ts index 1a6a09b..feee194 100644 --- a/src/decorator.ts +++ b/src/decorator.ts @@ -68,7 +68,7 @@ export function injectCustomTextmateTokens(rules: ValueEqualsSet) }, (reason) => { logMessage(LogLevel.Error, "Failed to update the tokenColorCustomizations config! : " + reason); - } + }, ); } } diff --git a/src/diagnostics.ts b/src/diagnostics.ts index f8417aa..2971649 100644 --- a/src/diagnostics.ts +++ b/src/diagnostics.ts @@ -129,7 +129,7 @@ export function refreshDiagnostics(doc: TextDocument, diagnosticCollection: Diag const diagnostic = new Diagnostic( range, `Inconsistent spacing detected (${indention} given, expected a multiple of ${firstIndentation}). Indentation must consist only of spaces in Ren'Py scripts. Each indentation level must consist of the same number of spaces. (4 spaces is strongly recommended.)`, - severity + severity, ); diagnostics.push(diagnostic); } @@ -175,7 +175,7 @@ export function subscribeToDocumentChanges(context: ExtensionContext, diagnostic if (editor) { refreshDiagnostics(editor.document, diagnostics); } - }) + }), ); context.subscriptions.push(workspace.onDidChangeTextDocument((e) => refreshDiagnostics(e.document, diagnostics))); diff --git a/src/extension.ts b/src/extension.ts index 43a050c..9843ef4 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -75,7 +75,7 @@ export async function activate(context: ExtensionContext): Promise { logMessage(LogLevel.Error, error as string); } } - }) + }), ); // custom command - refresh data @@ -166,7 +166,7 @@ export async function activate(context: ExtensionContext): Promise { request: "launch", program: rpyPath, }, - { noDebug: true } + { noDebug: true }, ); //call renpy @@ -280,8 +280,8 @@ export async function activate(context: ExtensionContext): Promise { ]; }, }, - DebugConfigurationProviderTriggerKind.Dynamic - ) + DebugConfigurationProviderTriggerKind.Dynamic, + ), ); const taskProvider = new RenpyTaskProvider(); diff --git a/src/navigation-data.ts b/src/navigation-data.ts index dbb2669..b22525a 100644 --- a/src/navigation-data.ts +++ b/src/navigation-data.ts @@ -664,7 +664,7 @@ export class NavigationData { 0, //location array[5], //documentation array[2], //args - array[4] //type + array[4], //type ); } diff --git a/src/semantics.ts b/src/semantics.ts index 10de5fe..ba42e96 100644 --- a/src/semantics.ts +++ b/src/semantics.ts @@ -26,7 +26,7 @@ export const semanticTokensProvider = languages.registerDocumentSemanticTokensPr }); }, }, - legend + legend, ); export function getSemanticTokens(document: TextDocument): SemanticTokens { diff --git a/src/signature.ts b/src/signature.ts index 1e6afd5..d6dd324 100644 --- a/src/signature.ts +++ b/src/signature.ts @@ -19,7 +19,7 @@ export const signatureProvider = languages.registerSignatureHelpProvider( }, "(", ",", - "=" + "=", ); /** diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts index c1bf830..b06dd0e 100644 --- a/src/tokenizer/debug-decorator.ts +++ b/src/tokenizer/debug-decorator.ts @@ -161,14 +161,14 @@ export async function registerDebugDecorator(context: ExtensionContext) { if (activeEditor && event.document === activeEditor.document) { await triggerUpdateDecorations(true); } - })) + })), ); // The active text editor was changed context.subscriptions.push( (activeEditorChangedEvent = window.onDidChangeActiveTextEditor(async () => { await triggerUpdateDecorations(); - })) + })), ); } @@ -256,7 +256,7 @@ Content: {${content?.replaceAll("\n", "\\n")}}`, logMessage( LogLevel.Error, `Start line number is incorrect!. Got: ${range.start.line + 1}, expected: ${start.line + 1}. On token: -${(decoration.hoverMessage as MarkdownString).value}` +${(decoration.hoverMessage as MarkdownString).value}`, ); } @@ -264,7 +264,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` logMessage( LogLevel.Error, `End line number is incorrect!. Got: ${range.end.line + 1}, expected: ${end.line + 1}. On token: -${(decoration.hoverMessage as MarkdownString).value}` +${(decoration.hoverMessage as MarkdownString).value}`, ); } @@ -273,7 +273,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` logMessage( LogLevel.Error, `Start char number is incorrect!. Got: ${range.start.character + 1}, expected: ${start.character + 1}. On token: -${(decoration.hoverMessage as MarkdownString).value}` +${(decoration.hoverMessage as MarkdownString).value}`, ); } @@ -281,7 +281,7 @@ ${(decoration.hoverMessage as MarkdownString).value}` logMessage( LogLevel.Error, `End char number is incorrect!. Got: ${range.end.character + 1}, expected: ${end.character + 1}. On token: -${(decoration.hoverMessage as MarkdownString).value}` +${(decoration.hoverMessage as MarkdownString).value}`, ); } diff --git a/src/tokenizer/token-definitions.ts b/src/tokenizer/token-definitions.ts index a688cd2..e1a5cab 100644 --- a/src/tokenizer/token-definitions.ts +++ b/src/tokenizer/token-definitions.ts @@ -229,7 +229,7 @@ export class TreeNode { }); return count; } - + /** * Flatten the node by building a Vector of Token's. * Each child token should be added to the vector, in the order they appear in the source code (aka. the token.startPos). diff --git a/src/tokenizer/tokenizer.ts b/src/tokenizer/tokenizer.ts index 40050c8..c3baab6 100644 --- a/src/tokenizer/tokenizer.ts +++ b/src/tokenizer/tokenizer.ts @@ -310,7 +310,7 @@ class DocumentTokenizer { if (match.indices === undefined) { return; // syntax error } - + let rootNode = parentNode; if (captures[0] !== undefined) { @@ -331,7 +331,7 @@ class DocumentTokenizer { logCatMessage( LogLevel.Debug, LogCategory.Tokenizer, - `There is no pattern defined for capture group '${i}', on a pattern that matched '${match[i]}' near L:${pos.line + 1} C:${pos.character + 1}.\nThis should probably be added or be a non-capturing group.` + `There is no pattern defined for capture group '${i}', on a pattern that matched '${match[i]}' near L:${pos.line + 1} C:${pos.character + 1}.\nThis should probably be added or be a non-capturing group.`, ); continue; } diff --git a/src/utilities/vector.ts b/src/utilities/vector.ts index 3b60949..4a8c809 100644 --- a/src/utilities/vector.ts +++ b/src/utilities/vector.ts @@ -133,7 +133,6 @@ export class Vector implements Iterable { ++this.itemCount; ++this.headPtr; } - /** * Remove the first occurrence of the item from the vector, if it exists @@ -211,8 +210,6 @@ export class Vector implements Iterable { return result; } - - /** * Swap the item at the specified index with the item at the back of the vector * @param index The index of the item to swap to the back