Merge pull request #423 from renpy/Fix-tokenizer-warnings

More bug fixes
renpy · Jul 12, 2024 · 914bcad · 914bcad
2 parents 1005afa + 43008f6
commit 914bcad
Show file tree

Hide file tree

Showing 18 changed files with 567 additions and 436 deletions.
diff --git a/src/extension.ts b/src/extension.ts
@@ -4,7 +4,7 @@
 
 import * as cp from "child_process";
 import * as fs from "fs";
-import { ExtensionContext, languages, commands, window, TextDocument, Position, debug, Range, workspace, Uri, DebugConfiguration, ProviderResult, DebugConfigurationProviderTriggerKind, tasks, LogLevel } from "vscode";
+import { ExtensionContext, languages, commands, window, TextDocument, Position, debug, Range, workspace, Uri, DebugConfiguration, ProviderResult, DebugConfigurationProviderTriggerKind, tasks, LogLevel, ExtensionMode } from "vscode";
 import { colorProvider } from "./color";
 import { getStatusBarText, NavigationData } from "./navigation-data";
 import { cleanUpPath, getAudioFolder, getImagesFolder, getNavigationJsonFilepath, getWorkspaceFolder, stripWorkspaceFromFile } from "./workspace";
@@ -23,7 +23,14 @@ import { Configuration } from "./configuration";
 import { RenpyAdapterDescriptorFactory, RenpyConfigurationProvider } from "./debugger";
 import { RenpyTaskProvider } from "./task-provider";
 
+let extensionMode: ExtensionMode = null!;
+
+export function isShippingBuild(): boolean {
+    return extensionMode !== ExtensionMode.Development;
+}
+
 export async function activate(context: ExtensionContext): Promise<void> {
+    extensionMode = context.extensionMode;
     initializeLoggingSystems(context);
     updateStatusBar("$(sync~spin) Loading Ren'Py extension...");
 

diff --git a/src/logger.ts b/src/logger.ts
@@ -1,14 +1,11 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
-import { ExtensionContext, StatusBarAlignment, window, LogLevel, ExtensionMode } from "vscode";
+import { ExtensionContext, StatusBarAlignment, window, LogLevel } from "vscode";
+import { isShippingBuild } from "./extension";
 
 const outputChannel = window.createOutputChannel("Ren'Py Language Extension", { log: true });
 const statusBar = window.createStatusBarItem(StatusBarAlignment.Right, 100);
 
-let extensionMode: ExtensionMode = null!;
-
 export function initializeLoggingSystems(context: ExtensionContext) {
-    extensionMode = context.extensionMode;
-
     context.subscriptions.push(outputChannel);
 
     outputChannel.clear();
@@ -79,7 +76,7 @@ export function logCatMessage(level: LogLevel, category: LogCategory, message: s
 }
 
 function debugLog(level: LogLevel, message: string) {
-    if (extensionMode !== ExtensionMode.Development) {
+    if (isShippingBuild()) {
         return;
     }
 

diff --git a/src/tokenizer/atl-token-patterns.g.ts b/src/tokenizer/atl-token-patterns.g.ts
@@ -5,7 +5,7 @@
 // THIS FILE HAS BEEN GENERATED BY THE `syntax-to-token-pattern.py` GENERATOR
 // DO NOT EDIT THIS FILE DIRECTLY! INSTEAD RUN THE PYTHON SCRIPT.
 // ANY MANUAL EDITS MADE TO THIS FILE WILL BE OVERWRITTEN. YOU HAVE BEEN WARNED.
-// Last generated: 12/07/2024 09:35:29 (UTC+0)
+// Last generated: 12/07/2024 18:50:56 (UTC+0)
 
 import { KeywordTokenType, EntityTokenType, MetaTokenType, CharacterTokenType } from "./renpy-tokens";
 import { TokenPattern } from "./token-pattern-types";
@@ -173,7 +173,7 @@ export const atlExpression: TokenPattern = {
         {
             debugName: "atlExpression.patterns![1]",
 
-            begin: /\b(with)\b[ \t]*/dg,
+            begin: /\b(with)\b/dg,
             beginCaptures: {
                 1: { token: KeywordTokenType.With, /*keyword.with.renpy*/ },
             },
@@ -189,7 +189,7 @@ export const atlKeywords: TokenPattern = {
             debugName: "atlKeywords.patterns![0]",
 
             // https://www.renpy.org/doc/html/atl.html#animation-statement
-            match: /^[ \t]*(animation)\b/dgm,
+            match: /(?<=^[ \t]*)(animation)\b/dgm,
             captures: {
                 1: { token: KeywordTokenType.Animation, /*keyword.animation.renpy*/ },
             },
@@ -198,7 +198,7 @@ export const atlKeywords: TokenPattern = {
             debugName: "atlKeywords.patterns![1]",
 
             // https://www.renpy.org/doc/html/atl.html#pass-statement
-            match: /^[ \t]*(pass)\b/dgm,
+            match: /(?<=^[ \t]*)(pass)\b/dgm,
             captures: {
                 1: { token: KeywordTokenType.Pass, /*keyword.control.flow.pass.renpy*/ },
             },
@@ -207,7 +207,7 @@ export const atlKeywords: TokenPattern = {
             debugName: "atlKeywords.patterns![2]",
 
             // https://www.renpy.org/doc/html/atl.html#repeat-statement and https://www.renpy.org/doc/html/atl.html#time-statement
-            begin: /^[ \t]*(?:(repeat)|(time)|(pause))\b[ \t]*/dgm,
+            begin: /(?<=^[ \t]*)(?:(repeat)|(time)|(pause))\b/dgm,
             beginCaptures: {
                 1: { token: KeywordTokenType.Repeat, /*keyword.control.flow.repeat.renpy*/ },
                 2: { token: KeywordTokenType.Time, /*keyword.control.flow.time.renpy*/ },
@@ -221,7 +221,7 @@ export const atlKeywords: TokenPattern = {
 
             // https://www.renpy.org/doc/html/atl.html#expression-statement
             contentToken: MetaTokenType.ATLWith, /*meta.atl.with.renpy*/
-            begin: /^[ \t]*(with)\b[ \t]*/dgm,
+            begin: /(?<=^[ \t]*)(with)\b/dgm,
             beginCaptures: {
                 1: { token: KeywordTokenType.With, /*keyword.with.renpy*/ },
             },
@@ -233,7 +233,7 @@ export const atlKeywords: TokenPattern = {
 
             // https://www.renpy.org/doc/html/atl.html#contains-statement
             contentToken: MetaTokenType.ATLContains, /*meta.atl.contains.renpy*/
-            begin: /^[ \t]*(contains)\b[ \t]*/dgm,
+            begin: /(?<=^[ \t]*)(contains)\b/dgm,
             beginCaptures: {
                 1: { token: KeywordTokenType.Contains, /*keyword.control.flow.contains.renpy*/ },
             },

diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts
@@ -446,8 +446,12 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case MetaTokenType.CameraStatement:
             case MetaTokenType.SceneStatement:
             case MetaTokenType.ShowStatement:
+            case MetaTokenType.HideStatement:
             case MetaTokenType.ImageStatement:
             case MetaTokenType.CallStatement:
+            case MetaTokenType.CallArguments:
+            case MetaTokenType.FromClause:
+            case MetaTokenType.FromArguments:
             case MetaTokenType.JumpStatement:
             case MetaTokenType.PlayAudioStatement:
             case MetaTokenType.QueueAudioStatement:
@@ -482,6 +486,7 @@ ${(decoration.hoverMessage as MarkdownString).value}`,
             case MetaTokenType.OnlayerParameters:
             case MetaTokenType.WithParameters:
             case MetaTokenType.ZorderParameters:
+            case MetaTokenType.PauseStatement:
             case MetaTokenType.PauseParameters:
             case MetaTokenType.ATLBlock:
             case MetaTokenType.ATLChoiceBlock: