From d0522c1ea162db84ff02010c419d1c5a1fcc108e Mon Sep 17 00:00:00 2001
From: Abdelrahman Aly Abounegm <abounegm@yandex-team.ru>
Date: Mon, 14 Oct 2024 11:19:55 +0300
Subject: [PATCH] Fix indentation token lengths (#1708)

When the dedent had some whitespace after it (not an empty line),
that whitespace was considered part of the dedent token,
causing the CST node range to be off
---
 .../langium/src/parser/indentation-aware.ts   | 20 ++++++-------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/packages/langium/src/parser/indentation-aware.ts b/packages/langium/src/parser/indentation-aware.ts
index 3891f585e..1d7636c1c 100644
--- a/packages/langium/src/parser/indentation-aware.ts
+++ b/packages/langium/src/parser/indentation-aware.ts
@@ -272,16 +272,7 @@ export class IndentationAwareTokenBuilder<Terminals extends string = string, Key
 
         this.indentationStack.push(currIndentLevel);
 
-        const indentToken = this.createIndentationTokenInstance(
-            this.indentTokenType,
-            text,
-            match?.[0] ?? '',
-            offset,
-        );
-        tokens.push(indentToken);
-
-        // Token already added, let the indentation now be consumed as whitespace and ignored
-        return null;
+        return match;
     }
 
     /**
@@ -321,19 +312,20 @@ export class IndentationAwareTokenBuilder<Terminals extends string = string, Key
         }
 
         const numberOfDedents = this.indentationStack.length - matchIndentIndex - 1;
+        const newlinesBeforeDedent = text.substring(0, offset).match(/[\r\n]+$/)?.[0].length ?? 1;
 
         for (let i = 0; i < numberOfDedents; i++) {
             const token = this.createIndentationTokenInstance(
                 this.dedentTokenType,
                 text,
-                match?.[0] ?? '',
-                offset,
+                '',  // Dedents are 0-width tokens
+                offset - (newlinesBeforeDedent - 1), // Place the dedent after the first new line character
             );
             tokens.push(token);
             this.indentationStack.pop();
         }
 
-        // Token already added, let the dedentation now be consumed as whitespace and ignored
+        // Token already added, let the dedentation now be consumed as whitespace (if any) and ignored
         return null;
     }
 
@@ -365,7 +357,7 @@ export class IndentationAwareTokenBuilder<Terminals extends string = string, Key
         const remainingDedents: IToken[] = [];
         while (this.indentationStack.length > 1) {
             remainingDedents.push(
-                this.createIndentationTokenInstance(this.dedentTokenType, text, this.options.dedentTokenName, text.length)
+                this.createIndentationTokenInstance(this.dedentTokenType, text, '', text.length)
             );
             this.indentationStack.pop();
         }