Skip to content

Commit

Permalink
Fix dedent tokens being nonempty
Browse files Browse the repository at this point in the history
When the dedent had some whitespace after it (not an empty line),
that whitespace was considered part of the dedent token,
causing the CST node range to be off
  • Loading branch information
aabounegm committed Oct 2, 2024
1 parent 3994c08 commit 1919efa
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions packages/langium/src/parser/indentation-aware.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,14 +317,14 @@ export class IndentationAwareTokenBuilder<Terminals extends string = string, Key
const token = this.createIndentationTokenInstance(
this.dedentTokenType,
text,
match?.[0] ?? '',
'', // Dedents are 0-width tokens
offset,
);
tokens.push(token);
this.indentationStack.pop();
}

// Token already added, let the dedentation now be consumed as whitespace and ignored
// Token already added, let the dedentation now be consumed as whitespace (if any) and ignored
return null;
}

Expand Down Expand Up @@ -356,7 +356,7 @@ export class IndentationAwareTokenBuilder<Terminals extends string = string, Key
const remainingDedents: IToken[] = [];
while (this.indentationStack.length > 1) {
remainingDedents.push(
this.createIndentationTokenInstance(this.dedentTokenType, text, this.options.dedentTokenName, text.length)
this.createIndentationTokenInstance(this.dedentTokenType, text, '', text.length)
);
this.indentationStack.pop();
}
Expand Down

0 comments on commit 1919efa

Please sign in to comment.