Dont predict EOText token

TJ-Solergibert · Aug 2, 2024 · d5228bb · d5228bb
1 parent 2d882db
commit d5228bb
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/nanotron/data/chat_tokenizer.py b/src/nanotron/data/chat_tokenizer.py
@@ -54,7 +54,9 @@ def __call__(self, conversation: List[dict]) -> Tuple[List[int], List[bool]]:
 
         # Append <|end_of_text|> token
         tokens.extend(self.tokenizer.encode("<|end_of_text|>", add_special_tokens=False))
-        is_completitions.append(True)
+        is_completitions.append(
+            False
+        )  # NOTE(tj.solergibert) No need to predict <|end_of_text|> token from <|eot_id|> token
 
         return tokens, is_completitions