Added support for LLongMA (#3616)

Added tokenizer for llongma
LAION-AI · Jul 29, 2023 · b9ac30a · b9ac30a
1 parent c2c1318
commit b9ac30a
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/model/model_training/utils/utils.py b/model/model_training/utils/utils.py
@@ -187,6 +187,7 @@ class TokenizerConfig(NamedTuple):
     "falcon": TokenizerConfig(
         special_tokens=SpecialTokens("<|endoftext|>", "<|endoftext|>", sep_token="<|endoftext|>")
     ),
+    "LLongMA": TokenizerConfig(special_tokens=SpecialTokens("</s>", "</s>", sep_token="<s>")),
 }