From b9ac30a89fea29b3e493eeb24e6611413b293b90 Mon Sep 17 00:00:00 2001 From: Shahul ES Date: Sat, 29 Jul 2023 23:00:03 +0530 Subject: [PATCH] Added support for LLongMA (#3616) Added tokenizer for llongma --- model/model_training/utils/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/model_training/utils/utils.py b/model/model_training/utils/utils.py index e3c9098899..677da36a22 100644 --- a/model/model_training/utils/utils.py +++ b/model/model_training/utils/utils.py @@ -187,6 +187,7 @@ class TokenizerConfig(NamedTuple): "falcon": TokenizerConfig( special_tokens=SpecialTokens("<|endoftext|>", "<|endoftext|>", sep_token="<|endoftext|>") ), + "LLongMA": TokenizerConfig(special_tokens=SpecialTokens("", "", sep_token="")), }