diff --git a/README.ja.md b/README.ja.md index d43d988b0..c0d2423ff 100644 --- a/README.ja.md +++ b/README.ja.md @@ -154,6 +154,11 @@ DB-GPTのアーキテクチャは以下の図に示されています： 私たちは、LLaMA/LLaMA2、Baichuan、ChatGLM、Wenxin、Tongyi、Zhipuなど、オープンソースおよびAPIエージェントからの数十の大規模言語モデル（LLM）を含む幅広いモデルをサポートしています。 - ニュース + - 🔥🔥🔥 [gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) + - 🔥🔥🔥 [gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) + - 🔥🔥🔥 [DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct) + - 🔥🔥🔥 [DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct) + - 🔥🔥🔥 [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct) - 🔥🔥🔥 [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct) - 🔥🔥🔥 [Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct) - 🔥🔥🔥 [Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct) diff --git a/README.md b/README.md index 9ee8375f8..f558aa413 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,10 @@ At present, we have introduced several key features to showcase our current capa We offer extensive model support, including dozens of large language models (LLMs) from both open-source and API agents, such as LLaMA/LLaMA2, Baichuan, ChatGLM, Wenxin, Tongyi, Zhipu, and many more. - News + - 🔥🔥🔥 [gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) + - 🔥🔥🔥 [gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) + - 🔥🔥🔥 [DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct) + - 🔥🔥🔥 [DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct) - 🔥🔥🔥 [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct) - 🔥🔥🔥 [Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct) - 🔥🔥🔥 [Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct) diff --git a/README.zh.md b/README.zh.md index 1035c3526..9d7b7b03d 100644 --- a/README.zh.md +++ b/README.zh.md @@ -152,6 +152,11 @@ 海量模型支持，包括开源、API代理等几十种大语言模型。如LLaMA/LLaMA2、Baichuan、ChatGLM、文心、通义、智谱等。当前已支持如下模型: - 新增支持模型 + - 🔥🔥🔥 [gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it) + - 🔥🔥🔥 [gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) + - 🔥🔥🔥 [DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct) + - 🔥🔥🔥 [DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct) + - 🔥🔥🔥 [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct) - 🔥🔥🔥 [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct) - 🔥🔥🔥 [Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct) - 🔥🔥🔥 [Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct) diff --git a/dbgpt/configs/model_config.py b/dbgpt/configs/model_config.py index 0876c29dc..53c46b0fb 100644 --- a/dbgpt/configs/model_config.py +++ b/dbgpt/configs/model_config.py @@ -225,8 +225,16 @@ def get_device() -> str: "gemma-7b-it": os.path.join(MODEL_PATH, "gemma-7b-it"), # https://huggingface.co/google/gemma-2b-it "gemma-2b-it": os.path.join(MODEL_PATH, "gemma-2b-it"), + "gemma-2-9b-it": os.path.join(MODEL_PATH, "gemma-2-9b-it"), + "gemma-2-27b-it": os.path.join(MODEL_PATH, "gemma-2-27b-it"), "starling-lm-7b-beta": os.path.join(MODEL_PATH, "Starling-LM-7B-beta"), "deepseek-v2-lite-chat": os.path.join(MODEL_PATH, "DeepSeek-V2-Lite-Chat"), + "deepseek-coder-v2-instruct": os.path.join( + MODEL_PATH, "DeepSeek-Coder-V2-Instruct" + ), + "deepseek-coder-v2-lite-instruct": os.path.join( + MODEL_PATH, "DeepSeek-Coder-V2-Lite-Instruct" + ), "sailor-14b-chat": os.path.join(MODEL_PATH, "Sailor-14B-Chat"), # https://huggingface.co/microsoft/Phi-3-medium-128k-instruct "phi-3-medium-128k-instruct": os.path.join( diff --git a/dbgpt/model/adapter/base.py b/dbgpt/model/adapter/base.py index 8d5bb9441..5c2c22ef2 100644 --- a/dbgpt/model/adapter/base.py +++ b/dbgpt/model/adapter/base.py @@ -112,6 +112,31 @@ def load(self, model_path: str, from_pretrained_kwargs: dict): """Load model and tokenizer""" raise NotImplementedError + def parse_max_length(self, model, tokenizer) -> Optional[int]: + """Parse the max_length of the model. + + Returns: + Optional[int]: The max_length of the model + """ + if not (tokenizer or model): + return None + try: + model_max_length = None + if tokenizer and hasattr(tokenizer, "model_max_length"): + model_max_length = tokenizer.model_max_length + if model_max_length and model_max_length < 100000000: + # Can't be too large + return model_max_length + if model and hasattr(model, "config"): + model_config = model.config + if hasattr(model_config, "max_sequence_length"): + return model_config.max_sequence_length + if hasattr(model_config, "max_position_embeddings"): + return model_config.max_position_embeddings + return None + except Exception: + return None + def load_from_params(self, params): """Load the model and tokenizer according to the given parameters""" raise NotImplementedError diff --git a/dbgpt/model/adapter/hf_adapter.py b/dbgpt/model/adapter/hf_adapter.py index 3576bb706..601b59bf6 100644 --- a/dbgpt/model/adapter/hf_adapter.py +++ b/dbgpt/model/adapter/hf_adapter.py @@ -73,6 +73,10 @@ def load(self, model_path: str, from_pretrained_kwargs: dict): ) from exc self.check_dependencies() + logger.info( + f"Load model from {model_path}, from_pretrained_kwargs: {from_pretrained_kwargs}" + ) + revision = from_pretrained_kwargs.get("revision", "main") try: tokenizer = AutoTokenizer.from_pretrained( @@ -235,6 +239,43 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None): ) +class Gemma2Adapter(NewHFChatModelAdapter): + """ + https://huggingface.co/google/gemma-2-27b-it + https://huggingface.co/google/gemma-2-9b-it + """ + + support_4bit: bool = True + support_8bit: bool = True + support_system_message: bool = False + + def use_fast_tokenizer(self) -> bool: + return True + + def check_transformer_version(self, current_version: str) -> None: + if not current_version >= "4.42.1": + raise ValueError( + "Gemma2 require transformers.__version__>=4.42.1, please upgrade your transformers package." + ) + + def do_match(self, lower_model_name_or_path: Optional[str] = None): + return ( + lower_model_name_or_path + and "gemma-2-" in lower_model_name_or_path + and "it" in lower_model_name_or_path + ) + + def load(self, model_path: str, from_pretrained_kwargs: dict): + import torch + + if not from_pretrained_kwargs: + from_pretrained_kwargs = {} + from_pretrained_kwargs["torch_dtype"] = torch.bfloat16 + # from_pretrained_kwargs["revision"] = "float16" + model, tokenizer = super().load(model_path, from_pretrained_kwargs) + return model, tokenizer + + class StarlingLMAdapter(NewHFChatModelAdapter): """ https://huggingface.co/Nexusflow/Starling-LM-7B-beta @@ -416,6 +457,17 @@ def load(self, model_path: str, from_pretrained_kwargs: dict): return model, tokenizer +class DeepseekCoderV2Adapter(DeepseekV2Adapter): + def do_match(self, lower_model_name_or_path: Optional[str] = None): + return ( + lower_model_name_or_path + and "deepseek" in lower_model_name_or_path + and "coder" in lower_model_name_or_path + and "v2" in lower_model_name_or_path + and "instruct" in lower_model_name_or_path + ) + + class SailorAdapter(QwenAdapter): """ https://huggingface.co/sail/Sailor-14B-Chat @@ -520,11 +572,13 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None): register_model_adapter(Mixtral8x7BAdapter) register_model_adapter(SOLARAdapter) register_model_adapter(GemmaAdapter) +register_model_adapter(Gemma2Adapter) register_model_adapter(StarlingLMAdapter) register_model_adapter(QwenAdapter) register_model_adapter(QwenMoeAdapter) register_model_adapter(Llama3Adapter) register_model_adapter(DeepseekV2Adapter) +register_model_adapter(DeepseekCoderV2Adapter) register_model_adapter(SailorAdapter) register_model_adapter(PhiAdapter) register_model_adapter(SQLCoderAdapter) diff --git a/dbgpt/model/cluster/worker/default_worker.py b/dbgpt/model/cluster/worker/default_worker.py index d1f2aea5b..04cd25bf7 100644 --- a/dbgpt/model/cluster/worker/default_worker.py +++ b/dbgpt/model/cluster/worker/default_worker.py @@ -116,7 +116,9 @@ def start( self.model, self.tokenizer = self.ml.loader_with_params( model_params, self.llm_adapter ) - model_max_length = _parse_model_max_length(self.model, self.tokenizer) + model_max_length = self.llm_adapter.parse_max_length( + self.model, self.tokenizer + ) if model_max_length: logger.info( f"Parse model max length {model_max_length} from model {self.model_name}." diff --git a/dbgpt/model/llm_out/hf_chat_llm.py b/dbgpt/model/llm_out/hf_chat_llm.py index 2129b19e2..4c0b996cb 100644 --- a/dbgpt/model/llm_out/hf_chat_llm.py +++ b/dbgpt/model/llm_out/hf_chat_llm.py @@ -21,7 +21,7 @@ def huggingface_chat_generate_stream( echo = params.get("echo", False) max_new_tokens = int(params.get("max_new_tokens", 2048)) stop_token_ids = params.get("stop_token_ids", []) - do_sample = params.get("do_sample", None) + do_sample = params.get("do_sample", True) custom_stop_words = params.get("custom_stop_words", []) input_ids = tokenizer(prompt).input_ids @@ -34,11 +34,6 @@ def huggingface_chat_generate_stream( input_echo_len = len(input_ids) input_ids = torch.as_tensor([input_ids], device=device) - # messages = params["messages"] - # messages = ModelMessage.to_openai_messages(messages) - # input_ids = tokenizer.apply_chat_template(conversation=messages, tokenize=True, add_generation_prompt=True, return_tensors='pt') - # input_ids = input_ids.to(device) - streamer = TextIteratorStreamer( tokenizer, skip_prompt=not echo, skip_special_tokens=True ) @@ -55,7 +50,9 @@ def huggingface_chat_generate_stream( if do_sample is not None: base_kwargs["do_sample"] = do_sample - logger.info(f"Predict with parameters: {base_kwargs}") + logger.info( + f"Predict with parameters: {base_kwargs}\ncustom_stop_words: {custom_stop_words}" + ) generate_kwargs = {"input_ids": input_ids, **base_kwargs} thread = Thread(target=model.generate, kwargs=generate_kwargs)