feat(model): Support gemma-2 model (#1675)

README.ja.md

@@ -154,6 +154,11 @@ DB-GPTのアーキテクチャは以下の図に示されています：
   私たちは、LLaMA/LLaMA2、Baichuan、ChatGLM、Wenxin、Tongyi、Zhipuなど、オープンソースおよびAPIエージェントからの数十の大規模言語モデル（LLM）を含む幅広いモデルをサポートしています。
 
   - ニュース
+    - 🔥🔥🔥  [gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it)
+    - 🔥🔥🔥  [gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it)
+    - 🔥🔥🔥  [DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct)
+    - 🔥🔥🔥  [DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct)
+    - 🔥🔥🔥  [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct)
     - 🔥🔥🔥  [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct)
     - 🔥🔥🔥  [Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct)
     - 🔥🔥🔥  [Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct)

README.md

@@ -158,6 +158,10 @@ At present, we have introduced several key features to showcase our current capa
   We offer extensive model support, including dozens of large language models (LLMs) from both open-source and API agents, such as LLaMA/LLaMA2, Baichuan, ChatGLM, Wenxin, Tongyi, Zhipu, and many more. 
 
   - News
+    - 🔥🔥🔥  [gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it)
+    - 🔥🔥🔥  [gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it)
+    - 🔥🔥🔥  [DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct)
+    - 🔥🔥🔥  [DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct)
     - 🔥🔥🔥  [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct)
     - 🔥🔥🔥  [Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct)
     - 🔥🔥🔥  [Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct)

README.zh.md

@@ -152,6 +152,11 @@
   海量模型支持，包括开源、API代理等几十种大语言模型。如LLaMA/LLaMA2、Baichuan、ChatGLM、文心、通义、智谱等。当前已支持如下模型: 
 
   - 新增支持模型
+    - 🔥🔥🔥  [gemma-2-27b-it](https://huggingface.co/google/gemma-2-27b-it)
+    - 🔥🔥🔥  [gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it)
+    - 🔥🔥🔥  [DeepSeek-Coder-V2-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Instruct)
+    - 🔥🔥🔥  [DeepSeek-Coder-V2-Lite-Instruct](https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct)
+    - 🔥🔥🔥  [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct)
     - 🔥🔥🔥  [Qwen2-57B-A14B-Instruct](https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct)
     - 🔥🔥🔥  [Qwen2-72B-Instruct](https://huggingface.co/Qwen/Qwen2-72B-Instruct)
     - 🔥🔥🔥  [Qwen2-7B-Instruct](https://huggingface.co/Qwen/Qwen2-7B-Instruct)

dbgpt/configs/model_config.py

@@ -225,8 +225,16 @@ def get_device() -> str:
     "gemma-7b-it": os.path.join(MODEL_PATH, "gemma-7b-it"),
     # https://huggingface.co/google/gemma-2b-it
     "gemma-2b-it": os.path.join(MODEL_PATH, "gemma-2b-it"),
+    "gemma-2-9b-it": os.path.join(MODEL_PATH, "gemma-2-9b-it"),
+    "gemma-2-27b-it": os.path.join(MODEL_PATH, "gemma-2-27b-it"),
     "starling-lm-7b-beta": os.path.join(MODEL_PATH, "Starling-LM-7B-beta"),
     "deepseek-v2-lite-chat": os.path.join(MODEL_PATH, "DeepSeek-V2-Lite-Chat"),
+    "deepseek-coder-v2-instruct": os.path.join(
+        MODEL_PATH, "DeepSeek-Coder-V2-Instruct"
+    ),
+    "deepseek-coder-v2-lite-instruct": os.path.join(
+        MODEL_PATH, "DeepSeek-Coder-V2-Lite-Instruct"
+    ),
     "sailor-14b-chat": os.path.join(MODEL_PATH, "Sailor-14B-Chat"),
     # https://huggingface.co/microsoft/Phi-3-medium-128k-instruct
     "phi-3-medium-128k-instruct": os.path.join(

dbgpt/model/adapter/base.py

@@ -112,6 +112,31 @@ def load(self, model_path: str, from_pretrained_kwargs: dict):
         """Load model and tokenizer"""
         raise NotImplementedError
 
+    def parse_max_length(self, model, tokenizer) -> Optional[int]:
+        """Parse the max_length of the model.
+
+        Returns:
+            Optional[int]: The max_length of the model
+        """
+        if not (tokenizer or model):
+            return None
+        try:
+            model_max_length = None
+            if tokenizer and hasattr(tokenizer, "model_max_length"):
+                model_max_length = tokenizer.model_max_length
+            if model_max_length and model_max_length < 100000000:
+                # Can't be too large
+                return model_max_length
+            if model and hasattr(model, "config"):
+                model_config = model.config
+                if hasattr(model_config, "max_sequence_length"):
+                    return model_config.max_sequence_length
+                if hasattr(model_config, "max_position_embeddings"):
+                    return model_config.max_position_embeddings
+            return None
+        except Exception:
+            return None
+
     def load_from_params(self, params):
         """Load the model and tokenizer according to the given parameters"""
         raise NotImplementedError

dbgpt/model/adapter/hf_adapter.py

@@ -73,6 +73,10 @@ def load(self, model_path: str, from_pretrained_kwargs: dict):
             ) from exc
         self.check_dependencies()
 
+        logger.info(
+            f"Load model from {model_path}, from_pretrained_kwargs: {from_pretrained_kwargs}"
+        )
+
         revision = from_pretrained_kwargs.get("revision", "main")
         try:
             tokenizer = AutoTokenizer.from_pretrained(
@@ -235,6 +239,43 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None):
         )
 
 
+class Gemma2Adapter(NewHFChatModelAdapter):
+    """
+    https://huggingface.co/google/gemma-2-27b-it
+    https://huggingface.co/google/gemma-2-9b-it
+    """
+
+    support_4bit: bool = True
+    support_8bit: bool = True
+    support_system_message: bool = False
+
+    def use_fast_tokenizer(self) -> bool:
+        return True
+
+    def check_transformer_version(self, current_version: str) -> None:
+        if not current_version >= "4.42.1":
+            raise ValueError(
+                "Gemma2 require transformers.__version__>=4.42.1, please upgrade your transformers package."
+            )
+
+    def do_match(self, lower_model_name_or_path: Optional[str] = None):
+        return (
+            lower_model_name_or_path
+            and "gemma-2-" in lower_model_name_or_path
+            and "it" in lower_model_name_or_path
+        )
+
+    def load(self, model_path: str, from_pretrained_kwargs: dict):
+        import torch
+
+        if not from_pretrained_kwargs:
+            from_pretrained_kwargs = {}
+        from_pretrained_kwargs["torch_dtype"] = torch.bfloat16
+        # from_pretrained_kwargs["revision"] = "float16"
+        model, tokenizer = super().load(model_path, from_pretrained_kwargs)
+        return model, tokenizer
+
+
 class StarlingLMAdapter(NewHFChatModelAdapter):
     """
     https://huggingface.co/Nexusflow/Starling-LM-7B-beta
@@ -416,6 +457,17 @@ def load(self, model_path: str, from_pretrained_kwargs: dict):
         return model, tokenizer
 
 
+class DeepseekCoderV2Adapter(DeepseekV2Adapter):
+    def do_match(self, lower_model_name_or_path: Optional[str] = None):
+        return (
+            lower_model_name_or_path
+            and "deepseek" in lower_model_name_or_path
+            and "coder" in lower_model_name_or_path
+            and "v2" in lower_model_name_or_path
+            and "instruct" in lower_model_name_or_path
+        )
+
+
 class SailorAdapter(QwenAdapter):
     """
     https://huggingface.co/sail/Sailor-14B-Chat
@@ -520,11 +572,13 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None):
 register_model_adapter(Mixtral8x7BAdapter)
 register_model_adapter(SOLARAdapter)
 register_model_adapter(GemmaAdapter)
+register_model_adapter(Gemma2Adapter)
 register_model_adapter(StarlingLMAdapter)
 register_model_adapter(QwenAdapter)
 register_model_adapter(QwenMoeAdapter)
 register_model_adapter(Llama3Adapter)
 register_model_adapter(DeepseekV2Adapter)
+register_model_adapter(DeepseekCoderV2Adapter)
 register_model_adapter(SailorAdapter)
 register_model_adapter(PhiAdapter)
 register_model_adapter(SQLCoderAdapter)

dbgpt/model/cluster/worker/default_worker.py

@@ -116,7 +116,9 @@ def start(
             self.model, self.tokenizer = self.ml.loader_with_params(
                 model_params, self.llm_adapter
             )
-            model_max_length = _parse_model_max_length(self.model, self.tokenizer)
+            model_max_length = self.llm_adapter.parse_max_length(
+                self.model, self.tokenizer
+            )
             if model_max_length:
                 logger.info(
                     f"Parse model max length {model_max_length} from model {self.model_name}."

dbgpt/model/llm_out/hf_chat_llm.py

@@ -21,7 +21,7 @@ def huggingface_chat_generate_stream(
     echo = params.get("echo", False)
     max_new_tokens = int(params.get("max_new_tokens", 2048))
     stop_token_ids = params.get("stop_token_ids", [])
-    do_sample = params.get("do_sample", None)
+    do_sample = params.get("do_sample", True)
     custom_stop_words = params.get("custom_stop_words", [])
 
     input_ids = tokenizer(prompt).input_ids
@@ -34,11 +34,6 @@ def huggingface_chat_generate_stream(
     input_echo_len = len(input_ids)
     input_ids = torch.as_tensor([input_ids], device=device)
 
-    # messages = params["messages"]
-    # messages = ModelMessage.to_openai_messages(messages)
-    # input_ids = tokenizer.apply_chat_template(conversation=messages, tokenize=True, add_generation_prompt=True, return_tensors='pt')
-    # input_ids = input_ids.to(device)
-
     streamer = TextIteratorStreamer(
         tokenizer, skip_prompt=not echo, skip_special_tokens=True
     )
@@ -55,7 +50,9 @@ def huggingface_chat_generate_stream(
     if do_sample is not None:
         base_kwargs["do_sample"] = do_sample
 
-    logger.info(f"Predict with parameters: {base_kwargs}")
+    logger.info(
+        f"Predict with parameters: {base_kwargs}\ncustom_stop_words: {custom_stop_words}"
+    )
 
     generate_kwargs = {"input_ids": input_ids, **base_kwargs}
     thread = Thread(target=model.generate, kwargs=generate_kwargs)