Update modeling_aquila.py

FlagAI-Open · Sep 20, 2023 · de64105 · de64105
1 parent 0ede7b6
commit de64105
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/examples/Aquila/Aquila-chat-hf/modeling_aquila.py b/examples/Aquila/Aquila-chat-hf/modeling_aquila.py
@@ -346,7 +346,7 @@ def forward(
         value_states = repeat_kv(value_states, self.num_key_value_groups)
 
         attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
-
+        attn_weights = torch.clamp(attn_weights, min=-1024., max=1024.)
         if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
             raise ValueError(
                 f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"