Check -1 index for vLLM

modal-labs · Feb 15, 2024 · 8af8fb0 · 8af8fb0
1 parent b03e00c
commit 8af8fb0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/inference.py b/src/inference.py
@@ -42,7 +42,7 @@ async def completion(self, input: str):
         t0 = time.time()
         index, tokens = 0, 0
         async for request_output in results_generator:
-            if "\ufffd" == request_output.outputs[0].text[-1]:
+            if request_output.outputs[0].text and "\ufffd" == request_output.outputs[0].text[-1]:
                 continue
             yield request_output.outputs[0].text[index:]
             index = len(request_output.outputs[0].text)