From 23268dd4d666311a9d13ea65ae1230ceea6f9fb0 Mon Sep 17 00:00:00 2001 From: Jack Tysoe <91137069+tysoekong@users.noreply.github.com> Date: Wed, 15 Jan 2025 09:14:02 +0000 Subject: [PATCH] fix(ai): fix Azure streaming with raw JSON transmission mode (#14148) FTI-6419 (cherry picked from commit 17836b48336e386a40084c38e2b5bc7547a33c78) --- changelog/unreleased/kong/fix-ai-azure-streaming.yml | 3 +++ kong/llm/drivers/shared.lua | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/kong/fix-ai-azure-streaming.yml diff --git a/changelog/unreleased/kong/fix-ai-azure-streaming.yml b/changelog/unreleased/kong/fix-ai-azure-streaming.yml new file mode 100644 index 000000000000..3068b36855e5 --- /dev/null +++ b/changelog/unreleased/kong/fix-ai-azure-streaming.yml @@ -0,0 +1,3 @@ +message: "**ai-proxy**: Fixed a bug where Azure streaming responses would be missing individual tokens." +type: bugfix +scope: Plugin diff --git a/kong/llm/drivers/shared.lua b/kong/llm/drivers/shared.lua index 55169a29b97d..5570598238cf 100644 --- a/kong/llm/drivers/shared.lua +++ b/kong/llm/drivers/shared.lua @@ -353,7 +353,7 @@ function _M.frame_to_events(frame, content_type) -- some new LLMs return the JSON object-by-object, -- because that totally makes sense to parse?! local frame_start = frame and frame:sub(1, 1) - if frame_start == "," or frame_start == "[" then + if (not kong or not kong.ctx.plugin.truncated_frame) and (frame_start == "," or frame_start == "[") then local done = false -- if this is the first frame, it will begin with array opener '[' @@ -416,7 +416,7 @@ function _M.frame_to_events(frame, content_type) if #dat > 0 and #event_lines == i then ngx.log(ngx.DEBUG, "[ai-proxy] truncated sse frame head") if kong then - kong.ctx.plugin.truncated_frame = dat + kong.ctx.plugin.truncated_frame = fmt("%s%s", (kong.ctx.plugin.truncated_frame or ""), dat) end break -- stop parsing immediately, server has done something wrong