fix: stream response (#4853)

docSite/content/zh-cn/docs/development/upgrading/4910.md

@@ -19,4 +19,6 @@ weight: 790
 
 ## 🐛 修复
 
-1. 全文检索多知识库时排序得分排序不正确
\ No newline at end of file
+1. 全文检索多知识库时排序得分排序不正确。
+2. 流响应捕获 finish_reason 可能不正确。
+3. 工具调用模式，未保存思考输出。
\ No newline at end of file

packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts

 import { createChatCompletion } from '../../../../ai/config';
 import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../../chat/utils';
-import {
-  type ChatCompletion,
-  type StreamChatType,
-  type ChatCompletionMessageParam,
-  type ChatCompletionCreateParams,
-  type ChatCompletionMessageFunctionCall,
-  type ChatCompletionFunctionMessageParam,
-  type ChatCompletionAssistantMessageParam
+import type {
+  ChatCompletion,
+  StreamChatType,
+  ChatCompletionMessageParam,
+  ChatCompletionCreateParams,
+  ChatCompletionMessageFunctionCall,
+  ChatCompletionFunctionMessageParam,
+  ChatCompletionAssistantMessageParam,
+  CompletionFinishReason
 } from '@fastgpt/global/core/ai/type.d';
 import { type NextApiResponse } from 'next';
 import { responseWriteController } from '../../../../../common/response';
@@ -259,14 +260,15 @@ export const runToolWithFunctionCall = async (
     }
   });
 
-  let { answer, functionCalls, inputTokens, outputTokens } = await (async () => {
+  let { answer, functionCalls, inputTokens, outputTokens, finish_reason } = await (async () => {
     if (isStreamResponse) {
       if (!res || res.closed) {
         return {
           answer: '',
           functionCalls: [],
           inputTokens: 0,
-          outputTokens: 0
+          outputTokens: 0,
+          finish_reason: 'close' as const
         };
       }
       const result = await streamResponse({
@@ -281,10 +283,12 @@ export const runToolWithFunctionCall = async (
         answer: result.answer,
         functionCalls: result.functionCalls,
         inputTokens: result.usage.prompt_tokens,
-        outputTokens: result.usage.completion_tokens
+        outputTokens: result.usage.completion_tokens,
+        finish_reason: result.finish_reason
       };
     } else {
       const result = aiResponse as ChatCompletion;
+      const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
       const function_call = result.choices?.[0]?.message?.function_call;
       const usage = result.usage;
 
@@ -315,7 +319,8 @@ export const runToolWithFunctionCall = async (
         answer,
         functionCalls: toolCalls,
         inputTokens: usage?.prompt_tokens,
-        outputTokens: usage?.completion_tokens
+        outputTokens: usage?.completion_tokens,
+        finish_reason
       };
     }
   })();
@@ -481,7 +486,8 @@ export const runToolWithFunctionCall = async (
         completeMessages,
         assistantResponses: toolNodeAssistants,
         runTimes,
-        toolWorkflowInteractiveResponse
+        toolWorkflowInteractiveResponse,
+        finish_reason
       };
     }
 
@@ -495,7 +501,8 @@ export const runToolWithFunctionCall = async (
         toolNodeInputTokens,
         toolNodeOutputTokens,
         assistantResponses: toolNodeAssistants,
-        runTimes
+        runTimes,
+        finish_reason
       }
     );
   } else {
@@ -523,7 +530,8 @@ export const runToolWithFunctionCall = async (
         : outputTokens,
       completeMessages,
       assistantResponses: [...assistantResponses, ...toolNodeAssistant.value],
-      runTimes: (response?.runTimes || 0) + 1
+      runTimes: (response?.runTimes || 0) + 1,
+      finish_reason
     };
   }
 };
@@ -546,28 +554,25 @@ async function streamResponse({
     readStream: stream
   });
 
-  let textAnswer = '';
   let functionCalls: ChatCompletionMessageFunctionCall[] = [];
   let functionId = getNanoid();
-  let usage = getLLMDefaultUsage();
 
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
 
   for await (const part of stream) {
-    usage = part.usage || usage;
     if (res.closed) {
       stream.controller?.abort();
+      updateFinishReason('close');
       break;
     }
 
-    const { content: toolChoiceContent, responseContent } = parsePart({
+    const { responseContent } = parsePart({
       part,
       parseThinkTag: false,
       retainDatasetCite
     });
 
     const responseChoice = part.choices?.[0]?.delta;
-    textAnswer += toolChoiceContent;
 
     if (responseContent) {
       workflowStreamResponse?.({
@@ -577,7 +582,7 @@ async function streamResponse({
           text: responseContent
         })
       });
-    } else if (responseChoice.function_call) {
+    } else if (responseChoice?.function_call) {
       const functionCall: {
         arguments?: string;
         name?: string;
@@ -640,5 +645,7 @@ async function streamResponse({
     }
   }
 
-  return { answer: textAnswer, functionCalls, usage };
+  const { content, finish_reason, usage } = getResponseData();
+
+  return { answer: content, functionCalls, finish_reason, usage };
 }

packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts

@@ -220,7 +220,8 @@ export const runToolWithPromptCall = async (
 
   const max_tokens = computedMaxToken({
     model: toolModel,
-    maxToken
+    maxToken,
+    min: 100
   });
   const filterMessages = await filterGPTMessageByMaxContext({
     messages,
@@ -592,28 +593,22 @@ async function streamResponse({
 
   let startResponseWrite = false;
   let answer = '';
-  let reasoning = '';
-  let finish_reason: CompletionFinishReason = null;
-  let usage = getLLMDefaultUsage();
 
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
 
   for await (const part of stream) {
-    usage = part.usage || usage;
     if (res.closed) {
       stream.controller?.abort();
-      finish_reason = 'close';
+      updateFinishReason('close');
       break;
     }
 
-    const { reasoningContent, content, responseContent, finishReason } = parsePart({
+    const { reasoningContent, content, responseContent } = parsePart({
       part,
       parseThinkTag: aiChatReasoning,
       retainDatasetCite
     });
-    finish_reason = finish_reason || finishReason;
     answer += content;
-    reasoning += reasoningContent;
 
     // Reasoning response
     if (aiChatReasoning && reasoningContent) {
@@ -658,7 +653,9 @@ async function streamResponse({
     }
   }
 
-  return { answer, reasoning, finish_reason, usage };
+  const { reasoningContent, content, finish_reason, usage } = getResponseData();
+
+  return { answer: content, reasoning: reasoningContent, finish_reason, usage };
 }
 
 const parseAnswer = (

packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts

@@ -7,17 +7,13 @@ import {
   type ChatCompletionToolMessageParam,
   type ChatCompletionMessageParam,
   type ChatCompletionTool,
-  type ChatCompletionAssistantMessageParam,
   type CompletionFinishReason
 } from '@fastgpt/global/core/ai/type';
 import { type NextApiResponse } from 'next';
 import { responseWriteController } from '../../../../../common/response';
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
-import {
-  ChatCompletionRequestMessageRoleEnum,
-  getLLMDefaultUsage
-} from '@fastgpt/global/core/ai/constants';
+import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import { dispatchWorkFlow } from '../../index';
 import {
   type DispatchToolModuleProps,
@@ -254,7 +250,8 @@ export const runToolWithToolChoice = async (
 
   const max_tokens = computedMaxToken({
     model: toolModel,
-    maxToken
+    maxToken,
+    min: 100
   });
 
   // Filter histories by maxToken
@@ -319,97 +316,101 @@ export const runToolWithToolChoice = async (
     }
   });
 
-  let { answer, toolCalls, finish_reason, inputTokens, outputTokens } = await (async () => {
-    if (isStreamResponse) {
-      if (!res || res.closed) {
+  let { reasoningContent, answer, toolCalls, finish_reason, inputTokens, outputTokens } =
+    await (async () => {
+      if (isStreamResponse) {
+        if (!res || res.closed) {
+          return {
+            reasoningContent: '',
+            answer: '',
+            toolCalls: [],
+            finish_reason: 'close' as const,
+            inputTokens: 0,
+            outputTokens: 0
+          };
+        }
+
+        const result = await streamResponse({
+          res,
+          workflowStreamResponse,
+          toolNodes,
+          stream: aiResponse,
+          aiChatReasoning,
+          retainDatasetCite
+        });
+
         return {
-          answer: '',
-          toolCalls: [],
-          finish_reason: 'close' as const,
-          inputTokens: 0,
-          outputTokens: 0
+          reasoningContent: result.reasoningContent,
+          answer: result.answer,
+          toolCalls: result.toolCalls,
+          finish_reason: result.finish_reason,
+          inputTokens: result.usage.prompt_tokens,
+          outputTokens: result.usage.completion_tokens
         };
-      }
+      } else {
+        const result = aiResponse as ChatCompletion;
+        const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
+        const calls = result.choices?.[0]?.message?.tool_calls || [];
+        const answer = result.choices?.[0]?.message?.content || '';
+        // @ts-ignore
+        const reasoningContent = result.choices?.[0]?.message?.reasoning_content || '';
+        const usage = result.usage;
+
+        if (aiChatReasoning && reasoningContent) {
+          workflowStreamResponse?.({
+            event: SseResponseEventEnum.fastAnswer,
+            data: textAdaptGptResponse({
+              reasoning_content: removeDatasetCiteText(reasoningContent, retainDatasetCite)
+            })
+          });
+        }
 
-      const result = await streamResponse({
-        res,
-        workflowStreamResponse,
-        toolNodes,
-        stream: aiResponse,
-        aiChatReasoning,
-        retainDatasetCite
-      });
+        // 格式化 toolCalls
+        const toolCalls = calls.map((tool) => {
+          const toolNode = toolNodes.find((item) => item.nodeId === tool.function?.name);
+
+          // 不支持 stream 模式的模型的这里需要补一个响应给客户端
+          workflowStreamResponse?.({
+            event: SseResponseEventEnum.toolCall,
+            data: {
+              tool: {
+                id: tool.id,
+                toolName: toolNode?.name || '',
+                toolAvatar: toolNode?.avatar || '',
+                functionName: tool.function.name,
+                params: tool.function?.arguments ?? '',
+                response: ''
+              }
+            }
+          });
 
-      return {
-        answer: result.answer,
-        toolCalls: result.toolCalls,
-        finish_reason: result.finish_reason,
-        inputTokens: result.usage.prompt_tokens,
-        outputTokens: result.usage.completion_tokens
-      };
-    } else {
-      const result = aiResponse as ChatCompletion;
-      const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
-      const calls = result.choices?.[0]?.message?.tool_calls || [];
-      const answer = result.choices?.[0]?.message?.content || '';
-      // @ts-ignore
-      const reasoningContent = result.choices?.[0]?.message?.reasoning_content || '';
-      const usage = result.usage;
-
-      if (aiChatReasoning && reasoningContent) {
-        workflowStreamResponse?.({
-          event: SseResponseEventEnum.fastAnswer,
-          data: textAdaptGptResponse({
-            reasoning_content: removeDatasetCiteText(reasoningContent, retainDatasetCite)
-          })
+          return {
+            ...tool,
+            toolName: toolNode?.name || '',
+            toolAvatar: toolNode?.avatar || ''
+          };
         });
-      }
 
-      // 格式化 toolCalls
-      const toolCalls = calls.map((tool) => {
-        const toolNode = toolNodes.find((item) => item.nodeId === tool.function?.name);
-
-        // 不支持 stream 模式的模型的这里需要补一个响应给客户端
-        workflowStreamResponse?.({
-          event: SseResponseEventEnum.toolCall,
-          data: {
-            tool: {
-              id: tool.id,
-              toolName: toolNode?.name || '',
-              toolAvatar: toolNode?.avatar || '',
-              functionName: tool.function.name,
-              params: tool.function?.arguments ?? '',
-              response: ''
-            }
-          }
-        });
+        if (answer) {
+          workflowStreamResponse?.({
+            event: SseResponseEventEnum.fastAnswer,
+            data: textAdaptGptResponse({
+              text: removeDatasetCiteText(answer, retainDatasetCite)
+            })
+          });
+        }
 
         return {
-          ...tool,
-          toolName: toolNode?.name || '',
-          toolAvatar: toolNode?.avatar || ''
+          reasoningContent: (reasoningContent as string) || '',
+          answer,
+          toolCalls: toolCalls,
+          finish_reason,
+          inputTokens: usage?.prompt_tokens,
+          outputTokens: usage?.completion_tokens
         };
-      });
-
-      if (answer) {
-        workflowStreamResponse?.({
-          event: SseResponseEventEnum.fastAnswer,
-          data: textAdaptGptResponse({
-            text: removeDatasetCiteText(answer, retainDatasetCite)
-          })
-        });
       }
-
-      return {
-        answer,
-        toolCalls: toolCalls,
-        finish_reason,
-        inputTokens: usage?.prompt_tokens,
-        outputTokens: usage?.completion_tokens
-      };
-    }
-  })();
-  if (!answer && toolCalls.length === 0) {
+    })();
+  if (!answer && !reasoningContent && toolCalls.length === 0) {
     return Promise.reject(getEmptyResponseTip());
   }
 
@@ -501,12 +502,13 @@ export const runToolWithToolChoice = async (
 
   if (toolCalls.length > 0) {
     // Run the tool, combine its results, and perform another round of AI calls
-    const assistantToolMsgParams: ChatCompletionAssistantMessageParam[] = [
-      ...(answer
+    const assistantToolMsgParams: ChatCompletionMessageParam[] = [
+      ...(answer || reasoningContent
         ? [
             {
               role: ChatCompletionRequestMessageRoleEnum.Assistant as 'assistant',
-              content: answer
+              content: answer,
+              reasoning_text: reasoningContent
             }
           ]
         : []),
@@ -627,9 +629,10 @@ export const runToolWithToolChoice = async (
     );
   } else {
     // No tool is invoked, indicating that the process is over
-    const gptAssistantResponse: ChatCompletionAssistantMessageParam = {
+    const gptAssistantResponse: ChatCompletionMessageParam = {
       role: ChatCompletionRequestMessageRoleEnum.Assistant,
-      content: answer
+      content: answer,
+      reasoning_text: reasoningContent
     };
     const completeMessages = filterMessages.concat(gptAssistantResponse);
     inputTokens = inputTokens || (await countGptMessagesTokens(requestMessages, tools));
@@ -671,34 +674,23 @@ async function streamResponse({
     readStream: stream
   });
 
-  let textAnswer = '';
   let callingTool: { name: string; arguments: string } | null = null;
   let toolCalls: ChatCompletionMessageToolCall[] = [];
-  let finish_reason: CompletionFinishReason = null;
-  let usage = getLLMDefaultUsage();
 
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
 
   for await (const part of stream) {
-    usage = part.usage || usage;
     if (res.closed) {
       stream.controller?.abort();
-      finish_reason = 'close';
+      updateFinishReason('close');
       break;
     }
 
-    const {
-      reasoningContent,
-      content: toolChoiceContent,
-      responseContent,
-      finishReason
-    } = parsePart({
+    const { reasoningContent, responseContent } = parsePart({
       part,
       parseThinkTag: true,
       retainDatasetCite
     });
-    textAnswer += toolChoiceContent;
-    finish_reason = finishReason || finish_reason;
 
     const responseChoice = part.choices?.[0]?.delta;
 
@@ -800,5 +792,13 @@ async function streamResponse({
     }
   }
 
-  return { answer: textAnswer, toolCalls: toolCalls.filter(Boolean), finish_reason, usage };
+  const { reasoningContent, content, finish_reason, usage } = getResponseData();
+
+  return {
+    reasoningContent,
+    answer: content,
+    toolCalls: toolCalls.filter(Boolean),
+    finish_reason,
+    usage
+  };
 }

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -556,30 +556,21 @@ async function streamResponse({
     res,
     readStream: stream
   });
-  let answer = '';
-  let reasoning = '';
-  let finish_reason: CompletionFinishReason = null;
-  let usage: CompletionUsage = getLLMDefaultUsage();
 
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
 
   for await (const part of stream) {
-    usage = part.usage || usage;
-
     if (res.closed) {
       stream.controller?.abort();
-      finish_reason = 'close';
+      updateFinishReason('close');
       break;
     }
 
-    const { reasoningContent, content, responseContent, finishReason } = parsePart({
+    const { reasoningContent, responseContent } = parsePart({
       part,
       parseThinkTag,
       retainDatasetCite
     });
-    finish_reason = finish_reason || finishReason;
-    answer += content;
-    reasoning += reasoningContent;
 
     if (aiChatReasoning && reasoningContent) {
       workflowStreamResponse?.({
@@ -602,5 +593,7 @@ async function streamResponse({
     }
   }
 
+  const { reasoningContent: reasoning, content: answer, finish_reason, usage } = getResponseData();
+
   return { answer, reasoning, finish_reason, usage };
 }