fix: stream response (#4853)

dd3c2516 · Archer · GitHub · aa55f059 · dd3c2516 · dd3c2516
Commit dd3c2516 authored May 21, 2025 by Archer Committed by GitHub May 21, 2025
6 changed files
--- a/docSite/content/zh-cn/docs/development/upgrading/4910.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/4910.md
@@ -19,4 +19,6 @@ weight: 790
 ## 🐛 修复
-1. 全文检索多知识库时排序得分排序不正确
+1. 全文检索多知识库时排序得分排序不正确。
\ No newline at end of file
+2. 流响应捕获 finish_reason 可能不正确。
+3. 工具调用模式，未保存思考输出。
\ No newline at end of file
--- a/packages/service/core/ai/utils.ts
+++ b/packages/service/core/ai/utils.ts
--- a/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts
+++ b/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts
 import { createChatCompletion } from '../../../../ai/config';
 import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../../chat/utils';
-import {
+import type {
-  type ChatCompletion,
+  ChatCompletion,
-  type StreamChatType,
+  StreamChatType,
-  type ChatCompletionMessageParam,
+  ChatCompletionMessageParam,
-  type ChatCompletionCreateParams,
+  ChatCompletionCreateParams,
-  type ChatCompletionMessageFunctionCall,
+  ChatCompletionMessageFunctionCall,
-  type ChatCompletionFunctionMessageParam,
+  ChatCompletionFunctionMessageParam,
-  type ChatCompletionAssistantMessageParam
+  ChatCompletionAssistantMessageParam,
+  CompletionFinishReason
 } from '@fastgpt/global/core/ai/type.d';
 import { type NextApiResponse } from 'next';
 import { responseWriteController } from '../../../../../common/response';
@@ -259,14 +260,15 @@ export const runToolWithFunctionCall = async (
    }
  });
-  let { answer, functionCalls, inputTokens, outputTokens } = await (async () => {
+  let { answer, functionCalls, inputTokens, outputTokens, finish_reason } = await (async () => {
    if (isStreamResponse) {
      if (!res || res.closed) {
        return {
          answer: '',
          functionCalls: [],
          inputTokens: 0,
-          outputTokens: 0
+          outputTokens: 0,
+          finish_reason: 'close' as const
        };
      }
      const result = await streamResponse({
@@ -281,10 +283,12 @@ export const runToolWithFunctionCall = async (
        answer: result.answer,
        functionCalls: result.functionCalls,
        inputTokens: result.usage.prompt_tokens,
-        outputTokens: result.usage.completion_tokens
+        outputTokens: result.usage.completion_tokens,
+        finish_reason: result.finish_reason
      };
    } else {
      const result = aiResponse as ChatCompletion;
+      const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
      const function_call = result.choices?.[0]?.message?.function_call;
      const usage = result.usage;
@@ -315,7 +319,8 @@ export const runToolWithFunctionCall = async (
        answer,
        functionCalls: toolCalls,
        inputTokens: usage?.prompt_tokens,
-        outputTokens: usage?.completion_tokens
+        outputTokens: usage?.completion_tokens,
+        finish_reason
      };
    }
  })();
@@ -481,7 +486,8 @@ export const runToolWithFunctionCall = async (
        completeMessages,
        assistantResponses: toolNodeAssistants,
        runTimes,
-        toolWorkflowInteractiveResponse
+        toolWorkflowInteractiveResponse,
+        finish_reason
      };
    }
@@ -495,7 +501,8 @@ export const runToolWithFunctionCall = async (
        toolNodeInputTokens,
        toolNodeOutputTokens,
        assistantResponses: toolNodeAssistants,
-        runTimes
+        runTimes,
+        finish_reason
      }
    );
  } else {
@@ -523,7 +530,8 @@ export const runToolWithFunctionCall = async (
        : outputTokens,
      completeMessages,
      assistantResponses: [...assistantResponses, ...toolNodeAssistant.value],
-      runTimes: (response?.runTimes || 0) + 1
+      runTimes: (response?.runTimes || 0) + 1,
+      finish_reason
    };
  }
 };
@@ -546,28 +554,25 @@ async function streamResponse({
    readStream: stream
  });
-  let textAnswer = '';
  let functionCalls: ChatCompletionMessageFunctionCall[] = [];
  let functionId = getNanoid();
-  let usage = getLLMDefaultUsage();
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
  for await (const part of stream) {
-    usage = part.usage || usage;
    if (res.closed) {
      stream.controller?.abort();
+      updateFinishReason('close');
      break;
    }
-    const { content: toolChoiceContent, responseContent } = parsePart({
+    const { responseContent } = parsePart({
      part,
      parseThinkTag: false,
      retainDatasetCite
    });
    const responseChoice = part.choices?.[0]?.delta;
-    textAnswer += toolChoiceContent;
    if (responseContent) {
      workflowStreamResponse?.({
@@ -577,7 +582,7 @@ async function streamResponse({
          text: responseContent
        })
      });
-    } else if (responseChoice.function_call) {
+    } else if (responseChoice?.function_call) {
      const functionCall: {
        arguments?: string;
        name?: string;
@@ -640,5 +645,7 @@ async function streamResponse({
    }
  }
-  return { answer: textAnswer, functionCalls, usage };
+  const { content, finish_reason, usage } = getResponseData();
+  return { answer: content, functionCalls, finish_reason, usage };
 }
--- a/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts
+++ b/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts
@@ -220,7 +220,8 @@ export const runToolWithPromptCall = async (
  const max_tokens = computedMaxToken({
    model: toolModel,
-    maxToken
+    maxToken,
+    min: 100
  });
  const filterMessages = await filterGPTMessageByMaxContext({
    messages,
@@ -592,28 +593,22 @@ async function streamResponse({
  let startResponseWrite = false;
  let answer = '';
-  let reasoning = '';
-  let finish_reason: CompletionFinishReason = null;
-  let usage = getLLMDefaultUsage();
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
  for await (const part of stream) {
-    usage = part.usage || usage;
    if (res.closed) {
      stream.controller?.abort();
-      finish_reason = 'close';
+      updateFinishReason('close');
      break;
    }
-    const { reasoningContent, content, responseContent, finishReason } = parsePart({
+    const { reasoningContent, content, responseContent } = parsePart({
      part,
      parseThinkTag: aiChatReasoning,
      retainDatasetCite
    });
-    finish_reason = finish_reason || finishReason;
    answer += content;
-    reasoning += reasoningContent;
    // Reasoning response
    if (aiChatReasoning && reasoningContent) {
@@ -658,7 +653,9 @@ async function streamResponse({
    }
  }
-  return { answer, reasoning, finish_reason, usage };
+  const { reasoningContent, content, finish_reason, usage } = getResponseData();
+  return { answer: content, reasoning: reasoningContent, finish_reason, usage };
 }
 const parseAnswer = (

--- a/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts
+++ b/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts
@@ -7,17 +7,13 @@ import {
  type ChatCompletionToolMessageParam,
  type ChatCompletionMessageParam,
  type ChatCompletionTool,
-  type ChatCompletionAssistantMessageParam,
  type CompletionFinishReason
 } from '@fastgpt/global/core/ai/type';
 import { type NextApiResponse } from 'next';
 import { responseWriteController } from '../../../../../common/response';
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
-import {
+import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
-  ChatCompletionRequestMessageRoleEnum,
-  getLLMDefaultUsage
-} from '@fastgpt/global/core/ai/constants';
 import { dispatchWorkFlow } from '../../index';
 import {
  type DispatchToolModuleProps,
@@ -254,7 +250,8 @@ export const runToolWithToolChoice = async (
  const max_tokens = computedMaxToken({
    model: toolModel,
-    maxToken
+    maxToken,
+    min: 100
  });
  // Filter histories by maxToken
@@ -319,97 +316,101 @@ export const runToolWithToolChoice = async (
    }
  });
-  let { answer, toolCalls, finish_reason, inputTokens, outputTokens } = await (async () => {
+  let { reasoningContent, answer, toolCalls, finish_reason, inputTokens, outputTokens } =
-    if (isStreamResponse) {
+    await (async () => {
-      if (!res || res.closed) {
+      if (isStreamResponse) {
+        if (!res || res.closed) {
+          return {
+            reasoningContent: '',
+            answer: '',
+            toolCalls: [],
+            finish_reason: 'close' as const,
+            inputTokens: 0,
+            outputTokens: 0
+          };
+        }
+        const result = await streamResponse({
+          res,
+          workflowStreamResponse,
+          toolNodes,
+          stream: aiResponse,
+          aiChatReasoning,
+          retainDatasetCite
+        });
        return {
-          answer: '',
+          reasoningContent: result.reasoningContent,
-          toolCalls: [],
+          answer: result.answer,
-          finish_reason: 'close' as const,
+          toolCalls: result.toolCalls,
-          inputTokens: 0,
+          finish_reason: result.finish_reason,
-          outputTokens: 0
+          inputTokens: result.usage.prompt_tokens,
+          outputTokens: result.usage.completion_tokens
        };
-      }
+      } else {
+        const result = aiResponse as ChatCompletion;
+        const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
+        const calls = result.choices?.[0]?.message?.tool_calls || [];
+        const answer = result.choices?.[0]?.message?.content || '';
+        // @ts-ignore
+        const reasoningContent = result.choices?.[0]?.message?.reasoning_content || '';
+        const usage = result.usage;
+        if (aiChatReasoning && reasoningContent) {
+          workflowStreamResponse?.({
+            event: SseResponseEventEnum.fastAnswer,
+            data: textAdaptGptResponse({
+              reasoning_content: removeDatasetCiteText(reasoningContent, retainDatasetCite)
+            })
+          });
+        }
-      const result = await streamResponse({
+        // 格式化 toolCalls
-        res,
+        const toolCalls = calls.map((tool) => {
-        workflowStreamResponse,
+          const toolNode = toolNodes.find((item) => item.nodeId === tool.function?.name);
-        toolNodes,
-        stream: aiResponse,
+          // 不支持 stream 模式的模型的这里需要补一个响应给客户端
-        aiChatReasoning,
+          workflowStreamResponse?.({
-        retainDatasetCite
+            event: SseResponseEventEnum.toolCall,
-      });
+            data: {
+              tool: {
+                id: tool.id,
+                toolName: toolNode?.name || '',
+                toolAvatar: toolNode?.avatar || '',
+                functionName: tool.function.name,
+                params: tool.function?.arguments ?? '',
+                response: ''
+              }
+            }
+          });
-      return {
+          return {
-        answer: result.answer,
+            ...tool,
-        toolCalls: result.toolCalls,
+            toolName: toolNode?.name || '',
-        finish_reason: result.finish_reason,
+            toolAvatar: toolNode?.avatar || ''
-        inputTokens: result.usage.prompt_tokens,
+          };
-        outputTokens: result.usage.completion_tokens
-      };
-    } else {
-      const result = aiResponse as ChatCompletion;
-      const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
-      const calls = result.choices?.[0]?.message?.tool_calls || [];
-      const answer = result.choices?.[0]?.message?.content || '';
-      // @ts-ignore
-      const reasoningContent = result.choices?.[0]?.message?.reasoning_content || '';
-      const usage = result.usage;
-      if (aiChatReasoning && reasoningContent) {
-        workflowStreamResponse?.({
-          event: SseResponseEventEnum.fastAnswer,
-          data: textAdaptGptResponse({
-            reasoning_content: removeDatasetCiteText(reasoningContent, retainDatasetCite)
-          })
        });
-      }
-      // 格式化 toolCalls
+        if (answer) {
-      const toolCalls = calls.map((tool) => {
+          workflowStreamResponse?.({
-        const toolNode = toolNodes.find((item) => item.nodeId === tool.function?.name);
+            event: SseResponseEventEnum.fastAnswer,
+            data: textAdaptGptResponse({
-        // 不支持 stream 模式的模型的这里需要补一个响应给客户端
+              text: removeDatasetCiteText(answer, retainDatasetCite)
-        workflowStreamResponse?.({
+            })
-          event: SseResponseEventEnum.toolCall,
+          });
-          data: {
+        }
-            tool: {
-              id: tool.id,
-              toolName: toolNode?.name || '',
-              toolAvatar: toolNode?.avatar || '',
-              functionName: tool.function.name,
-              params: tool.function?.arguments ?? '',
-              response: ''
-            }
-          }
-        });
        return {
-          ...tool,
+          reasoningContent: (reasoningContent as string) || '',
-          toolName: toolNode?.name || '',
+          answer,
-          toolAvatar: toolNode?.avatar || ''
+          toolCalls: toolCalls,
+          finish_reason,
+          inputTokens: usage?.prompt_tokens,
+          outputTokens: usage?.completion_tokens
        };
-      });
-      if (answer) {
-        workflowStreamResponse?.({
-          event: SseResponseEventEnum.fastAnswer,
-          data: textAdaptGptResponse({
-            text: removeDatasetCiteText(answer, retainDatasetCite)
-          })
-        });
      }
+    })();
-      return {
+  if (!answer && !reasoningContent && toolCalls.length === 0) {
-        answer,
-        toolCalls: toolCalls,
-        finish_reason,
-        inputTokens: usage?.prompt_tokens,
-        outputTokens: usage?.completion_tokens
-      };
-    }
-  })();
-  if (!answer && toolCalls.length === 0) {
    return Promise.reject(getEmptyResponseTip());
  }
@@ -501,12 +502,13 @@ export const runToolWithToolChoice = async (
  if (toolCalls.length > 0) {
    // Run the tool, combine its results, and perform another round of AI calls
-    const assistantToolMsgParams: ChatCompletionAssistantMessageParam[] = [
+    const assistantToolMsgParams: ChatCompletionMessageParam[] = [
-      ...(answer
+      ...(answer || reasoningContent
        ? [
            {
              role: ChatCompletionRequestMessageRoleEnum.Assistant as 'assistant',
-              content: answer
+              content: answer,
+              reasoning_text: reasoningContent
            }
          ]
        : []),
@@ -627,9 +629,10 @@ export const runToolWithToolChoice = async (
    );
  } else {
    // No tool is invoked, indicating that the process is over
-    const gptAssistantResponse: ChatCompletionAssistantMessageParam = {
+    const gptAssistantResponse: ChatCompletionMessageParam = {
      role: ChatCompletionRequestMessageRoleEnum.Assistant,
-      content: answer
+      content: answer,
+      reasoning_text: reasoningContent
    };
    const completeMessages = filterMessages.concat(gptAssistantResponse);
    inputTokens = inputTokens || (await countGptMessagesTokens(requestMessages, tools));
@@ -671,34 +674,23 @@ async function streamResponse({
    readStream: stream
  });
-  let textAnswer = '';
  let callingTool: { name: string; arguments: string } | null = null;
  let toolCalls: ChatCompletionMessageToolCall[] = [];
-  let finish_reason: CompletionFinishReason = null;
-  let usage = getLLMDefaultUsage();
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
  for await (const part of stream) {
-    usage = part.usage || usage;
    if (res.closed) {
      stream.controller?.abort();
-      finish_reason = 'close';
+      updateFinishReason('close');
      break;
    }
-    const {
+    const { reasoningContent, responseContent } = parsePart({
-      reasoningContent,
-      content: toolChoiceContent,
-      responseContent,
-      finishReason
-    } = parsePart({
      part,
      parseThinkTag: true,
      retainDatasetCite
    });
-    textAnswer += toolChoiceContent;
-    finish_reason = finishReason || finish_reason;
    const responseChoice = part.choices?.[0]?.delta;
@@ -800,5 +792,13 @@ async function streamResponse({
    }
  }
-  return { answer: textAnswer, toolCalls: toolCalls.filter(Boolean), finish_reason, usage };
+  const { reasoningContent, content, finish_reason, usage } = getResponseData();
+  return {
+    reasoningContent,
+    answer: content,
+    toolCalls: toolCalls.filter(Boolean),
+    finish_reason,
+    usage
+  };
 }
--- a/packages/service/core/workflow/dispatch/chat/oneapi.ts
+++ b/packages/service/core/workflow/dispatch/chat/oneapi.ts
@@ -556,30 +556,21 @@ async function streamResponse({
    res,
    readStream: stream
  });
-  let answer = '';
-  let reasoning = '';
-  let finish_reason: CompletionFinishReason = null;
-  let usage: CompletionUsage = getLLMDefaultUsage();
-  const { parsePart } = parseLLMStreamResponse();
+  const { parsePart, getResponseData, updateFinishReason } = parseLLMStreamResponse();
  for await (const part of stream) {
-    usage = part.usage || usage;
    if (res.closed) {
      stream.controller?.abort();
-      finish_reason = 'close';
+      updateFinishReason('close');
      break;
    }
-    const { reasoningContent, content, responseContent, finishReason } = parsePart({
+    const { reasoningContent, responseContent } = parsePart({
      part,
      parseThinkTag,
      retainDatasetCite
    });
-    finish_reason = finish_reason || finishReason;
-    answer += content;
-    reasoning += reasoningContent;
    if (aiChatReasoning && reasoningContent) {
      workflowStreamResponse?.({
@@ -602,5 +593,7 @@ async function streamResponse({
    }
  }
+  const { reasoningContent: reasoning, content: answer, finish_reason, usage } = getResponseData();
  return { answer, reasoning, finish_reason, usage };
 }