Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 4d20274a by Archer Committed by GitHub
Options

feat: think tag parse (#3805) (#3808) 

* feat: think tag parse

* remove some model config

* feat: parse think tag test
parent 4447e403
Showing with 414 additions and 531 deletions
docSite/content/zh-cn/docs/development/upgrading/490.md 0 → 100644
---
title: 'V4.9.0(进行中)'
description: 'FastGPT V4.9.0 更新说明'
icon: 'upgrade'
draft: false
toc: true
weight: 803
---
## 完整更新内容
1. 新增 - AI 对话节点解析 <think></think> 标签内容，便于各类模型进行思考链输出。
2. 修复 - 思考链流输出时，有时与正文顺序偏差。
\ No newline at end of file
packages/global/core/ai/type.d.ts
import openai from 'openai'; import openai from 'openai';
import type { import type {
ChatCompletionMessageToolCall, ChatCompletionMessageToolCall,
ChatCompletionChunk,
ChatCompletionMessageParam as SdkChatCompletionMessageParam, ChatCompletionMessageParam as SdkChatCompletionMessageParam,
ChatCompletionToolMessageParam, ChatCompletionToolMessageParam,
ChatCompletionContentPart as SdkChatCompletionContentPart, ChatCompletionContentPart as SdkChatCompletionContentPart,
ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam, ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam,
ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam, ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam,
ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam, ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam
ChatCompletionContentPartText
} from 'openai/resources'; } from 'openai/resources';
import { ChatMessageTypeEnum } from './constants'; import { ChatMessageTypeEnum } from './constants';
import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type'; import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
...@@ -71,7 +69,8 @@ export type ChatCompletionMessageFunctionCall = ...@@ -71,7 +69,8 @@ export type ChatCompletionMessageFunctionCall =
}; };
// Stream response // Stream response
export type StreamChatType = Stream<ChatCompletionChunk>; export type StreamChatType = Stream<openai.Chat.Completions.ChatCompletionChunk>;
export type UnStreamChatType = openai.Chat.Completions.ChatCompletion;
export default openai; export default openai;
export * from 'openai'; export * from 'openai';
......
packages/global/core/workflow/runtime/utils.ts
...@@ -10,6 +10,7 @@ import { FlowNodeOutputItemType, ReferenceValueType } from '../type/io'; ...@@ -10,6 +10,7 @@ import { FlowNodeOutputItemType, ReferenceValueType } from '../type/io';
import { ChatItemType, NodeOutputItemType } from '../../../core/chat/type'; import { ChatItemType, NodeOutputItemType } from '../../../core/chat/type';
import { ChatItemValueTypeEnum, ChatRoleEnum } from '../../../core/chat/constants'; import { ChatItemValueTypeEnum, ChatRoleEnum } from '../../../core/chat/constants';
import { replaceVariable, valToStr } from '../../../common/string/tools'; import { replaceVariable, valToStr } from '../../../common/string/tools';
import { ChatCompletionChunk } from 'openai/resources';
export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number => { export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number => {
let limit = 10; let limit = 10;
...@@ -419,3 +420,137 @@ export function rewriteNodeOutputByHistories( ...@@ -419,3 +420,137 @@ export function rewriteNodeOutputByHistories(
}; };
}); });
} }
// Parse <think></think> tags to think and answer - unstream response
export const parseReasoningContent = (text: string): [string, string] => {
const regex = /<think>([\s\S]*?)<\/think>/;
const match = text.match(regex);
if (!match) {
return ['', text];
}
const thinkContent = match[1].trim();
// Add answer (remaining text after think tag)
const answerContent = text.slice(match.index! + match[0].length);
return [thinkContent, answerContent];
};
// Parse <think></think> tags to think and answer - stream response
export const parseReasoningStreamContent = () => {
let isInThinkTag: boolean | undefined;
const startTag = '<think>';
let startTagBuffer = '';
const endTag = '</think>';
let endTagBuffer = '';
/*
parseReasoning - 只控制是否主动解析 <think></think>，如果接口已经解析了，仍然会返回 think 内容。
*/
const parsePart = (
part: {
choices: {
delta: {
content?: string;
reasoning_content?: string;
};
}[];
},
parseReasoning = false
): [string, string] => {
const content = part.choices?.[0]?.delta?.content || '';
// @ts-ignore
const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
if (reasoningContent || !parseReasoning) {
isInThinkTag = false;
return [reasoningContent, content];
}
if (!content) {
return ['', ''];
}
// 如果不在 think 标签中，或者有 reasoningContent(接口已解析），则返回 reasoningContent 和 content
if (isInThinkTag === false) {
return ['', content];
}
// 检测是否为 think 标签开头的数据
if (isInThinkTag === undefined) {
// Parse content think and answer
startTagBuffer += content;
// 太少内容时候，暂时不解析
if (startTagBuffer.length < startTag.length) {
return ['', ''];
}
if (startTagBuffer.startsWith(startTag)) {
isInThinkTag = true;
return [startTagBuffer.slice(startTag.length), ''];
}
// 如果未命中 think 标签，则认为不在 think 标签中，返回 buffer 内容作为 content
isInThinkTag = false;
return ['', startTagBuffer];
}
// 确认是 think 标签内容，开始返回 think 内容，并实时检测 </think>
/*
检测 </think> 方案。
存储所有疑似 </think> 的内容，直到检测到完整的 </think> 标签或超出 </think> 长度。
content 返回值包含以下几种情况:
abc - 完全未命中尾标签
abc<th - 命中一部分尾标签
abc</think> - 完全命中尾标签
abc</think>abc - 完全命中尾标签
</think>abc - 完全命中尾标签
k>abc - 命中一部分尾标签
*/
// endTagBuffer 专门用来记录疑似尾标签的内容
if (endTagBuffer) {
endTagBuffer += content;
if (endTagBuffer.includes(endTag)) {
isInThinkTag = false;
const answer = endTagBuffer.slice(endTag.length);
return ['', answer];
} else if (endTagBuffer.length >= endTag.length) {
// 缓存内容超出尾标签长度，且仍未命中 </think>，则认为本次猜测 </think> 失败，仍处于 think 阶段。
const tmp = endTagBuffer;
endTagBuffer = '';
return [tmp, ''];
}
return ['', ''];
} else if (content.includes(endTag)) {
// 返回内容，完整命中</think>，直接结束
isInThinkTag = false;
const [think, answer] = content.split(endTag);
return [think, answer];
} else {
// 无 buffer，且未命中 </think>，开始疑似 </think> 检测。
for (let i = 1; i < endTag.length; i++) {
const partialEndTag = endTag.slice(0, i);
// 命中一部分尾标签
if (content.endsWith(partialEndTag)) {
const think = content.slice(0, -partialEndTag.length);
endTagBuffer += partialEndTag;
return [think, ''];
}
}
}
// 完全未命中尾标签，还是 think 阶段。
return [content, ''];
};
const getStartTagBuffer = () => startTagBuffer;
return {
parsePart,
getStartTagBuffer
};
};
packages/service/core/ai/config.ts
import OpenAI from '@fastgpt/global/core/ai'; import OpenAI from '@fastgpt/global/core/ai';
import { import {
ChatCompletionCreateParamsNonStreaming, ChatCompletionCreateParamsNonStreaming,
ChatCompletionCreateParamsStreaming ChatCompletionCreateParamsStreaming,
StreamChatType,
UnStreamChatType
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/type';
import { getErrText } from '@fastgpt/global/common/error/utils'; import { getErrText } from '@fastgpt/global/common/error/utils';
import { addLog } from '../../common/system/log'; import { addLog } from '../../common/system/log';
...@@ -38,29 +40,30 @@ export const getAxiosConfig = (props?: { userKey?: OpenaiAccountType }) => { ...@@ -38,29 +40,30 @@ export const getAxiosConfig = (props?: { userKey?: OpenaiAccountType }) => {
}; };
}; };
type CompletionsBodyType = export const createChatCompletion = async ({
| ChatCompletionCreateParamsNonStreaming
| ChatCompletionCreateParamsStreaming;
type InferResponseType<T extends CompletionsBodyType> =
T extends ChatCompletionCreateParamsStreaming
? OpenAI.Chat.Completions.ChatCompletionChunk
: OpenAI.Chat.Completions.ChatCompletion;
export const createChatCompletion = async <T extends CompletionsBodyType>({
body, body,
userKey, userKey,
timeout, timeout,
options options
}: { }: {
body: T; body: ChatCompletionCreateParamsNonStreaming | ChatCompletionCreateParamsStreaming;
userKey?: OpenaiAccountType; userKey?: OpenaiAccountType;
timeout?: number; timeout?: number;
options?: OpenAI.RequestOptions; options?: OpenAI.RequestOptions;
}): Promise<{ }): Promise<
response: InferResponseType<T>; {
isStreamResponse: boolean; getEmptyResponseTip: () => string;
getEmptyResponseTip: () => string; } & (
}> => { | {
response: StreamChatType;
isStreamResponse: true;
}
| {
response: UnStreamChatType;
isStreamResponse: false;
}
)
> => {
try { try {
const modelConstantsData = getLLMModel(body.model); const modelConstantsData = getLLMModel(body.model);
...@@ -96,9 +99,17 @@ export const createChatCompletion = async <T extends CompletionsBodyType>({ ...@@ -96,9 +99,17 @@ export const createChatCompletion = async <T extends CompletionsBodyType>({
return i18nT('chat:LLM_model_response_empty'); return i18nT('chat:LLM_model_response_empty');
}; };
if (isStreamResponse) {
return {
response,
isStreamResponse: true,
getEmptyResponseTip
};
}
return { return {
response: response as InferResponseType<T>, response,
isStreamResponse, isStreamResponse: false,
getEmptyResponseTip getEmptyResponseTip
}; };
} catch (error) { } catch (error) {
......
packages/service/core/ai/config/provider/PPIO.json
{ {
"provider": "PPIO", "provider": "PPIO",
"list": [ "list": []
{ }
"model": "deepseek/deepseek-r1/community",
"name": "deepseek/deepseek-r1/community",
"maxContext": 64000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-v3/community",
"name": "deepseek/deepseek-v3/community",
"maxContext": 64000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-r1",
"name": "deepseek/deepseek-r1",
"maxContext": 64000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-v3",
"name": "deepseek/deepseek-v3",
"maxContext": 64000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-r1-distill-llama-70b",
"name": "deepseek/deepseek-r1-distill-llama-70b",
"maxContext": 32000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-r1-distill-qwen-32b",
"name": "deepseek/deepseek-r1-distill-qwen-32b",
"maxContext": 64000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-r1-distill-qwen-14b",
"name": "deepseek/deepseek-r1-distill-qwen-14b",
"maxContext": 64000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "deepseek/deepseek-r1-distill-llama-8b",
"name": "deepseek/deepseek-r1-distill-llama-8b",
"maxContext": 32000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "qwen/qwen-2.5-72b-instruct",
"name": "qwen/qwen-2.5-72b-instruct",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "qwen/qwen-2-vl-72b-instruct",
"name": "qwen/qwen-2-vl-72b-instruct",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "meta-llama/llama-3.2-3b-instruct",
"name": "meta-llama/llama-3.2-3b-instruct",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "qwen/qwen2.5-32b-instruct",
"name": "qwen/qwen2.5-32b-instruct",
"maxContext": 32000,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "baichuan/baichuan2-13b-chat",
"name": "baichuan/baichuan2-13b-chat",
"maxContext": 14336,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "meta-llama/llama-3.1-70b-instruct",
"name": "meta-llama/llama-3.1-70b-instruct",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "meta-llama/llama-3.1-8b-instruct",
"name": "meta-llama/llama-3.1-8b-instruct",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "01-ai/yi-1.5-34b-chat",
"name": "01-ai/yi-1.5-34b-chat",
"maxContext": 16384,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "01-ai/yi-1.5-9b-chat",
"name": "01-ai/yi-1.5-9b-chat",
"maxContext": 16384,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "thudm/glm-4-9b-chat",
"name": "thudm/glm-4-9b-chat",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
},
{
"model": "qwen/qwen-2-7b-instruct",
"name": "qwen/qwen-2-7b-instruct",
"maxContext": 32768,
"maxResponse": 8000,
"quoteMaxToken": 50000,
"maxTemperature": 2,
"vision": false,
"toolChoice": false,
"functionCall": false,
"defaultSystemChatPrompt": "",
"datasetProcess": true,
"usedInClassify": true,
"customCQPrompt": "",
"usedInExtractFields": true,
"usedInQueryExtension": true,
"customExtractPrompt": "",
"usedInToolCall": true,
"defaultConfig": {},
"fieldMap": {},
"type": "llm",
"showTopP": true,
"showStopSign": true
}
]
}
\ No newline at end of file
packages/service/core/ai/utils.ts
...@@ -37,25 +37,26 @@ export const computedTemperature = ({ ...@@ -37,25 +37,26 @@ export const computedTemperature = ({
return temperature; return temperature;
}; };
type CompletionsBodyType = ( type CompletionsBodyType =
| ChatCompletionCreateParamsNonStreaming | ChatCompletionCreateParamsNonStreaming
| ChatCompletionCreateParamsStreaming | ChatCompletionCreateParamsStreaming;
) & {
response_format?: any;
json_schema?: string;
stop?: string;
};
type InferCompletionsBody<T> = T extends { stream: true } type InferCompletionsBody<T> = T extends { stream: true }
? ChatCompletionCreateParamsStreaming ? ChatCompletionCreateParamsStreaming
: ChatCompletionCreateParamsNonStreaming; : T extends { stream: false }
? ChatCompletionCreateParamsNonStreaming
: ChatCompletionCreateParamsNonStreaming | ChatCompletionCreateParamsStreaming;
export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>( export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>(
body: T, body: T & {
response_format?: any;
json_schema?: string;
stop?: string;
},
model: string | LLMModelItemType model: string | LLMModelItemType
): InferCompletionsBody<T> => { ): InferCompletionsBody<T> => {
const modelData = typeof model === 'string' ? getLLMModel(model) : model; const modelData = typeof model === 'string' ? getLLMModel(model) : model;
if (!modelData) { if (!modelData) {
return body as InferCompletionsBody<T>; return body as unknown as InferCompletionsBody<T>;
} }
const response_format = body.response_format; const response_format = body.response_format;
...@@ -91,9 +92,7 @@ export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>( ...@@ -91,9 +92,7 @@ export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>(
}); });
} }
// console.log(requestBody); return requestBody as unknown as InferCompletionsBody<T>;
return requestBody as InferCompletionsBody<T>;
}; };
export const llmStreamResponseToText = async (response: StreamChatType) => { export const llmStreamResponseToText = async (response: StreamChatType) => {
......
packages/service/core/workflow/dispatch/chat/oneapi.ts
...@@ -3,13 +3,13 @@ import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../chat ...@@ -3,13 +3,13 @@ import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../chat
import type { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type.d'; import type { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type.d';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils'; import {
parseReasoningContent,
parseReasoningStreamContent,
textAdaptGptResponse
} from '@fastgpt/global/core/workflow/runtime/utils';
import { createChatCompletion } from '../../../ai/config'; import { createChatCompletion } from '../../../ai/config';
import type { import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
ChatCompletion,
ChatCompletionMessageParam,
StreamChatType
} from '@fastgpt/global/core/ai/type.d';
import { formatModelChars2Points } from '../../../../support/wallet/usage/utils'; import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
import { postTextCensor } from '../../../../common/api/requestPlusApi'; import { postTextCensor } from '../../../../common/api/requestPlusApi';
...@@ -195,7 +195,13 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp ...@@ -195,7 +195,13 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
}); });
const { answerText, reasoningText } = await (async () => { const { answerText, reasoningText } = await (async () => {
if (res && isStreamResponse) { if (isStreamResponse) {
if (!res) {
return {
answerText: '',
reasoningText: ''
};
}
// sse response // sse response
const { answer, reasoning } = await streamResponse({ const { answer, reasoning } = await streamResponse({
res, res,
...@@ -210,34 +216,49 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp ...@@ -210,34 +216,49 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
reasoningText: reasoning reasoningText: reasoning
}; };
} else { } else {
const unStreamResponse = response as ChatCompletion; const { content, reasoningContent } = (() => {
const answer = unStreamResponse.choices?.[0]?.message?.content || ''; const content = response.choices?.[0]?.message?.content || '';
// @ts-ignore // @ts-ignore
const reasoning = unStreamResponse.choices?.[0]?.message?.reasoning_content || ''; const reasoningContent: string = response.choices?.[0]?.message?.reasoning_content || '';
// API already parse reasoning content
if (reasoningContent || !aiChatReasoning) {
return {
content,
reasoningContent
};
}
const [think, answer] = parseReasoningContent(content);
return {
content: answer,
reasoningContent: think
};
})();
// Some models do not support streaming // Some models do not support streaming
if (stream) { if (stream) {
if (isResponseAnswerText && answer) { if (aiChatReasoning && reasoningContent) {
workflowStreamResponse?.({ workflowStreamResponse?.({
event: SseResponseEventEnum.fastAnswer, event: SseResponseEventEnum.fastAnswer,
data: textAdaptGptResponse({ data: textAdaptGptResponse({
text: answer reasoning_content: reasoningContent
}) })
}); });
} }
if (aiChatReasoning && reasoning) { if (isResponseAnswerText && content) {
workflowStreamResponse?.({ workflowStreamResponse?.({
event: SseResponseEventEnum.fastAnswer, event: SseResponseEventEnum.fastAnswer,
data: textAdaptGptResponse({ data: textAdaptGptResponse({
reasoning_content: reasoning text: content
}) })
}); });
} }
} }
return { return {
answerText: answer, answerText: content,
reasoningText: reasoning reasoningText: reasoningContent
}; };
} }
})(); })();
...@@ -267,7 +288,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp ...@@ -267,7 +288,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
}); });
return { return {
answerText, answerText: answerText.trim(),
reasoningText, reasoningText,
[DispatchNodeResponseKeyEnum.nodeResponse]: { [DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: externalProvider.openaiAccount?.key ? 0 : totalPoints, totalPoints: externalProvider.openaiAccount?.key ? 0 : totalPoints,
...@@ -500,36 +521,43 @@ async function streamResponse({ ...@@ -500,36 +521,43 @@ async function streamResponse({
}); });
let answer = ''; let answer = '';
let reasoning = ''; let reasoning = '';
const { parsePart, getStartTagBuffer } = parseReasoningStreamContent();
for await (const part of stream) { for await (const part of stream) {
if (res.closed) { if (res.closed) {
stream.controller?.abort(); stream.controller?.abort();
break; break;
} }
const content = part.choices?.[0]?.delta?.content || ''; const [reasoningContent, content] = parsePart(part, aiChatReasoning);
answer += content; answer += content;
if (isResponseAnswerText && content) { reasoning += reasoningContent;
if (aiChatReasoning && reasoningContent) {
workflowStreamResponse?.({ workflowStreamResponse?.({
write, write,
event: SseResponseEventEnum.answer, event: SseResponseEventEnum.answer,
data: textAdaptGptResponse({ data: textAdaptGptResponse({
text: content reasoning_content: reasoningContent
}) })
}); });
} }
const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || ''; if (isResponseAnswerText && content) {
reasoning += reasoningContent;
if (aiChatReasoning && reasoningContent) {
workflowStreamResponse?.({ workflowStreamResponse?.({
write, write,
event: SseResponseEventEnum.answer, event: SseResponseEventEnum.answer,
data: textAdaptGptResponse({ data: textAdaptGptResponse({
reasoning_content: reasoningContent text: content
}) })
}); });
} }
} }
// if answer is empty, try to get value from startTagBuffer. (Cause: The response content is too short to exceed the minimum parse length)
if (answer === '') {
answer = getStartTagBuffer();
}
return { answer, reasoning }; return { answer, reasoning };
} }
packages/service/core/workflow/dispatch/index.ts
...@@ -243,12 +243,17 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons ...@@ -243,12 +243,17 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
chatAssistantResponse = chatAssistantResponse.concat(assistantResponses); chatAssistantResponse = chatAssistantResponse.concat(assistantResponses);
} else { } else {
if (reasoningText) { if (reasoningText) {
chatAssistantResponse.push({ const isResponseReasoningText = inputs.find(
type: ChatItemValueTypeEnum.reasoning, (item) => item.key === NodeInputKeyEnum.aiChatReasoning
reasoning: { )?.value;
content: reasoningText if (isResponseReasoningText) {
} chatAssistantResponse.push({
}); type: ChatItemValueTypeEnum.reasoning,
reasoning: {
content: reasoningText
}
});
}
} }
if (answerText) { if (answerText) {
// save assistant text response // save assistant text response
......
projects/app/src/pages/api/v1/chat/utils.test.ts 0 → 100644
import '@/pages/api/__mocks__/base';
import { parseReasoningStreamContent } from '@fastgpt/global/core/workflow/runtime/utils';
test('Parse reasoning stream content test', async () => {
const partList = [
{
data: [{ content: '你好1' }, { content: '你好2' }, { content: '你好3' }],
correct: { answer: '你好1你好2你好3', reasoning: '' }
},
{
data: [
{ reasoning_content: '这是' },
{ reasoning_content: '思考' },
{ reasoning_content: '过程' },
{ content: '你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<t' },
{ content: 'hink>' },
{ content: '这是' },
{ content: '思考' },
{ content: '过程' },
{ content: '</think>' },
{ content: '你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<think>' },
{ content: '这是' },
{ content: '思考' },
{ content: '过程' },
{ content: '</think>' },
{ content: '你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<think>这是' },
{ content: '思考' },
{ content: '过程' },
{ content: '</think>' },
{ content: '你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<think>这是' },
{ content: '思考' },
{ content: '过程</' },
{ content: 'think>' },
{ content: '你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<think>这是' },
{ content: '思考' },
{ content: '过程</think>' },
{ content: '你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<think>这是' },
{ content: '思考' },
{ content: '过程</think>你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程' }
},
{
data: [
{ content: '<think>这是' },
{ content: '思考' },
{ content: '过程</th' },
{ content: '假的' },
{ content: '你好2' },
{ content: '你好3' },
{ content: '过程</think>你好1' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '你好1你好2你好3', reasoning: '这是思考过程</th假的你好2你好3过程' }
},
{
data: [
{ content: '<think>这是' },
{ content: '思考' },
{ content: '过程</th' },
{ content: '假的' },
{ content: '你好2' },
{ content: '你好3' }
],
correct: { answer: '', reasoning: '这是思考过程</th假的你好2你好3' }
}
];
partList.forEach((part) => {
const { parsePart } = parseReasoningStreamContent();
let answer = '';
let reasoning = '';
part.data.forEach((item) => {
const formatPart = {
choices: [
{
delta: {
role: 'assistant',
content: item.content,
reasoning_content: item.reasoning_content
}
}
]
};
const [reasoningContent, content] = parsePart(formatPart, true);
answer += content;
reasoning += reasoningContent;
});
expect(answer).toBe(part.correct.answer);
expect(reasoning).toBe(part.correct.reasoning);
});
});
projects/app/src/web/common/api/fetch.ts
...@@ -24,7 +24,11 @@ export type StreamResponseType = { ...@@ -24,7 +24,11 @@ export type StreamResponseType = {
[DispatchNodeResponseKeyEnum.nodeResponse]: ChatHistoryItemResType[]; [DispatchNodeResponseKeyEnum.nodeResponse]: ChatHistoryItemResType[];
}; };
type ResponseQueueItemType = type ResponseQueueItemType =
| { event: SseResponseEventEnum.fastAnswer | SseResponseEventEnum.answer; text: string } | {
event: SseResponseEventEnum.fastAnswer | SseResponseEventEnum.answer;
text?: string;
reasoningText?: string;
}
| { event: SseResponseEventEnum.interactive; [key: string]: any } | { event: SseResponseEventEnum.interactive; [key: string]: any }
| { | {
event: event:
...@@ -79,7 +83,7 @@ export const streamFetch = ({ ...@@ -79,7 +83,7 @@ export const streamFetch = ({
if (abortCtrl.signal.aborted) { if (abortCtrl.signal.aborted) {
responseQueue.forEach((item) => { responseQueue.forEach((item) => {
onMessage(item); onMessage(item);
if (isAnswerEvent(item.event)) { if (isAnswerEvent(item.event) && item.text) {
responseText += item.text; responseText += item.text;
} }
}); });
...@@ -91,7 +95,7 @@ export const streamFetch = ({ ...@@ -91,7 +95,7 @@ export const streamFetch = ({
for (let i = 0; i < fetchCount; i++) { for (let i = 0; i < fetchCount; i++) {
const item = responseQueue[i]; const item = responseQueue[i];
onMessage(item); onMessage(item);
if (isAnswerEvent(item.event)) { if (isAnswerEvent(item.event) && item.text) {
responseText += item.text; responseText += item.text;
} }
} }
...@@ -180,7 +184,7 @@ export const streamFetch = ({ ...@@ -180,7 +184,7 @@ export const streamFetch = ({
// console.log(parseJson, event); // console.log(parseJson, event);
if (event === SseResponseEventEnum.answer) { if (event === SseResponseEventEnum.answer) {
const reasoningText = parseJson.choices?.[0]?.delta?.reasoning_content || ''; const reasoningText = parseJson.choices?.[0]?.delta?.reasoning_content || '';
onMessage({ pushDataToQueue({
event, event,
reasoningText reasoningText
}); });
...@@ -194,7 +198,7 @@ export const streamFetch = ({ ...@@ -194,7 +198,7 @@ export const streamFetch = ({
} }
} else if (event === SseResponseEventEnum.fastAnswer) { } else if (event === SseResponseEventEnum.fastAnswer) {
const reasoningText = parseJson.choices?.[0]?.delta?.reasoning_content || ''; const reasoningText = parseJson.choices?.[0]?.delta?.reasoning_content || '';
onMessage({ pushDataToQueue({
event, event,
reasoningText reasoningText
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment