v4.4.7-2 (#388)

bcf94919 · Archer · GitHub · d0041a98 · bcf94919 · bcf94919
Commit bcf94919 authored Oct 11, 2023 by Archer Committed by GitHub Oct 11, 2023
51 changed files
--- a/docSite/assets/imgs/getfile_id.png
+++ b/docSite/assets/imgs/getfile_id.png
--- a/docSite/content/docs/development/configuration.md
+++ b/docSite/content/docs/development/configuration.md
@@ -77,7 +77,7 @@ weight: 520
    "price": 0,
    "prompt": ""
  },
-  "CQModel": { // 问题分类模型
+  "CQModel": { // Classify Question: 问题分类模型
    "model": "gpt-3.5-turbo-16k",
    "functionCall": true,
    "name": "GPT35-16k",
@@ -85,7 +85,7 @@ weight: 520
    "price": 0,
    "prompt": ""
  },
-  "QGModel": { // 生成下一步指引模型
+  "QGModel": { // Question Generation: 生成下一步指引模型
    "model": "gpt-3.5-turbo",
    "name": "GPT35-4k",
    "maxToken": 4000,

--- a/docSite/content/docs/development/openApi.md
+++ b/docSite/content/docs/development/openApi.md
@@ -225,9 +225,10 @@ data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2
 此部分 API 需使用全局通用的 API Key。
 {{% /alert %}}
-### 如何获取知识库ID（kbId）
+| 如何获取知识库ID（kbId） | 如何获取文件ID（file_id） |
+| --------------------- | --------------------- |
+| ![](/imgs/getKbId.png) | ![](/imgs/getfile_id.png) |
-![](/imgs/getKbId.png)
 ### 知识库添加数据
@@ -248,6 +249,8 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa
         {
             "a": "test",
             "q": "1111",
+            "file_id": "关联的文件ID/URL/manual/mark",
+            "source": "来源名称"，
         },
         {
             "a": "test2",
@@ -271,7 +274,8 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa
    "data": [
        {
            "q": "生成索引的内容，index 模式下最大 tokens 为3000，建议不超过 1000",
-            "a": "预期回答/补充"
+            "a": "预期回答/补充",
+            "file_id": "如果推送数据到手动录入，这里可以留空; 如果希望关联到某个文件中，需要填写对应文件的ID; 如果希望加入到手动标注中，可设置为: mark",
        },
        {
            "q": "生成索引的内容，qa 模式下最大 tokens 为10000，建议 8000 左右",
@@ -292,7 +296,16 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa
    "code": 200,
    "statusText": "",
    "data": {
-        "insertLen": 1 // 最终插入成功的数量，可能因为超出 tokens 或者插入异常，index 可以重复插入，会自动去重
+        "insertLen": 1, // 最终插入成功的数量
+        "overToken": [], // 超出 token 的
+        "fileIdInvalid": [  // file_id 无效的
+            {
+                "a": "飞飞dsaf飞",
+                "q": "测试是32否收到",
+                "file_id": "32dwe"
+            }
+        ],
+        "error": [] // 其他错误
    }
 }
 ```

--- a/docSite/content/docs/installation/upgrading/447.md
+++ b/docSite/content/docs/installation/upgrading/447.md
@@ -27,3 +27,5 @@ curl --location --request POST 'https://{{host}}/api/admin/initv447' \
 1. 优化了数据库文件 crud。
 2. 兼容链接读取，作为 source。
+3. 区分手动录入和标注，可追数据至某个文件。
+4. 升级 openai sdk。
\ No newline at end of file
--- a/packages/core/ai/config.ts
+++ b/packages/core/ai/config.ts
 import { UserModelSchema } from '../user/type';
-import { Configuration, OpenAIApi } from 'openai';
+import OpenAI from 'openai';
 export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
 export const baseUrl = process.env.ONEAPI_URL || openaiBaseUrl;
 export const systemAIChatKey = process.env.CHAT_API_KEY || '';
-export const getAIChatApi = (props?: UserModelSchema['openaiAccount']) => {
+export const getAIApi = (props?: UserModelSchema['openaiAccount'], timeout = 6000) => {
-  return new OpenAIApi(
+  return new OpenAI({
-    new Configuration({
+    apiKey: props?.key || systemAIChatKey,
-      basePath: props?.baseUrl || baseUrl,
+    baseURL: props?.baseUrl || baseUrl,
-      apiKey: props?.key || systemAIChatKey
+    httpAgent: global.httpsAgent,
-    })
+    timeout
-  );
+  });
-};
-/* openai axios config */
-export const axiosConfig = (props?: UserModelSchema['openaiAccount']) => {
-  return {
-    baseURL: props?.baseUrl || baseUrl, // 此处仅对非 npm 模块有效
-    httpsAgent: global.httpsAgent,
-    headers: {
-      Authorization: `Bearer ${props?.key || systemAIChatKey}`,
-      auth: process.env.OPENAI_BASE_URL_AUTH || ''
-    }
-  };
 };
--- a/packages/core/ai/constant.ts
+++ b/packages/core/ai/constant.ts
-export { ChatCompletionRequestMessageRoleEnum } from 'openai';
+export enum ChatCompletionRequestMessageRoleEnum {
+  'System' = 'system',
+  'User' = 'user',
+  'Assistant' = 'assistant',
+  'Function' = 'function'
+}
--- a/packages/core/ai/functions/createQuestionGuide.ts
+++ b/packages/core/ai/functions/createQuestionGuide.ts
 import { ChatCompletionRequestMessage } from '../type';
-import { getAIChatApi } from '../config';
+import { getAIApi } from '../config';
 export const Prompt_QuestionGuide = `我不太清楚问你什么问题，请帮我生成 3 个问题，引导我继续提问。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
@@ -10,8 +10,8 @@ export async function createQuestionGuide({
  messages: ChatCompletionRequestMessage[];
  model: string;
 }) {
-  const chatAPI = getAIChatApi();
+  const ai = getAIApi();
-  const { data } = await chatAPI.createChatCompletion({
+  const data = await ai.chat.completions.create({
    model: model,
    temperature: 0,
    max_tokens: 200,

--- a/packages/core/ai/type.d.ts
+++ b/packages/core/ai/type.d.ts
-export type { CreateChatCompletionRequest, ChatCompletionRequestMessage } from 'openai';
+import OpenAI from 'openai';
+export type ChatCompletionRequestMessage = OpenAI.Chat.CreateChatCompletionRequestMessage;
+export type ChatCompletion = OpenAI.Chat.ChatCompletion;
+export type CreateChatCompletionRequest = OpenAI.Chat.ChatCompletionCreateParams;
+export type StreamChatType = Stream<OpenAI.Chat.ChatCompletionChunk>;
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -2,10 +2,11 @@
  "name": "@fastgpt/core",
  "version": "1.0.0",
  "dependencies": {
-    "openai": "^3.3.0",
-    "tunnel": "^0.0.6",
    "@fastgpt/common": "workspace:*",
-    "@fastgpt/support": "workspace:*"
+    "@fastgpt/support": "workspace:*",
+    "encoding": "^0.1.13",
+    "openai": "^4.11.1",
+    "tunnel": "^0.0.6"
  },
  "devDependencies": {
    "@types/tunnel": "^0.0.4"

--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/projects/app/public/docs/versionIntro.md
+++ b/projects/app/public/docs/versionIntro.md
-### Fast GPT V4.4.6
+### Fast GPT V4.4.7
-1. 高级编排新增模块 - 应用调用
+1. 优化数据集管理，区分手动录入和标注，可追数据至某个文件，保留链接读取的原始链接。
-2. 新增 - 必要连接校验
+2. [使用文档](https://doc.fastgpt.run/docs/intro/)
-3. 新增 - 下一步指引选项，可以通过模型生成 3 个预测问题。
+3. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
-4. 新增 - 分享链接 hook 身份校验。
+4. [点击查看商业版](https://doc.fastgpt.run/docs/commercial/)
-5. [使用文档](https://doc.fastgpt.run/docs/intro/)
-6. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
-7. [点击查看商业版](https://doc.fastgpt.run/docs/commercial/)
--- a/projects/app/public/icon/human.svg
+++ b/projects/app/public/icon/human.svg
--- a/projects/app/public/locales/en/common.json
+++ b/projects/app/public/locales/en/common.json
@@ -131,6 +131,7 @@
    "Rename Success": "Rename Success",
    "Search": "Search",
    "Status": "Status",
+    "Unknow": "Unknow",
    "Update Successful": "Update Successful",
    "export": ""
  },

--- a/projects/app/public/locales/zh/common.json
+++ b/projects/app/public/locales/zh/common.json
@@ -131,6 +131,7 @@
    "Rename Success": "重命名成功",
    "Search": "搜索",
    "Status": "状态",
+    "Unknow": "未知",
    "Update Successful": "更新成功",
    "export": ""
  },

--- a/projects/app/src/api/app.ts
+++ b/projects/app/src/api/app.ts
@@ -2,9 +2,9 @@ import { GET, POST, DELETE, PUT } from './request';
 import type { AppSchema } from '@/types/mongoSchema';
 import type { AppListItemType, AppUpdateParams } from '@/types/app';
 import { RequestPaging } from '../types/index';
-import type { Props as CreateAppProps } from '@/pages/api/app/create';
 import { addDays } from 'date-fns';
 import { GetAppChatLogsParams } from './request/app';
+import type { CreateAppParams } from '@/types/app';
 /**
 * 获取模型列表
@@ -14,7 +14,7 @@ export const getMyModels = () => GET<AppListItemType[]>('/app/myApps');
 /**
 * 创建一个模型
 */
-export const postCreateApp = (data: CreateAppProps) => POST<string>('/app/create', data);
+export const postCreateApp = (data: CreateAppParams) => POST<string>('/app/create', data);
 /**
 * 根据 ID 删除模型

--- a/projects/app/src/components/ChatBox/QuoteModal.tsx
+++ b/projects/app/src/components/ChatBox/QuoteModal.tsx
 import React, { useCallback, useMemo, useState } from 'react';
-import { ModalBody, Box, useTheme } from '@chakra-ui/react';
+import { ModalBody, Box, useTheme, Flex, Progress } from '@chakra-ui/react';
 import { getDatasetDataItemById } from '@/api/core/dataset/data';
 import { useLoading } from '@/hooks/useLoading';
 import { useToast } from '@/hooks/useToast';
@@ -8,22 +8,19 @@ import { QuoteItemType } from '@/types/chat';
 import MyIcon from '@/components/Icon';
 import InputDataModal, { RawFileText } from '@/pages/kb/detail/components/InputDataModal';
 import MyModal from '../MyModal';
-import type { PgDataItemType } from '@/types/core/dataset/data';
+import { useTranslation } from 'react-i18next';
 import { useRouter } from 'next/router';
-type SearchType = PgDataItemType & {
-  kb_id?: string;
-};
 const QuoteModal = ({
  onUpdateQuote,
  rawSearch = [],
  onClose
 }: {
  onUpdateQuote: (quoteId: string, sourceText?: string) => Promise<void>;
-  rawSearch: SearchType[];
+  rawSearch: QuoteItemType[];
  onClose: () => void;
 }) => {
+  const { t } = useTranslation();
  const theme = useTheme();
  const router = useRouter();
  const { toast } = useToast();
@@ -36,7 +33,7 @@ const QuoteModal = ({
   * click edit, get new kbDataItem
   */
  const onclickEdit = useCallback(
-    async (item: SearchType) => {
+    async (item: QuoteItemType) => {
      if (!item.id) return;
      try {
        setIsLoading(true);
@@ -95,9 +92,30 @@ const QuoteModal = ({
              _hover={{ '& .edit': { display: 'flex' } }}
              overflow={'hidden'}
            >
-              {item.source && !isShare && (
+              {!isShare && (
-                <RawFileText filename={item.source} fileId={item.file_id} />
+                <Flex alignItems={'center'} mb={1}>
+                  <RawFileText
+                    filename={item.source || t('common.Unknow') || 'Unknow'}
+                    fileId={item.file_id}
+                  />
+                  <Box flex={'1'} />
+                  {item.score && (
+                    <>
+                      <Progress
+                        mx={2}
+                        w={['60px', '100px']}
+                        value={item.score * 100}
+                        size="sm"
+                        borderRadius={'20px'}
+                        colorScheme="gray"
+                        border={theme.borders.base}
+                      />
+                      <Box>{item.score.toFixed(4)}</Box>
+                    </>
                  )}
+                </Flex>
+              )}
              <Box>{item.q}</Box>
              <Box>{item.a}</Box>
              {item.id && !isShare && (

--- a/projects/app/src/components/Layout/index.tsx
+++ b/projects/app/src/components/Layout/index.tsx
@@ -102,7 +102,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
          </>
        )}
      </Box>
-      <Loading loading={loading} />
+      <Loading loading={loading} zIndex={9999} />
    </>
  );
 };

--- a/projects/app/src/components/Loading/index.tsx
+++ b/projects/app/src/components/Loading/index.tsx
@@ -4,16 +4,18 @@ import { Spinner, Flex, Box } from '@chakra-ui/react';
 const Loading = ({
  fixed = true,
  text = '',
-  bg = 'rgba(255,255,255,0.5)'
+  bg = 'rgba(255,255,255,0.5)',
+  zIndex = 1000
 }: {
  fixed?: boolean;
  text?: string;
  bg?: string;
+  zIndex?: number;
 }) => {
  return (
    <Flex
      position={fixed ? 'fixed' : 'absolute'}
-      zIndex={1000}
+      zIndex={zIndex}
      bg={bg}
      top={0}
      left={0}

--- a/projects/app/src/constants/chat.ts
+++ b/projects/app/src/constants/chat.ts
@@ -59,5 +59,5 @@ export enum OutLinkTypeEnum {
  apikey = 'apikey'
 }
-export const HUMAN_ICON = `/icon/human.png`;
+export const HUMAN_ICON = `/icon/human.svg`;
 export const LOGO_ICON = `/icon/logo.svg`;
--- a/projects/app/src/constants/flow/ModuleTemplate.ts
+++ b/projects/app/src/constants/flow/ModuleTemplate.ts
-import { SystemInputEnum } from '../app';
+import { AppTypeEnum, SystemInputEnum } from '../app';
 import { TaskResponseKeyEnum } from '../chat';
 import {
  FlowModuleTypeEnum,
@@ -575,12 +575,17 @@ export const ModuleTemplatesFlat = [
 ];
 // template
-export const appTemplates: (AppItemType & { avatar: string; intro: string })[] = [
+export const appTemplates: (AppItemType & {
+  avatar: string;
+  intro: string;
+  type: `${AppTypeEnum}`;
+})[] = [
  {
    id: 'simpleChat',
    avatar: '/imgs/module/AI.png',
    name: '简单的对话',
    intro: '一个极其简单的 AI 对话应用',
+    type: AppTypeEnum.basic,
    modules: [
      {
        moduleId: 'userGuide',
@@ -797,6 +802,7 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
    avatar: '/imgs/module/db.png',
    name: '知识库 + 对话引导',
    intro: '每次提问时进行一次知识库搜索，将搜索结果注入 LLM 模型进行参考回答',
+    type: AppTypeEnum.basic,
    modules: [
      {
        moduleId: 'userGuide',
@@ -811,7 +817,7 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
            key: 'welcomeText',
            type: 'input',
            label: '开场白',
-            value: '你好，我是 laf 助手，有什么可以帮助你的么？',
+            value: '你好，我是知识库助手，请不要忘记选择知识库噢~',
            connected: true
          }
        ],
@@ -1162,6 +1168,7 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
    avatar: '/imgs/module/userGuide.png',
    name: '对话引导 + 变量',
    intro: '可以在对话开始发送一段提示，或者让用户填写一些内容，作为本次对话的变量',
+    type: AppTypeEnum.basic,
    modules: [
      {
        moduleId: 'userGuide',
@@ -1174,27 +1181,15 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
        inputs: [
          {
            key: 'welcomeText',
-            type: 'input',
+            type: 'hidden',
            label: '开场白',
            value: '你好，我可以为你翻译各种语言，请告诉我你需要翻译成什么语言？',
            connected: true
-          }
-        ],
-        outputs: []
          },
          {
-        moduleId: 'variable',
-        name: '全局变量',
-        flowType: 'variable',
-        position: {
-          x: 444.0369195277651,
-          y: 1008.5185781784537
-        },
-        inputs: [
-          {
            key: 'variables',
-            type: 'systemInput',
+            type: 'hidden',
-            label: '变量输入',
+            label: '对话框变量',
            value: [
              {
                id: '35c640eb-cf22-431f-bb57-3fc21643880e',
@@ -1227,6 +1222,13 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
              }
            ],
            connected: true
+          },
+          {
+            key: 'questionGuide',
+            type: 'switch',
+            label: '问题引导',
+            value: false,
+            connected: true
          }
        ],
        outputs: []
@@ -1275,7 +1277,7 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
            key: 'maxContext',
            type: 'numberInput',
            label: '最长记录数',
-            value: 10,
+            value: 2,
            min: 0,
            max: 50,
            connected: true
@@ -1317,7 +1319,6 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
            type: 'custom',
            label: '对话模型',
            value: 'gpt-3.5-turbo-16k',
-            list: [],
            connected: true
          },
          {
@@ -1346,7 +1347,7 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
            label: '回复上限',
            value: 8000,
            min: 100,
-            max: 16000,
+            max: 4000,
            step: 50,
            markList: [
              {
@@ -1354,8 +1355,8 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
                value: 100
              },
              {
-                label: '16000',
+                label: '4000',
-                value: 16000
+                value: 4000
              }
            ],
            connected: true
@@ -1364,11 +1365,28 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
            key: 'systemPrompt',
            type: 'textarea',
            label: '系统提示词',
+            max: 300,
            valueType: 'string',
            description:
              '模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}',
            placeholder:
              '模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}',
+            value: '请直接将我的问题翻译成{{language}}，不需要回答问题。',
+            connected: true
+          },
+          {
+            key: 'quoteTemplate',
+            type: 'hidden',
+            label: '引用内容模板',
+            valueType: 'string',
+            value: '',
+            connected: true
+          },
+          {
+            key: 'quotePrompt',
+            type: 'hidden',
+            label: '引用内容提示词',
+            valueType: 'string',
            value: '',
            connected: true
          },
@@ -1381,8 +1399,9 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
          },
          {
            key: 'quoteQA',
-            type: 'target',
+            type: 'custom',
            label: '引用内容',
+            description: "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
            valueType: 'kb_quote',
            connected: false
          },
@@ -1406,8 +1425,9 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
          {
            key: 'answerText',
            label: '模型回复',
-            description: '直接响应，无需配置',
+            description: '将在 stream 回复完毕后触发',
-            type: 'hidden',
+            valueType: 'string',
+            type: 'source',
            targets: []
          },
          {
@@ -1417,6 +1437,14 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
            valueType: 'boolean',
            type: 'source',
            targets: []
+          },
+          {
+            key: 'history',
+            label: '新的上下文',
+            description: '将本次回复内容拼接上历史记录，作为新的上下文返回',
+            valueType: 'chat_history',
+            type: 'source',
+            targets: []
          }
        ]
      }
@@ -1427,6 +1455,7 @@ export const appTemplates: (AppItemType & { avatar: string; intro: string })[] =
    avatar: '/imgs/module/cq.png',
    name: '问题分类 + 知识库',
    intro: '先对用户的问题进行分类，再根据不同类型问题，执行不同的操作',
+    type: AppTypeEnum.advanced,
    modules: [
      {
        moduleId: '7z5g5h',

--- a/projects/app/src/hooks/useLoading.tsx
+++ b/projects/app/src/hooks/useLoading.tsx
@@ -8,13 +8,17 @@ export const useLoading = (props?: { defaultLoading: boolean }) => {
    ({
      loading,
      fixed = true,
-      text = ''
+      text = '',
+      zIndex
    }: {
      loading?: boolean;
      fixed?: boolean;
      text?: string;
+      zIndex?: number;
    }): JSX.Element | null => {
-      return isLoading || loading ? <LoadingComponent fixed={fixed} text={text} /> : null;
+      return isLoading || loading ? (
+        <LoadingComponent fixed={fixed} text={text} zIndex={zIndex} />
+      ) : null;
    },
    [isLoading]
  );

--- a/projects/app/src/pages/api/admin/initv447.ts
+++ b/projects/app/src/pages/api/admin/initv447.ts
@@ -15,16 +15,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    await connectToDatabase();
    await authUser({ req, authRoot: true });
-    console.log('add index');
-    await PgClient.query(
-      `
-      ALTER TABLE modeldata
-      ALTER COLUMN source TYPE VARCHAR(256),
-      ALTER COLUMN file_id TYPE VARCHAR(256);
-      CREATE INDEX IF NOT EXISTS modelData_fileId_index ON modeldata (file_id);
-      `
-    );
-    console.log('index success');
    console.log('count rows');
    // 去重获取 fileId
    const { rows } = await PgClient.query(`SELECT DISTINCT file_id
@@ -36,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      await init(rows.slice(i, i + limit), initFileIds);
      console.log(i);
    }
-    console.log('filter success');
-    console.log('start update');
    for (let i = 0; i < initFileIds.length; i++) {
      await PgClient.query(`UPDATE ${PgDatasetTableName}
@@ -49,9 +37,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const { rows: emptyIds } = await PgClient.query(
      `SELECT id FROM ${PgDatasetTableName} WHERE file_id IS NULL OR file_id=''`
    );
+    console.log('filter success');
    console.log(emptyIds.length);
    await delay(5000);
+    console.log('start update');
    async function start(start: number) {
      for (let i = start; i < emptyIds.length; i += limit) {
@@ -65,12 +55,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      start(i);
    }
-    // await PgClient.query(
-    //   `UPDATE ${PgDatasetTableName}
-    //     SET file_id = '${DatasetSpecialIdEnum.manual}'
-    //     WHERE file_id IS NULL OR file_id = ''`
-    // );
    console.log('update success');
    jsonRes(res, {

--- a/projects/app/src/pages/api/app/create.ts
+++ b/projects/app/src/pages/api/app/create.ts
@@ -4,17 +4,17 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { App } from '@/service/models/app';
-import { AppModuleItemType } from '@/types/app';
+import type { CreateAppParams } from '@/types/app';
+import { AppTypeEnum } from '@/constants/app';
-export type Props = {
-  name: string;
-  avatar?: string;
-  modules: AppModuleItemType[];
-};
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    const { name, avatar, modules } = req.body as Props;
+    const {
+      name = 'APP',
+      avatar,
+      type = AppTypeEnum.advanced,
+      modules
+    } = req.body as CreateAppParams;
    if (!name || !Array.isArray(modules)) {
      throw new Error('缺少参数');
@@ -38,7 +38,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      avatar,
      name,
      userId,
-      modules
+      modules,
+      type
    });
    jsonRes(res, {

--- a/projects/app/src/pages/api/app/update.ts
+++ b/projects/app/src/pages/api/app/update.ts
@@ -9,7 +9,7 @@ import { authApp } from '@/service/utils/auth';
 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    const { name, avatar, type, chat, share, intro, modules } = req.body as AppUpdateParams;
+    const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams;
    const { appId } = req.query as { appId: string };
    if (!appId) {
@@ -37,7 +37,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
        type,
        avatar,
        intro,
-        chat,
        ...(share && {
          'share.isShare': share.isShare,
          'share.isShareDetail': share.isShareDetail

--- a/projects/app/src/pages/api/core/dataset/data/insertData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts
@@ -13,6 +13,7 @@ import { getVectorModel } from '@/service/utils/data';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
 import { DatasetDataItemType } from '@/types/core/dataset/data';
 import { countPromptTokens } from '@/utils/common/tiktoken';
+import { authFileIdValid } from '@/service/dataset/auth';
 export type Props = {
  kbId: string;
@@ -72,6 +73,8 @@ export async function getVectorAndInsertDataset(
    return Promise.reject('已经存在完全一致的数据');
  }
+  await authFileIdValid(data.file_id);
  const { vectors } = await getVector({
    model: kb.vectorModel,
    input: [q],

--- a/projects/app/src/pages/api/core/dataset/data/pushData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts
+/* push data to training queue */
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, TrainingData, KB } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { authKb } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
-import { PgDatasetTableName, TrainingModeEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
 import { startQueue } from '@/service/utils/tools';
-import { PgClient } from '@/service/pg';
 import { getVectorModel } from '@/service/utils/data';
 import { DatasetDataItemType } from '@/types/core/dataset/data';
 import { countPromptTokens } from '@/utils/common/tiktoken';
 import type { PushDataProps, PushDataResponse } from '@/api/core/dataset/data.d';
+import { authFileIdValid } from '@/service/dataset/auth';
 const modeMap = {
  [TrainingModeEnum.index]: true,
@@ -80,12 +81,21 @@ export async function pushDataToKb({
    [TrainingModeEnum.qa]: global.qaModel.maxToken * 0.8
  };
-  // 过滤重复的 qa 内容
+  // filter repeat or equal content
  const set = new Set();
-  const filterData: DatasetDataItemType[] = [];
+  const filterResult: Record<string, DatasetDataItemType[]> = {
+    success: [],
+    overToken: [],
+    fileIdInvalid: [],
+    error: []
+  };
-  data.forEach((item) => {
+  await Promise.all(
-    if (!item.q) return;
+    data.map(async (item) => {
+      if (!item.q) {
+        filterResult.error.push(item);
+        return;
+      }
      const text = item.q + item.a;
@@ -93,56 +103,27 @@ export async function pushDataToKb({
      const token = countPromptTokens(item.q, 'system');
      if (token > modeMaxToken[mode]) {
+        filterResult.overToken.push(item);
        return;
      }
-    if (!set.has(text)) {
-      filterData.push(item);
-      set.add(text);
-    }
-  });
-  // 数据库去重
-  const insertData = (
-    await Promise.allSettled(
-      filterData.map(async (data) => {
-        let { q, a } = data;
-        if (mode !== TrainingModeEnum.index) {
-          return Promise.resolve(data);
-        }
-        if (!q) {
-          return Promise.reject('q为空');
-        }
-        q = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
-        a = a.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
-        // Exactly the same data, not push
      try {
-          const { rows } = await PgClient.query(`
+        await authFileIdValid(item.file_id);
-            SELECT COUNT(*) > 0 AS exists
-            FROM  ${PgDatasetTableName} 
-            WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
-          `);
-          const exists = rows[0]?.exists || false;
-          if (exists) {
-            return Promise.reject('已经存在');
-          }
      } catch (error) {
-          console.log(error);
+        filterResult.fileIdInvalid.push(item);
+        return;
+      }
+      if (!set.has(text)) {
+        filterResult.success.push(item);
+        set.add(text);
      }
-        return Promise.resolve(data);
    })
-    )
+  );
-  )
-    .filter((item) => item.status === 'fulfilled')
-    .map<DatasetDataItemType>((item: any) => item.value);
  // 插入记录
  const insertRes = await TrainingData.insertMany(
-    insertData.map((item) => ({
+    filterResult.success.map((item) => ({
      ...item,
      userId,
      kbId,
@@ -154,9 +135,11 @@ export async function pushDataToKb({
  );
  insertRes.length > 0 && startQueue();
+  delete filterResult.success;
  return {
-    insertLen: insertRes.length
+    insertLen: insertRes.length,
+    ...filterResult
  };
 }

--- a/projects/app/src/pages/api/core/dataset/file/list.ts
+++ b/projects/app/src/pages/api/core/dataset/file/list.ts
@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { PgClient } from '@/service/pg';
+import { PgClient, updateDataFileId } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { FileStatusEnum } from '@/constants/dataset';
 import { strIsLink } from '@fastgpt/common/tools/str';
@@ -35,8 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      .join(' ')}
      ${searchText ? `AND source ILIKE '%${searchText}%'` : ''}`;
-    const [{ rows }, { rowCount: total }] = await Promise.all([
+    let [{ rows }, { rowCount: total }] = await Promise.all([
-      PgClient.query(`SELECT file_id, COUNT(*) AS count
+      PgClient.query<{ file_id: string; count: number }>(`SELECT file_id, COUNT(*) AS count
    FROM ${PgDatasetTableName}
    where ${pgWhere}
    GROUP BY file_id
@@ -49,6 +49,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    `)
    ]);
+    // If fileId is invalid, reset it to manual
+    await Promise.all(
+      rows.map((row) => {
+        if (!strIsLink(row.file_id) && row.file_id.length !== 24) {
+          return updateDataFileId({
+            oldFileId: row.file_id,
+            userId,
+            newFileId: DatasetSpecialIdEnum.manual
+          });
+        }
+      })
+    );
+    // just filter link or fileData
+    rows = rows.filter((row) => strIsLink(row.file_id) || row.file_id.length === 24);
    // find files
    const gridFs = new GridFSStorage('dataset', userId);
    const collection = gridFs.Collection();
@@ -96,6 +111,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    const data = await Promise.all([
      getSpecialData(),
      ...rows.map(async (row) => {
+        if (!row.file_id) return null;
        // link data
        if (strIsLink(row.file_id)) {
          const { rows } = await PgClient.select(PgDatasetTableName, {

--- a/projects/app/src/pages/api/core/dataset/searchTest.ts
+++ b/projects/app/src/pages/api/core/dataset/searchTest.ts
@@ -47,7 +47,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      data: response?.[2]?.rows || []
    });
  } catch (err) {
-    console.log(err);
    jsonRes(res, {
      code: 500,
      error: err

--- a/projects/app/src/pages/api/openapi/plugin/vector.ts
+++ b/projects/app/src/pages/api/openapi/plugin/vector.ts
@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authBalanceByUid, authUser } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
-import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config';
+import { getAIApi } from '@fastgpt/core/ai/config';
 import { pushGenerateVectorBill } from '@/service/common/bill/push';
 type Props = {
@@ -54,29 +54,31 @@ export async function getVector({
  }
  // 获取 chatAPI
-  const chatAPI = getAIChatApi();
+  const ai = getAIApi();
  // 把输入的内容转成向量
-  const result = await chatAPI
+  const result = await ai.embeddings
-    .createEmbedding(
+    .create(
      {
        model,
        input
      },
      {
-        timeout: 60000,
+        timeout: 60000
-        ...axiosConfig()
      }
    )
    .then(async (res) => {
-      if (!res.data?.data?.[0]?.embedding) {
+      if (!res.data) {
-        console.log(res.data);
+        return Promise.reject('Embedding API 404');
+      }
+      if (!res?.data?.[0]?.embedding) {
+        console.log(res?.data);
        // @ts-ignore
        return Promise.reject(res.data?.err?.message || 'Embedding API Error');
      }
      return {
-        tokenLen: res.data.usage.total_tokens || 0,
+        tokenLen: res.usage.total_tokens || 0,
-        vectors: await Promise.all(res.data.data.map((item) => unityDimensional(item.embedding)))
+        vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
      };
    });

--- a/projects/app/src/pages/api/user/account/update.ts
+++ b/projects/app/src/pages/api/user/account/update.ts
@@ -5,7 +5,7 @@ import { User } from '@/service/models/user';
 import { connectToDatabase } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { UserUpdateParams } from '@/types/user';
-import { axiosConfig, getAIChatApi, openaiBaseUrl } from '@fastgpt/core/ai/config';
+import { getAIApi, openaiBaseUrl } from '@fastgpt/core/ai/config';
 /* update user info */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -22,20 +22,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      const baseUrl = openaiAccount?.baseUrl || openaiBaseUrl;
      openaiAccount.baseUrl = baseUrl;
-      const chatAPI = getAIChatApi(openaiAccount);
+      const ai = getAIApi(openaiAccount);
-      const response = await chatAPI.createChatCompletion(
+      const response = await ai.chat.completions.create({
-        {
        model: 'gpt-3.5-turbo',
        max_tokens: 1,
        messages: [{ role: 'user', content: 'hi' }]
-        },
+      });
-        {
+      if (response?.choices?.[0]?.message?.content === undefined) {
-          ...axiosConfig(openaiAccount)
+        throw new Error('Key response is empty');
-        }
-      );
-      if (response?.data?.choices?.[0]?.message?.content === undefined) {
-        throw new Error(JSON.stringify(response?.data));
      }
    }

--- a/projects/app/src/pages/app/detail/index.tsx
+++ b/projects/app/src/pages/app/detail/index.tsx
@@ -6,6 +6,7 @@ import dynamic from 'next/dynamic';
 import { defaultApp } from '@/constants/model';
 import { useToast } from '@/hooks/useToast';
 import { useQuery } from '@tanstack/react-query';
+import { feConfigs } from '@/store/static';
 import Tabs from '@/components/Tabs';
 import SideTabs from '@/components/SideTabs';
@@ -52,7 +53,9 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
  const tabList = useMemo(
    () => [
      { label: '简易配置', id: TabEnum.basicEdit, icon: 'overviewLight' },
-      { label: '高级编排', id: TabEnum.adEdit, icon: 'settingLight' },
+      ...(feConfigs?.hide_app_flow
+        ? []
+        : [{ label: '高级编排', id: TabEnum.adEdit, icon: 'settingLight' }]),
      { label: '外部使用', id: TabEnum.outLink, icon: 'shareLight' },
      { label: '对话日志', id: TabEnum.logs, icon: 'logsLight' },
      { label: '立即对话', id: TabEnum.startChat, icon: 'chat' }

--- a/projects/app/src/pages/app/list/component/CreateModal.tsx
+++ b/projects/app/src/pages/app/list/component/CreateModal.tsx
@@ -21,6 +21,7 @@ import { useRouter } from 'next/router';
 import { appTemplates } from '@/constants/flow/ModuleTemplate';
 import { useGlobalStore } from '@/store/global';
 import { useRequest } from '@/hooks/useRequest';
+import { feConfigs } from '@/store/static';
 import Avatar from '@/components/Avatar';
 import MyTooltip from '@/components/MyTooltip';
 import MyModal from '@/components/MyModal';
@@ -74,10 +75,15 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
  const { mutate: onclickCreate, isLoading: creating } = useRequest({
    mutationFn: async (data: FormType) => {
+      const template = appTemplates.find((item) => item.id === data.templateId);
+      if (!template) {
+        return Promise.reject('模板不存在');
+      }
      return postCreateApp({
        avatar: data.avatar,
        name: data.name,
-        modules: appTemplates.find((item) => item.id === data.templateId)?.modules || []
+        type: template.type,
+        modules: template.modules || []
      });
    },
    onSuccess(id: string) {
@@ -118,6 +124,8 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
            })}
          />
        </Flex>
+        {!feConfigs?.hide_app_flow && (
+          <>
            <Box mt={[4, 7]} mb={[0, 3]} color={'myGray.800'} fontWeight={'bold'}>
              从模板中选择
            </Box>
@@ -160,6 +168,8 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
                </Card>
              ))}
            </Grid>
+          </>
+        )}
      </ModalBody>
      <ModalFooter>

--- a/projects/app/src/pages/kb/detail/components/InputDataModal.tsx
+++ b/projects/app/src/pages/kb/detail/components/InputDataModal.tsx
@@ -263,6 +263,10 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp
  const { setLoading } = useGlobalStore();
  const hasFile = useMemo(() => fileId && !datasetSpecialIds.includes(fileId), [fileId]);
+  const formatName = useMemo(
+    () => (filename.startsWith('kb') ? t(filename) : filename),
+    [filename, t]
+  );
  return (
    <MyTooltip label={hasFile ? t('file.Click to view file') || '' : ''} shouldWrapChildren={false}>
@@ -293,7 +297,7 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp
          : {})}
        {...props}
      >
-        {t(filename)}
+        {formatName}
      </Box>
    </MyTooltip>
  );

--- a/projects/app/src/service/dataset/auth.ts
+++ b/projects/app/src/service/dataset/auth.ts
+import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
+import { GridFSStorage } from '../lib/gridfs';
+import { Types } from 'mongoose';
+export async function authFileIdValid(fileId?: string) {
+  if (!fileId) return true;
+  if (isSpecialFileId(fileId)) return true;
+  try {
+    // find file
+    const gridFs = new GridFSStorage('dataset', '');
+    const collection = gridFs.Collection();
+    const file = await collection.findOne(
+      { _id: new Types.ObjectId(fileId) },
+      { projection: { _id: 1 } }
+    );
+    if (!file) {
+      return Promise.reject('Invalid fileId');
+    }
+  } catch (error) {
+    return Promise.reject('Invalid fileId');
+  }
+}
--- a/projects/app/src/service/errorCode.ts
+++ b/projects/app/src/service/errorCode.ts
@@ -17,19 +17,6 @@ export const TOKEN_ERROR_CODE: Record<number, string> = {
  403: '登录状态无效,请重新登录'
 };
-export const openaiError: Record<string, string> = {
-  context_length_exceeded: '内容超长了，请重置对话',
-  Unauthorized: 'API-KEY 不合法',
-  rate_limit_reached: 'API被限制，请稍后再试',
-  'Bad Request': 'Bad Request~ 可能内容太多了',
-  'Bad Gateway': '网关异常，请重试'
-};
-export const openaiAccountError: Record<string, string> = {
-  insufficient_quota: 'API 余额不足',
-  invalid_api_key: 'openai 账号异常',
-  account_deactivated: '账号已停用',
-  invalid_request_error: '无效请求'
-};
 export const proxyError: Record<string, boolean> = {
  ECONNABORTED: true,
  ECONNRESET: true

--- a/projects/app/src/service/events/generateQA.ts
+++ b/projects/app/src/service/events/generateQA.ts
@@ -4,7 +4,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import { ERROR_ENUM } from '../errorCode';
 import { sendInform } from '@/pages/api/user/inform/send';
 import { authBalanceByUid } from '../utils/auth';
-import { axiosConfig, getAIChatApi } from '@fastgpt/core/ai/config';
+import { getAIApi } from '@fastgpt/core/ai/config';
 import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
 import { addLog } from '../utils/tools';
 import { splitText2Chunks } from '@/utils/file';
@@ -58,8 +58,6 @@ export async function generateQA(): Promise<any> {
    const startTime = Date.now();
-    const chatAPI = getAIChatApi();
    // request LLM to get QA
    const text = data.q;
    const messages: ChatCompletionRequestMessage[] = [
@@ -73,19 +71,13 @@ export async function generateQA(): Promise<any> {
            })
      }
    ];
+    const ai = getAIApi(undefined, 480000);
-    const { data: chatResponse } = await chatAPI.createChatCompletion(
+    const chatResponse = await ai.chat.completions.create({
-      {
      model: global.qaModel.model,
      temperature: 0.01,
      messages,
      stream: false
-      },
+    });
-      {
-        timeout: 480000,
-        ...axiosConfig()
-      }
-    );
    const answer = chatResponse.choices?.[0].message?.content;
    const totalTokens = chatResponse.usage?.total_tokens || 0;

--- a/projects/app/src/service/models/user.ts
+++ b/projects/app/src/service/models/user.ts
@@ -23,7 +23,7 @@ const UserSchema = new Schema({
  },
  avatar: {
    type: String,
-    default: '/icon/human.png'
+    default: '/icon/human.svg'
  },
  balance: {
    type: Number,

--- a/projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts
+++ b/projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts
@@ -2,7 +2,7 @@ import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
 import { ChatContextFilter } from '@/service/common/tiktoken';
 import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat';
-import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config';
+import { getAIApi } from '@fastgpt/core/ai/config';
 import type { ClassifyQuestionAgentItemType } from '@/types/app';
 import { SystemInputEnum } from '@/constants/app';
 import { SpecialInputKeyEnum } from '@/constants/flow';
@@ -105,27 +105,22 @@ async function functionCall({
      required: ['type']
    }
  };
-  const chatAPI = getAIChatApi(user.openaiAccount);
+  const ai = getAIApi(user.openaiAccount);
-  const response = await chatAPI.createChatCompletion(
+  const response = await ai.chat.completions.create({
-    {
    model: cqModel.model,
    temperature: 0,
    messages: [...adaptMessages],
    function_call: { name: agentFunName },
    functions: [agentFunction]
-    },
+  });
-    {
-      ...axiosConfig(user.openaiAccount)
-    }
-  );
  try {
-    const arg = JSON.parse(response.data.choices?.[0]?.message?.function_call?.arguments || '');
+    const arg = JSON.parse(response.choices?.[0]?.message?.function_call?.arguments || '');
    return {
      arg,
-      tokens: response.data.usage?.total_tokens || 0
+      tokens: response.usage?.total_tokens || 0
    };
  } catch (error) {
    console.log('Your model may not support function_call');
@@ -155,20 +150,14 @@ Human:${userChatInput}`
    }
  ];
-  const chatAPI = getAIChatApi(user.openaiAccount);
+  const ai = getAIApi(user.openaiAccount, 480000);
-  const { data } = await chatAPI.createChatCompletion(
+  const data = await ai.chat.completions.create({
-    {
    model: extractModel.model,
    temperature: 0.01,
    messages: adaptChat2GptMessages({ messages, reserveId: false }),
    stream: false
-    },
+  });
-    {
-      timeout: 480000,
-      ...axiosConfig(user.openaiAccount)
-    }
-  );
  const answer = data.choices?.[0].message?.content || '';
  const totalTokens = data.usage?.total_tokens || 0;

--- a/projects/app/src/service/moduleDispatch/agent/extract.ts
+++ b/projects/app/src/service/moduleDispatch/agent/extract.ts
@@ -2,7 +2,7 @@ import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
 import { ChatContextFilter } from '@/service/common/tiktoken';
 import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat';
-import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config';
+import { getAIApi } from '@fastgpt/core/ai/config';
 import type { ContextExtractAgentItemType } from '@/types/app';
 import { ContextExtractEnum } from '@/constants/flow/flowField';
 import { FlowModuleTypeEnum } from '@/constants/flow';
@@ -126,30 +126,25 @@ async function functionCall({
    }
  };
-  const chatAPI = getAIChatApi(user.openaiAccount);
+  const ai = getAIApi(user.openaiAccount);
-  const response = await chatAPI.createChatCompletion(
+  const response = await ai.chat.completions.create({
-    {
    model: extractModel.model,
    temperature: 0,
    messages: [...adaptMessages],
    function_call: { name: agentFunName },
    functions: [agentFunction]
-    },
+  });
-    {
-      ...axiosConfig(user.openaiAccount)
-    }
-  );
  const arg: Record<string, any> = (() => {
    try {
-      return JSON.parse(response.data.choices?.[0]?.message?.function_call?.arguments || '{}');
+      return JSON.parse(response.choices?.[0]?.message?.function_call?.arguments || '{}');
    } catch (error) {
      return {};
    }
  })();
-  const tokens = response.data.usage?.total_tokens || 0;
+  const tokens = response.usage?.total_tokens || 0;
  return {
    tokens,
    arg
@@ -181,20 +176,14 @@ Human: ${content}`
    }
  ];
-  const chatAPI = getAIChatApi(user.openaiAccount);
+  const ai = getAIApi(user.openaiAccount, 480000);
-  const { data } = await chatAPI.createChatCompletion(
+  const data = await ai.chat.completions.create({
-    {
    model: extractModel.model,
    temperature: 0.01,
    messages: adaptChat2GptMessages({ messages, reserveId: false }),
    stream: false
-    },
+  });
-    {
-      timeout: 480000,
-      ...axiosConfig(user.openaiAccount)
-    }
-  );
  const answer = data.choices?.[0].message?.content || '';
  const totalTokens = data.usage?.total_tokens || 0;

--- a/projects/app/src/service/moduleDispatch/chat/oneapi.ts
+++ b/projects/app/src/service/moduleDispatch/chat/oneapi.ts
@@ -3,9 +3,9 @@ import { ChatContextFilter } from '@/service/common/tiktoken';
 import type { ChatItemType, QuoteItemType } from '@/types/chat';
 import type { ChatHistoryItemResType } from '@/types/chat';
 import { ChatRoleEnum, sseResponseEventEnum } from '@/constants/chat';
-import { SSEParseData, parseStreamChunk } from '@/utils/sse';
 import { textAdaptGptResponse } from '@/utils/adapt';
-import { getAIChatApi, axiosConfig } from '@fastgpt/core/ai/config';
+import { getAIApi } from '@fastgpt/core/ai/config';
+import type { ChatCompletion, StreamChatType } from '@fastgpt/core/ai/type';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { getChatModel } from '@/service/utils/data';
 import { countModelPrice } from '@/service/common/bill/push';
@@ -20,9 +20,7 @@ import type { AIChatProps } from '@/types/core/aiChat';
 import { replaceVariable } from '@/utils/common/tools/text';
 import { FlowModuleTypeEnum } from '@/constants/flow';
 import type { ModuleDispatchProps } from '@/types/core/chat/type';
-import { Readable } from 'stream';
 import { responseWrite, responseWriteController } from '@/service/common/stream';
-import { addLog } from '@/service/utils/tools';
 export type ChatProps = ModuleDispatchProps<
  AIChatProps & {
@@ -106,10 +104,9 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
  // FastGPT temperature range: 1~10
  temperature = +(modelConstantsData.maxTemperature * (temperature / 10)).toFixed(2);
  temperature = Math.max(temperature, 0.01);
-  const chatAPI = getAIChatApi(user.openaiAccount);
+  const ai = getAIApi(user.openaiAccount, 480000);
-  const response = await chatAPI.createChatCompletion(
+  const response = await ai.chat.completions.create({
-    {
    model,
    temperature,
    max_tokens,
@@ -125,13 +122,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
      ...messages
    ],
    stream
-    },
+  });
-    {
-      timeout: 480000,
-      responseType: stream ? 'stream' : 'json',
-      ...axiosConfig(user.openaiAccount)
-    }
-  );
  const { answerText, totalTokens, completeMessages } = await (async () => {
    if (stream) {
@@ -139,7 +130,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
      const { answer } = await streamResponse({
        res,
        detail,
-        response
+        stream: response
      });
      // count tokens
      const completeMessages = filterMessages.concat({
@@ -159,8 +150,9 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
        completeMessages
      };
    } else {
-      const answer = response.data.choices?.[0].message?.content || '';
+      const unStreamResponse = response as ChatCompletion;
-      const totalTokens = response.data.usage?.total_tokens || 0;
+      const answer = unStreamResponse.choices?.[0].message?.content || '';
+      const totalTokens = unStreamResponse.usage?.total_tokens || 0;
      const completeMessages = filterMessages.concat({
        obj: ChatRoleEnum.AI,
@@ -208,7 +200,7 @@ function filterQuote({
      obj: ChatRoleEnum.System,
      value: replaceVariable(quoteTemplate || defaultQuoteTemplate, {
        ...item,
-        index: `${index + 1}`
+        index: index + 1
      })
    }))
  });
@@ -340,37 +332,24 @@ function targetResponse({
 async function streamResponse({
  res,
  detail,
-  response
+  stream
 }: {
  res: NextApiResponse;
  detail: boolean;
-  response: any;
+  stream: StreamChatType;
 }) {
-  return new Promise<{ answer: string }>((resolve, reject) => {
-    const stream = response.data as Readable;
-    let answer = '';
-    const parseData = new SSEParseData();
  const write = responseWriteController({
    res,
    readStream: stream
  });
+  let answer = '';
-    stream.on('data', (data) => {
+  for await (const part of stream) {
    if (res.closed) {
-        stream.destroy();
+      stream.controller?.abort();
-        return resolve({ answer });
+      break;
    }
+    const content = part.choices[0]?.delta?.content || '';
-      const parse = parseStreamChunk(data);
-      parse.forEach((item) => {
-        const { data } = parseData.parse(item);
-        if (!data || data === '[DONE]') return;
-        const content: string = data?.choices?.[0]?.delta?.content || '';
-        if (data.error) {
-          addLog.error(`SSE response`, data.error);
-        } else {
    answer += content;
    responseWrite({
@@ -381,18 +360,12 @@ async function streamResponse({
      })
    });
  }
-      });
-    });
+  if (!answer) {
-    stream.on('end', () => {
+    return Promise.reject('Chat API is error or undefined');
-      resolve({ answer });
+  }
-    });
-    stream.on('close', () => {
+  return { answer };
-      resolve({ answer });
-    });
-    stream.on('error', (err) => {
-      reject(err);
-    });
-  });
 }
 function getHistoryPreview(completeMessages: ChatItemType[]) {

--- a/projects/app/src/service/moduleDispatch/kb/search.ts
+++ b/projects/app/src/service/moduleDispatch/kb/search.ts
@@ -46,7 +46,9 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
  const res: any = await PgClient.query(
    `BEGIN;
    SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
-    select kb_id,id,q,a,source,file_id from ${PgDatasetTableName} where kb_id IN (${kbList
+    select id, kb_id, q, a, source, file_id, (vector <#> '[${
+      vectors[0]
+    }]') * -1 AS score from ${PgDatasetTableName} where kb_id IN (${kbList
      .map((item) => `'${item.kbId}'`)
      .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
      vectors[0]

--- a/projects/app/src/service/pg.ts
+++ b/projects/app/src/service/pg.ts
@@ -3,6 +3,7 @@ import type { QueryResultRow } from 'pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { addLog } from './utils/tools';
 import type { DatasetDataItemType } from '@/types/core/dataset/data';
+import { DatasetSpecialIdEnum, datasetSpecialIdMap } from '@fastgpt/core/dataset/constant';
 export const connectPg = async (): Promise<Pool> => {
  if (global.pgClient) {
@@ -179,8 +180,13 @@ export const insertData2Dataset = ({
    values: data.map((item) => [
      { key: 'user_id', value: userId },
      { key: 'kb_id', value: kbId },
-      { key: 'source', value: item.source?.slice(0, 200)?.trim() || '' },
+      {
-      { key: 'file_id', value: item.file_id?.slice(0, 200)?.trim() || '' },
+        key: 'source',
+        value:
+          item.source?.slice(0, 200)?.trim() ||
+          datasetSpecialIdMap[DatasetSpecialIdEnum.manual].sourceName
+      },
+      { key: 'file_id', value: item.file_id?.slice(0, 200)?.trim() || DatasetSpecialIdEnum.manual },
      { key: 'q', value: item.q.replace(/'/g, '"') },
      { key: 'a', value: item.a.replace(/'/g, '"') },
      { key: 'vector', value: `[${item.vector}]` }
@@ -188,6 +194,25 @@ export const insertData2Dataset = ({
  });
 };
+/**
+ * Update data file_id
+ */
+export const updateDataFileId = async ({
+  oldFileId,
+  userId,
+  newFileId = DatasetSpecialIdEnum.manual
+}: {
+  oldFileId: string;
+  userId: string;
+  newFileId?: string;
+}) => {
+  await PgClient.update(PgDatasetTableName, {
+    where: [['file_id', oldFileId], 'AND', ['user_id', userId]],
+    values: [{ key: 'file_id', value: newFileId }]
+  });
+  return newFileId;
+};
 export async function initPg() {
  try {
    await connectPg();
@@ -203,10 +228,6 @@ export async function initPg() {
          q TEXT NOT NULL,
          a TEXT
      );
-      CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgDatasetTableName} USING HASH (user_id);
-      CREATE INDEX IF NOT EXISTS modelData_kb_id_index ON ${PgDatasetTableName} (kb_id);
-      CREATE INDEX IF NOT EXISTS modelData_fileId_index ON ${PgDatasetTableName} (file_id);
-      CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgDatasetTableName} (md5(q), md5(a), user_id, kb_id);
    `);
    console.log('init pg successful');
  } catch (error) {

--- a/projects/app/src/service/response.ts
+++ b/projects/app/src/service/response.ts
 import { sseResponseEventEnum } from '@/constants/chat';
 import { NextApiResponse } from 'next';
-import {
+import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode';
-  openaiError,
-  openaiAccountError,
-  proxyError,
-  ERROR_RESPONSE,
-  ERROR_ENUM
-} from './errorCode';
 import { clearCookie, sseResponse, addLog } from './utils/tools';
 export interface ResponseType<T = any> {
@@ -47,10 +41,8 @@ export const jsonRes = <T = any>(
      msg = '网络连接异常';
    } else if (error?.response?.data?.error?.message) {
      msg = error?.response?.data?.error?.message;
-    } else if (openaiAccountError[error?.response?.data?.error?.code]) {
+    } else if (error?.error?.message) {
-      msg = openaiAccountError[error?.response?.data?.error?.code];
+      msg = error?.error?.message;
-    } else if (openaiError[error?.response?.statusText]) {
-      msg = openaiError[error.response.statusText];
    }
    addLog.error(`response error: ${msg}`, error);
@@ -88,10 +80,8 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
    msg = '网络连接异常';
  } else if (error?.response?.data?.error?.message) {
    msg = error?.response?.data?.error?.message;
-  } else if (openaiAccountError[error?.response?.data?.error?.code]) {
+  } else if (error?.error?.message) {
-    msg = openaiAccountError[error?.response?.data?.error?.code];
+    msg = error?.error?.message;
-  } else if (openaiError[error?.response?.statusText]) {
-    msg = openaiError[error.response.statusText];
  }
  addLog.error(`sse error: ${msg}`, error);

--- a/projects/app/src/types/app.d.ts
+++ b/projects/app/src/types/app.d.ts
@@ -22,12 +22,17 @@ export type AppListItemType = {
  intro: string;
 };
+export type CreateAppParams = {
+  name?: string;
+  avatar?: string;
+  type?: `${AppTypeEnum}`;
+  modules: AppSchema['modules'];
+};
 export interface AppUpdateParams {
  name?: string;
  type?: `${AppTypeEnum}`;
  avatar?: string;
  intro?: string;
-  chat?: AppSchema['chat'];
  share?: AppSchema['share'];
  modules?: AppSchema['modules'];
 }

--- a/projects/app/src/types/chat.d.ts
+++ b/projects/app/src/types/chat.d.ts
@@ -45,6 +45,7 @@ export type ShareChatType = InitShareChatResponse & {
 export type QuoteItemType = PgDataItemType & {
  kb_id: string;
+  score?: number;
 };
 // response data

--- a/projects/app/src/types/core/chat/type.d.ts
+++ b/projects/app/src/types/core/chat/type.d.ts
@@ -3,7 +3,7 @@ import type { NextApiResponse } from 'next';
 import { RunningModuleItemType } from '@/types/app';
 import { UserModelSchema } from '@/types/mongoSchema';
-export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string };
+export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string; content: string };
 // module dispatch props type
 export type ModuleDispatchProps<T> = {

--- a/projects/app/src/types/index.d.ts
+++ b/projects/app/src/types/index.d.ts
@@ -29,6 +29,7 @@ export type FeConfigsType = {
  show_pay?: boolean;
  show_openai_account?: boolean;
  show_promotion?: boolean;
+  hide_app_flow?: boolean;
  openAPIUrl?: string;
  systemTitle?: string;
  authorText?: string;

--- a/projects/app/src/utils/adapt.ts
+++ b/projects/app/src/utils/adapt.ts
@@ -25,7 +25,7 @@ export const adaptBill = (bill: BillSchema): UserBillType => {
 };
 export const gptMessage2ChatType = (messages: MessageItemType[]): ChatItemType[] => {
-  const roleMap: Record<`${ChatCompletionRequestMessageRoleEnum}`, `${ChatRoleEnum}`> = {
+  const roleMap = {
    [ChatCompletionRequestMessageRoleEnum.Assistant]: ChatRoleEnum.AI,
    [ChatCompletionRequestMessageRoleEnum.User]: ChatRoleEnum.Human,
    [ChatCompletionRequestMessageRoleEnum.System]: ChatRoleEnum.System,

--- a/projects/app/src/utils/common/tools/text.ts
+++ b/projects/app/src/utils/common/tools/text.ts
 /* 
    replace {{variable}} to value
 */
-export function replaceVariable(text: string, obj: Record<string, string>) {
+export function replaceVariable(text: string, obj: Record<string, string | number>) {
  for (const key in obj) {
    const val = obj[key];
    if (typeof val !== 'string') continue;

--- a/projects/app/src/utils/file.ts
+++ b/projects/app/src/utils/file.ts
@@ -11,7 +11,14 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
  const overlapLen = Math.floor(maxLen * 0.25); // Overlap length
  try {
-    const splitTexts = text.split(/(?<=[。！？；.!?;\n])/g);
+    const tempMarker = 'SPLIT_HERE';
+    text = text.replace(/\n{3,}/g, '\n');
+    text = text.replace(/\s/g, ' ');
+    text = text.replace('\n\n', '');
+    const splitTexts = text
+      .replace(/([。！？；]|\.\s|!\s|\?\s|;\s|\n)/g, `$1${tempMarker}`)
+      .split(tempMarker)
+      .filter((part) => part);
    const chunks: string[] = [];
    let preChunk = '';

--- a/projects/app/src/utils/web/core/dataset.ts
+++ b/projects/app/src/utils/web/core/dataset.ts
@@ -9,7 +9,7 @@ export async function chunksUpload({
  mode,
  chunks,
  prompt,
-  rate = 50,
+  rate = 150,
  onUploading
 }: {
  kbId: string;