fix: package plus request (#4492)

* fix plus request (#4476)

* perf: package plus request

* perf: plus request fix

* fix: doc

---------

Co-authored-by: heheer <heheer@sealos.io>

docSite/content/zh-cn/docs/development/upgrading/494.md

@@ -62,4 +62,5 @@ curl --location --request POST 'https://{{host}}/api/admin/initv494' \
 ## 🐛 修复
 
 1. 搜索应用/知识库时，无法点击目录进入下一层。
-2. 重新训练时，参数未成功初始化。
\ No newline at end of file
+2. 重新训练时，参数未成功初始化。
+3. package/service 部分请求在多 app 中不一致。
\ No newline at end of file

packages/service/common/api/type.d.ts

+import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
+import {
+  DeepRagSearchProps,
+  SearchDatasetDataResponse
+} from '../../core/dataset/search/controller';
+import { AuthOpenApiLimitProps } from '../../support/openapi/auth';
+import { CreateUsageProps, ConcatUsageProps } from '@fastgpt/global/support/wallet/usage/api';
+import {
+  GetProApiDatasetFileContentParams,
+  GetProApiDatasetFileListParams,
+  GetProApiDatasetFilePreviewUrlParams
+} from '../../core/dataset/apiDataset/proApi';
+
+declare global {
+  var textCensorHandler: (params: { text: string }) => Promise<{ code: number; message?: string }>;
+  var deepRagHandler: (data: DeepRagSearchProps) => Promise<SearchDatasetDataResponse>;
+  var authOpenApiHandler: (data: AuthOpenApiLimitProps) => Promise<any>;
+  var createUsageHandler: (data: CreateUsageProps) => Promise<void>;
+  var concatUsageHandler: (data: ConcatUsageProps) => Promise<void>;
+
+  // API dataset
+  var getProApiDatasetFileList: (data: GetProApiDatasetFileListParams) => Promise<APIFileItem[]>;
+  var getProApiDatasetFileContent: (
+    data: GetProApiDatasetFileContentParams
+  ) => Promise<ApiFileReadContentResponse>;
+  var getProApiDatasetFilePreviewUrl: (
+    data: GetProApiDatasetFilePreviewUrlParams
+  ) => Promise<string>;
+}

packages/service/common/system/constants.ts

 export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : '';
-export const isFastGPTMainService = !!process.env.PRO_URL;
 // @ts-ignore
 export const isFastGPTProService = () => !!global.systemConfig;

packages/service/common/api/requestPlusApi.ts → packages/service/core/chat/postTextCensor.ts

-import { POST } from './plusRequest';
-
 export const postTextCensor = (data: { text: string }) =>
-  POST<{ code?: number; message: string }>('/common/censor/check', data)
+  global
+    .textCensorHandler(data)
     .then((res) => {
       if (res?.code === 5000) {
         return Promise.reject(res);

packages/service/core/dataset/apiDataset/proApi.ts

+import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
+import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
+
+export enum ProApiDatasetOperationTypeEnum {
+  LIST = 'list',
+  READ = 'read',
+  CONTENT = 'content'
+}
+
+export type ProApiDatasetCommonParams = {
+  feishuServer?: FeishuServer;
+  yuqueServer?: YuqueServer;
+};
+
+export type GetProApiDatasetFileListParams = ProApiDatasetCommonParams & {
+  parentId?: ParentIdType;
+};
+
+export type GetProApiDatasetFileContentParams = ProApiDatasetCommonParams & {
+  apiFileId: string;
+};
+
+export type GetProApiDatasetFilePreviewUrlParams = ProApiDatasetCommonParams & {
+  apiFileId: string;
+};

packages/service/core/dataset/read.ts

@@ -9,7 +9,6 @@ import { readRawContentByFileBuffer } from '../../common/file/read/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
 import { useApiDatasetRequest } from './apiDataset/api';
-import { POST } from '../../common/api/plusRequest';
 
 export const readFileRawTextByUrl = async ({
   teamId,
@@ -168,11 +167,7 @@ export const readApiServerFileContent = async ({
   }
 
   if (feishuServer || yuqueServer) {
-    return POST<{
-      title?: string;
-      rawText: string;
-    }>(`/core/dataset/systemApiDataset`, {
-      type: 'content',
+    return global.getProApiDatasetFileContent({
       feishuServer,
       yuqueServer,
       apiFileId

packages/service/core/dataset/search/controller.ts

@@ -24,7 +24,6 @@ import { MongoDatasetCollectionTags } from '../tag/schema';
 import { readFromSecondary } from '../../../common/mongo/utils';
 import { MongoDatasetDataText } from '../data/dataTextSchema';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
-import { POST } from '../../../common/api/plusRequest';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { datasetSearchQueryExtension } from './utils';
 import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
@@ -850,5 +849,4 @@ export type DeepRagSearchProps = SearchDatasetDataProps & {
   [NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
   [NodeInputKeyEnum.datasetDeepSearchBg]?: string;
 };
-export const deepRagSearch = (data: DeepRagSearchProps) =>
-  POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);
+export const deepRagSearch = (data: DeepRagSearchProps) => global.deepRagHandler(data);

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -29,9 +29,9 @@ import { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/templ
 import { getFileContentFromLinks, getHistoryFileLinks } from '../../tools/readFiles';
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { postTextCensor } from '../../../../../common/api/requestPlusApi';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
 import { getDocumentQuotePrompt } from '@fastgpt/global/core/ai/prompt/AIChat';
+import { postTextCensor } from '../../../../chat/postTextCensor';
 
 type Response = DispatchNodeResultType<{
   [NodeOutputKeyEnum.answerText]: string;

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -13,7 +13,6 @@ import type {
 } from '@fastgpt/global/core/ai/type.d';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
 import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
-import { postTextCensor } from '../../../../common/api/requestPlusApi';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import type {
   ChatDispatchProps,
@@ -51,6 +50,7 @@ import { getFileContentFromLinks, getHistoryFileLinks } from '../tools/readFiles
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { i18nT } from '../../../../../web/i18n/utils';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
+import { postTextCensor } from '../../../chat/postTextCensor';
 
 export type ChatProps = ModuleDispatchProps<
   AIChatNodeProps & {

packages/service/support/openapi/auth.ts

 import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 import { updateApiKeyUsedTime } from './tools';
 import { MongoOpenApi } from './schema';
-import { POST } from '../../common/api/plusRequest';
 import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
 
 export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
@@ -17,11 +16,10 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
     }
 
     // auth limit
-    // @ts-ignore
     if (global.feConfigs?.isPlus) {
-      await POST('/support/openapi/authLimit', {
+      await global.authOpenApiHandler({
         openApi
-      } as AuthOpenApiLimitProps);
+      });
     }
 
     updateApiKeyUsedTime(openApi._id);

packages/service/support/wallet/usage/controller.ts

 import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
 import { MongoUsage } from './schema';
-import { ClientSession, Types } from '../../../common/mongo';
+import { ClientSession } from '../../../common/mongo';
 import { addLog } from '../../../common/system/log';
 import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
 import { i18nT } from '../../../../web/i18n/utils';
-import { pushConcatBillTask, pushReduceTeamAiPointsTask } from './utils';
-
-import { POST } from '../../../common/api/plusRequest';
-import { isFastGPTMainService } from '../../../common/system/constants';
 
 export async function createUsage(data: CreateUsageProps) {
   try {
-    // In FastGPT server
-    if (isFastGPTMainService) {
-      await POST('/support/wallet/usage/createUsage', data);
-    } else if (global.reduceAiPointsQueue) {
-      // In FastGPT pro server
-      await MongoUsage.create(data);
-      pushReduceTeamAiPointsTask({ teamId: data.teamId, totalPoints: data.totalPoints });
-
-      if (data.totalPoints === 0) {
-        addLog.info('0 totalPoints', data);
-      }
-    }
+    await global.createUsageHandler(data);
   } catch (error) {
     addLog.error('createUsage error', error);
   }
 }
 export async function concatUsage(data: ConcatUsageProps) {
   try {
-    // In FastGPT server
-    if (isFastGPTMainService) {
-      await POST('/support/wallet/usage/concatUsage', data);
-    } else if (global.reduceAiPointsQueue) {
-      const {
-        teamId,
-        billId,
-        totalPoints = 0,
-        listIndex,
-        inputTokens = 0,
-        outputTokens = 0
-      } = data;
-
-      // billId is required and valid
-      if (!billId || !Types.ObjectId.isValid(billId)) return;
-
-      // In FastGPT pro server
-      pushConcatBillTask([
-        {
-          billId,
-          listIndex,
-          inputTokens,
-          outputTokens,
-          totalPoints
-        }
-      ]);
-      pushReduceTeamAiPointsTask({ teamId, totalPoints });
-
-      if (data.totalPoints === 0) {
-        addLog.info('0 totalPoints', data);
-      }
-    }
+    await global.concatUsageHandler(data);
   } catch (error) {
     addLog.error('concatUsage error', error);
   }

packages/service/support/wallet/usage/utils.ts

 import { findAIModel } from '../../../core/ai/model';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
-import { ConcatBillQueueItemType } from './type';
 
 export const formatModelChars2Points = ({
   model,
@@ -35,20 +34,3 @@ export const formatModelChars2Points = ({
     totalPoints
   };
 };
-
-export const pushReduceTeamAiPointsTask = ({
-  teamId,
-  totalPoints
-}: {
-  teamId: string;
-  totalPoints: number;
-}) => {
-  global.reduceAiPointsQueue.push({
-    teamId: String(teamId),
-    totalPoints
-  });
-};
-
-export const pushConcatBillTask = (data: ConcatBillQueueItemType[]) => {
-  global.concatBillQueue.push(...data);
-};

packages/web/i18n/en/dataset.json

@@ -97,7 +97,7 @@
   "permission.des.read": "View knowledge base content",
   "permission.des.write": "Ability to add and change knowledge base content",
   "preview_chunk": "Preview chunks",
-  "preview_chunk_empty": "Unable to read the contents of the file",
+  "preview_chunk_empty": "File content is empty",
   "preview_chunk_intro": "A total of {{total}} blocks, up to 10",
   "preview_chunk_not_selected": "Click on the file on the left to preview",
   "process.Auto_Index": "Automatic index generation",

packages/web/i18n/zh-CN/dataset.json

@@ -97,7 +97,7 @@
   "permission.des.read": "可查看知识库内容",
   "permission.des.write": "可增加和变更知识库内容",
   "preview_chunk": "分块预览",
-  "preview_chunk_empty": "无法读取该文件内容",
+  "preview_chunk_empty": "文件内容为空",
   "preview_chunk_intro": "共 {{total}} 个分块，最多展示 10 个",
   "preview_chunk_not_selected": "点击左侧文件后进行预览",
   "process.Auto_Index": "自动索引生成",

packages/web/i18n/zh-Hant/dataset.json

@@ -97,7 +97,7 @@
   "permission.des.read": "可檢視資料集內容",
   "permission.des.write": "可新增和變更資料集內容",
   "preview_chunk": "分塊預覽",
-  "preview_chunk_empty": "無法讀取該文件內容",
+  "preview_chunk_empty": "文件內容為空",
   "preview_chunk_intro": "共 {{total}} 個分塊，最多展示 10 個",
   "preview_chunk_not_selected": "點擊左側文件後進行預覽",
   "process.Auto_Index": "自動索引生成",

projects/app/src/pages/api/core/dataset/apiDataset/list.ts

-import { getFeishuAndYuqueDatasetFileList } from '@/service/core/dataset/apiDataset/controller';
 import { NextAPI } from '@/service/middleware/entry';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
-import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
 import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
@@ -14,8 +12,6 @@ export type GetApiDatasetFileListProps = {
   datasetId: string;
 };
 
-export type GetApiDatasetFileListResponse = APIFileItem[];
-
 async function handler(req: NextApiRequest) {
   let { searchKey = '', parentId = null, datasetId } = req.body;
 
@@ -35,7 +31,7 @@ async function handler(req: NextApiRequest) {
     return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
   }
   if (feishuServer || yuqueServer) {
-    return getFeishuAndYuqueDatasetFileList({
+    return global.getProApiDatasetFileList({
       feishuServer,
       yuqueServer,
       parentId

projects/app/src/pages/api/core/dataset/collection/read.ts

@@ -10,7 +10,6 @@ import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
 import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
 import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
-import { POST } from '@fastgpt/service/common/api/plusRequest';
 
 export type readCollectionSourceQuery = {};
 
@@ -106,8 +105,7 @@ async function handler(
       }
 
       if (feishuServer || yuqueServer) {
-        return POST<string>(`/core/dataset/systemApiDataset`, {
-          type: 'read',
+        return global.getProApiDatasetFilePreviewUrl({
           apiFileId: collection.apiFileId,
           feishuServer,
           yuqueServer

projects/app/src/service/common/system/index.ts

@@ -11,6 +11,18 @@ import { defaultGroup, defaultTemplateTypes } from '@fastgpt/web/core/workflow/c
 import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
 import { MongoTemplateTypes } from '@fastgpt/service/core/app/templates/templateTypeSchema';
 import { loadSystemModels } from '@fastgpt/service/core/ai/config/utils';
+import { POST } from '@fastgpt/service/common/api/plusRequest';
+import {
+  DeepRagSearchProps,
+  SearchDatasetDataResponse
+} from '@fastgpt/service/core/dataset/search/controller';
+import { AuthOpenApiLimitProps } from '@fastgpt/service/support/openapi/auth';
+import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
+import {
+  getProApiDatasetFileContentRequest,
+  getProApiDatasetFileListRequest,
+  getProApiDatasetFilePreviewUrlRequest
+} from '@/service/core/dataset/apiDataset/controller';
 
 export const readConfigData = async (name: string) => {
   const splitName = name.split('.');
@@ -36,12 +48,37 @@ export const readConfigData = async (name: string) => {
 
 /* Init global variables */
 export function initGlobalVariables() {
-  if (global.communityPlugins) return;
+  function initPlusRequest() {
+    global.textCensorHandler = function textCensorHandler({ text }: { text: string }) {
+      return POST<{ code: number; message?: string }>('/common/censor/check', { text });
+    };
+
+    global.deepRagHandler = function deepRagHandler(data: DeepRagSearchProps) {
+      return POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);
+    };
+
+    global.authOpenApiHandler = function authOpenApiHandler(data: AuthOpenApiLimitProps) {
+      return POST<AuthOpenApiLimitProps>('/support/openapi/authLimit', data);
+    };
+
+    global.createUsageHandler = function createUsageHandler(data: CreateUsageProps) {
+      return POST('/support/wallet/usage/createUsage', data);
+    };
+
+    global.concatUsageHandler = function concatUsageHandler(data: ConcatUsageProps) {
+      return POST('/support/wallet/usage/concatUsage', data);
+    };
+
+    global.getProApiDatasetFileList = getProApiDatasetFileListRequest;
+    global.getProApiDatasetFileContent = getProApiDatasetFileContentRequest;
+    global.getProApiDatasetFilePreviewUrl = getProApiDatasetFilePreviewUrlRequest;
+  }
 
   global.communityPlugins = [];
   global.qaQueueLen = global.qaQueueLen ?? 0;
   global.vectorQueueLen = global.vectorQueueLen ?? 0;
   initHttpAgent();
+  initPlusRequest();
 }
 
 /* Init system data(Need to connected db). It only needs to run once */

projects/app/src/service/core/dataset/apiDataset/controller.ts

-import { GetApiDatasetFileListResponse } from '@/pages/api/core/dataset/apiDataset/list';
-import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
+import { APIFileItem, ApiFileReadContentResponse } from '@fastgpt/global/core/dataset/apiDataset';
 import { POST } from '@fastgpt/service/common/api/plusRequest';
+import {
+  GetProApiDatasetFileContentParams,
+  GetProApiDatasetFileListParams,
+  GetProApiDatasetFilePreviewUrlParams,
+  ProApiDatasetOperationTypeEnum
+} from '@fastgpt/service/core/dataset/apiDataset/proApi';
 
-export const getFeishuAndYuqueDatasetFileList = async (data: {
-  feishuServer?: FeishuServer;
-  yuqueServer?: YuqueServer;
-  parentId?: string;
-}) => {
-  const res = await POST<GetApiDatasetFileListResponse>('/core/dataset/systemApiDataset', {
-    type: 'list',
+export const getProApiDatasetFileListRequest = async (data: GetProApiDatasetFileListParams) => {
+  const res = await POST<APIFileItem[]>('/core/dataset/systemApiDataset', {
+    type: ProApiDatasetOperationTypeEnum.LIST,
+    ...data
+  });
+  return res;
+};
+
+export const getProApiDatasetFileContentRequest = async (
+  data: GetProApiDatasetFileContentParams
+) => {
+  const res = await POST<ApiFileReadContentResponse>('/core/dataset/systemApiDataset', {
+    type: ProApiDatasetOperationTypeEnum.CONTENT,
+    ...data
+  });
+  return res;
+};
+
+export const getProApiDatasetFilePreviewUrlRequest = async (
+  data: GetProApiDatasetFilePreviewUrlParams
+) => {
+  const res = await POST<string>('/core/dataset/systemApiDataset', {
+    type: ProApiDatasetOperationTypeEnum.READ,
     ...data
   });
   return res;

projects/app/src/web/core/dataset/api.ts

@@ -52,10 +52,7 @@ import type {
 import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller';
 import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create';
 import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
-import type {
-  GetApiDatasetFileListProps,
-  GetApiDatasetFileListResponse
-} from '@/pages/api/core/dataset/apiDataset/list';
+import type { GetApiDatasetFileListProps } from '@/pages/api/core/dataset/apiDataset/list';
 import type {
   listExistIdQuery,
   listExistIdResponse
@@ -74,6 +71,7 @@ import type {
   getTrainingErrorBody,
   getTrainingErrorResponse
 } from '@/pages/api/core/dataset/training/getTrainingError';
+import type { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
 
 /* ======================== dataset ======================= */
 export const getDatasets = (data: GetDatasetListBody) =>
@@ -254,6 +252,6 @@ export const getCollectionSource = (data: readCollectionSourceBody) =>
 
 /* ================== apiDataset ======================== */
 export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) =>
-  POST<GetApiDatasetFileListResponse>('/core/dataset/apiDataset/list', data);
+  POST<APIFileItem[]>('/core/dataset/apiDataset/list', data);
 export const getApiDatasetFileListExistId = (data: listExistIdQuery) =>
   GET<listExistIdResponse>('/core/dataset/apiDataset/listExistId', data);