4.8.14 test (#3164)

* perf: match base 64 image * perf: register plugins

4.8.14 test (#3164)
* perf: match base 64 image * perf: register plugins
58745f8c · Archer · GitHub · f699061d · 58745f8c · 58745f8c
Commit 58745f8c authored Nov 15, 2024 by Archer Committed by GitHub Nov 15, 2024
9 changed files
--- a/packages/global/common/string/markdown.ts
+++ b/packages/global/common/string/markdown.ts
 import { batchRun } from '../fn/utils';
-import { simpleText } from './tools';
+import { getNanoid, simpleText } from './tools';
+import type { ImageType } from '../../../service/worker/readFile/type';
 /* Delete redundant text in markdown */
 export const simpleMarkdownText = (rawText: string) => {
@@ -92,3 +93,25 @@ export const markdownProcess = async ({
  return simpleMarkdownText(imageProcess);
 };
+export const matchMdImgTextAndUpload = (text: string) => {
+  const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g;
+  const imageList: ImageType[] = [];
+  const images = Array.from(text.match(base64Regex) || []);
+  for (const image of images) {
+    const uuid = `IMAGE_${getNanoid(12)}_IMAGE`;
+    const mime = image.split(';')[0].split(':')[1];
+    const base64 = image.split(',')[1];
+    text = text.replace(image, uuid);
+    imageList.push({
+      uuid,
+      base64,
+      mime
+    });
+  }
+  return {
+    text,
+    imageList
+  };
+};
--- a/packages/plugins/register.ts
+++ b/packages/plugins/register.ts
@@ -46,6 +46,8 @@ export const getCommunityPlugins = () => {
 };
 export const getSystemPluginTemplates = () => {
+  if (!global.systemPlugins) return [];
  const oldPlugins = global.communityPlugins ?? [];
  return [...oldPlugins, ...cloneDeep(global.systemPlugins)];
 };

--- a/packages/service/common/file/read/utils.ts
+++ b/packages/service/common/file/read/utils.ts
@@ -4,12 +4,12 @@ import FormData from 'form-data';
 import { WorkerNameEnum, runWorker } from '../../../worker/utils';
 import fs from 'fs';
-import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
 import type { ReadFileResponse } from '../../../worker/readFile/type';
 import axios from 'axios';
 import { addLog } from '../../system/log';
 import { batchRun } from '@fastgpt/global/common/fn/utils';
 import { addHours } from 'date-fns';
+import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
 export type readRawTextByLocalFileParams = {
  teamId: string;
@@ -79,6 +79,7 @@ export const readRawContentByFileBuffer = async ({
      data: {
        page: number;
        markdown: string;
+        duration: number;
      };
    }>(customReadfileUrl, data, {
      timeout: 600000,
@@ -90,10 +91,12 @@ export const readRawContentByFileBuffer = async ({
    addLog.info(`Use custom read file service, time: ${Date.now() - start}ms`);
    const rawText = response.data.markdown;
+    const { text, imageList } = matchMdImgTextAndUpload(rawText);
    return {
-      rawText,
+      rawText: text,
-      formatText: rawText
+      formatText: rawText,
+      imageList
    };
  };
@@ -120,6 +123,9 @@ export const readRawContentByFileBuffer = async ({
        }
      });
      rawText = rawText.replace(item.uuid, src);
+      if (formatText) {
+        formatText = formatText.replace(item.uuid, src);
+      }
    });
  }
@@ -128,7 +134,7 @@ export const readRawContentByFileBuffer = async ({
    if (isQAImport) {
      rawText = rawText || '';
    } else {
-      rawText = formatText || '';
+      rawText = formatText || rawText;
    }
  }

--- a/packages/service/worker/htmlStr2Md/utils.ts
+++ b/packages/service/worker/htmlStr2Md/utils.ts
 import TurndownService from 'turndown';
 import { ImageType } from '../readFile/type';
+import { matchMdImgTextAndUpload } from '@fastgpt/global/common/string/markdown';
 // @ts-ignore
 const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
@@ -24,23 +25,10 @@ export const html2md = (
    turndownService.remove(['i', 'script', 'iframe', 'style']);
    turndownService.use(turndownPluginGfm.gfm);
-    const base64Regex = /"(data:image\/[^;]+;base64[^"]+)"/g;
+    const { text, imageList } = matchMdImgTextAndUpload(html);
-    const imageList: ImageType[] = [];
-    const images = Array.from(html.match(base64Regex) || []);
-    for (const image of images) {
-      const uuid = crypto.randomUUID();
-      const mime = image.split(';')[0].split(':')[1];
-      const base64 = image.split(',')[1];
-      html = html.replace(image, uuid);
-      imageList.push({
-        uuid,
-        base64,
-        mime
-      });
-    }
    return {
-      rawText: turndownService.turndown(html),
+      rawText: turndownService.turndown(text),
      imageList
    };
  } catch (error) {

--- a/projects/app/src/components/support/user/inform/SystemMsgModal.tsx
+++ b/projects/app/src/components/support/user/inform/SystemMsgModal.tsx
 import React, { useCallback } from 'react';
 import MyModal from '@fastgpt/web/components/common/MyModal';
 import { useUserStore } from '@/web/support/user/useUserStore';
-import { useQuery } from '@tanstack/react-query';
 import { Button, ModalBody, ModalFooter, useDisclosure } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
 import { LOGO_ICON } from '@fastgpt/global/common/system/constants';
 import { getSystemMsgModalData } from '@/web/support/user/inform/api';
 import dynamic from 'next/dynamic';
+import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 const Markdown = dynamic(() => import('@/components/Markdown'), { ssr: false });
 const SystemMsgModal = ({}: {}) => {
@@ -15,7 +15,9 @@ const SystemMsgModal = ({}: {}) => {
  const { isOpen, onOpen, onClose } = useDisclosure();
-  const { data } = useQuery(['initSystemMsgModal', systemMsgReadId], getSystemMsgModalData, {
+  const { data } = useRequest2(getSystemMsgModalData, {
+    refreshDeps: [systemMsgReadId],
+    manual: false,
    onSuccess(res) {
      if (res?.content && (!systemMsgReadId || res.id !== systemMsgReadId)) {
        onOpen();

--- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/Handle/ConnectionHandle.tsx
+++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/Handle/ConnectionHandle.tsx
@@ -6,6 +6,7 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { useContextSelector } from 'use-context-selector';
 import { WorkflowContext } from '../../../../context';
 import { WorkflowNodeEdgeContext } from '../../../../context/workflowInitContext';
+import { FlowNodeItemType } from '@fastgpt/global/core/workflow/type/node';
 export const ConnectionSourceHandle = ({
  nodeId,
@@ -141,15 +142,23 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
  const { connectingEdge, nodeList } = useContextSelector(WorkflowContext, (ctx) => ctx);
  const { LeftHandle, rightHandle, topHandle, bottomHandle } = useMemo(() => {
-    const node = nodeList.find((node) => node.nodeId === nodeId);
+    let node: FlowNodeItemType | undefined = undefined,
-    const connectingNode = nodeList.find((node) => node.nodeId === connectingEdge?.nodeId);
+      connectingNode: FlowNodeItemType | undefined = undefined;
+    for (const item of nodeList) {
+      if (item.nodeId === nodeId) {
+        node = item;
+      }
+      if (item.nodeId === connectingEdge?.nodeId) {
+        connectingNode = item;
+      }
+      if (node && (connectingNode || !connectingEdge?.nodeId)) break;
+    }
-    const connectingNodeSourceNodeIdMap = new Map<string, number>();
    let forbidConnect = false;
-    edges.forEach((edge) => {
+    for (const edge of edges) {
-      if (edge.target === connectingNode?.nodeId) {
+      if (forbidConnect) break;
-        connectingNodeSourceNodeIdMap.set(edge.source, 1);
-      } else if (edge.target === nodeId) {
+      if (edge.target === nodeId) {
        // Node has be connected tool, it cannot be connect by other handle
        if (edge.targetHandle === NodeOutputKeyEnum.selectedTools) {
          forbidConnect = true;
@@ -163,7 +172,7 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
          forbidConnect = true;
        }
      }
-    });
+    }
    const showHandle = (() => {
      if (forbidConnect) return false;
@@ -178,9 +187,6 @@ export const ConnectionTargetHandle = React.memo(function ConnectionTargetHandle
      // Not the same parent node
      if (connectingNode && connectingNode?.parentNodeId !== node?.parentNodeId) return false;
-      // Unable to connect to the source node
-      if (connectingNodeSourceNodeIdMap.has(nodeId)) return false;
      return true;
    })();

--- a/projects/app/src/service/common/system/volumnMongoWatch.ts
+++ b/projects/app/src/service/common/system/volumnMongoWatch.ts
-import { getSystemPluginCb, getSystemPlugins } from '@/service/core/app/plugin';
+import { getSystemPluginCb } from '@/service/core/app/plugin';
 import { initSystemConfig } from '.';
 import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils';
 import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema';
@@ -29,7 +29,7 @@ const refetchSystemPlugins = () => {
  changeStream.on('change', async (change) => {
    setTimeout(() => {
      try {
-        getSystemPlugins(true);
+        getSystemPluginCb(true);
      } catch (error) {}
    }, 5000);
  });

--- a/projects/app/src/service/core/app/plugin.ts
+++ b/projects/app/src/service/core/app/plugin.ts
@@ -11,7 +11,8 @@ const getCommercialPlugins = () => {
  return GET<SystemPluginTemplateItemType[]>('/core/app/plugin/getSystemPlugins');
 };
 export const getSystemPlugins = async (refresh = false) => {
-  if (isProduction && global.systemPlugins && !refresh) return cloneDeep(global.systemPlugins);
+  if (isProduction && global.systemPlugins && global.systemPlugins.length > 0 && !refresh)
+    return cloneDeep(global.systemPlugins);
  try {
    if (!global.systemPlugins) {
@@ -22,8 +23,6 @@ export const getSystemPlugins = async (refresh = false) => {
    addLog.info(`Load system plugin successfully: ${global.systemPlugins.length}`);
-    getSystemPluginCb();
    return cloneDeep(global.systemPlugins);
  } catch (error) {
    //@ts-ignore
@@ -56,16 +55,22 @@ const getCommercialCb = async () => {
    {}
  );
 };
-export const getSystemPluginCb = async () => {
-  if (isProduction && global.systemPluginCb) return global.systemPluginCb;
+export const getSystemPluginCb = async (refresh = false) => {
+  if (
+    isProduction &&
+    global.systemPluginCb &&
+    Object.keys(global.systemPluginCb).length > 0 &&
+    !refresh
+  )
+    return global.systemPluginCb;
  try {
    global.systemPluginCb = {};
+    await getSystemPlugins();
    global.systemPluginCb = FastGPTProUrl ? await getCommercialCb() : await getCommunityCb();
    return global.systemPluginCb;
  } catch (error) {
-    //@ts-ignore
-    global.systemPluginCb = undefined;
    return Promise.reject(error);
  }
 };
--- a/projects/app/src/service/mongo.ts
+++ b/projects/app/src/service/mongo.ts
@@ -12,7 +12,7 @@ import { startMongoWatch } from './common/system/volumnMongoWatch';
 import { startTrainingQueue } from './core/dataset/training/utils';
 import { systemStartCb } from '@fastgpt/service/common/system/tools';
 import { addLog } from '@fastgpt/service/common/system/log';
-import { getSystemPlugins } from './core/app/plugin';
+import { getSystemPluginCb } from './core/app/plugin';
 /**
 * This function is equivalent to the entry to the service
@@ -34,7 +34,7 @@ export function connectToDatabase() {
      //init system config；init vector database；init root user
      await Promise.all([getInitConfig(), initVectorStore(), initRootUser()]);
-      getSystemPlugins();
+      getSystemPluginCb();
      startMongoWatch();
      // cron
      startCron();