Add mongo forward read. Fix markdown code block. (#2174)

* perf: read from mongo forward node * fix: code block ui * perf: markdown code css

Add mongo forward read. Fix markdown code block. (#2174)
* perf: read from mongo forward node * fix: code block ui * perf: markdown code css
23f22cda · Archer · GitHub · 0a9a7691 · 23f22cda · 23f22cda
Commit 23f22cda authored Jul 29, 2024 by Archer Committed by GitHub Jul 29, 2024
10 changed files
--- a/packages/service/common/file/image/controller.ts
+++ b/packages/service/common/file/image/controller.ts
@@ -3,6 +3,7 @@ import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
 import { MongoImage } from './schema';
 import { ClientSession } from '../../../common/mongo';
 import { guessBase64ImageType } from '../utils';
+import { readFromSecondary } from '../../mongo/utils';
 export function getMongoImgUrl(id: string, extension: string) {
  return `${imageBaseUrl}${id}.${extension}`;
@@ -44,10 +45,13 @@ export async function uploadMongoImg({
 export async function readMongoImg({ id }: { id: string }) {
  const formatId = id.replace(/\.[^/.]+$/, '');
-  const data = await MongoImage.findById(formatId);
+  const data = await MongoImage.findById(formatId, undefined, {
+    ...readFromSecondary
+  });
  if (!data) {
    return Promise.reject('Image not found');
  }
  return {
    binary: data.binary,
    mime: data.metadata?.mime ?? guessBase64ImageType(data.binary.toString('base64'))

--- a/packages/service/common/mongo/utils.ts
+++ b/packages/service/common/mongo/utils.ts
+import { ReadPreference } from './index';
 export const readFromSecondary = {
-  readPreference: 'secondaryPreferred',
+  readPreference: ReadPreference.SECONDARY_PREFERRED, // primary | primaryPreferred | secondary | secondaryPreferred | nearest
-  readConcern: 'local'
+  readConcern: 'local' as any // local | majority | linearizable | available
 };
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -61,7 +61,7 @@ importers:
        version: 4.0.2
      next:
        specifier: 14.2.5
-        version: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+        version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      openai:
        specifier: 4.53.0
        version: 4.53.0(encoding@0.1.13)
@@ -180,7 +180,7 @@ importers:
        version: 1.4.5-lts.1
      next:
        specifier: 14.2.5
-        version: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+        version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      nextjs-cors:
        specifier: ^2.2.0
        version: 2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))
@@ -10036,7 +10036,7 @@ snapshots:
      '@chakra-ui/react': 2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
      '@emotion/cache': 11.11.0
      '@emotion/react': 11.11.1(@types/react@18.3.1)(react@18.3.1)
-      next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+      next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      react: 18.3.1
  '@chakra-ui/number-input@2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)':
@@ -16652,7 +16652,7 @@ snapshots:
      hoist-non-react-statics: 3.3.2
      i18next: 23.11.5
      i18next-fs-backend: 2.3.1
-      next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+      next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      react: 18.3.1
      react-i18next: 14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -16682,10 +16682,36 @@ snapshots:
      - '@babel/core'
      - babel-plugin-macros
+  next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8):
+    dependencies:
+      '@next/env': 14.2.5
+      '@swc/helpers': 0.5.5
+      busboy: 1.6.0
+      caniuse-lite: 1.0.30001642
+      graceful-fs: 4.2.11
+      postcss: 8.4.31
+      react: 18.3.1
+      react-dom: 18.3.1(react@18.3.1)
+      styled-jsx: 5.1.1(react@18.3.1)
+    optionalDependencies:
+      '@next/swc-darwin-arm64': 14.2.5
+      '@next/swc-darwin-x64': 14.2.5
+      '@next/swc-linux-arm64-gnu': 14.2.5
+      '@next/swc-linux-arm64-musl': 14.2.5
+      '@next/swc-linux-x64-gnu': 14.2.5
+      '@next/swc-linux-x64-musl': 14.2.5
+      '@next/swc-win32-arm64-msvc': 14.2.5
+      '@next/swc-win32-ia32-msvc': 14.2.5
+      '@next/swc-win32-x64-msvc': 14.2.5
+      sass: 1.77.8
+    transitivePeerDependencies:
+      - '@babel/core'
+      - babel-plugin-macros
  nextjs-cors@2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)):
    dependencies:
      cors: 2.8.5
-      next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+      next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
  nextjs-node-loader@1.1.5(webpack@5.92.1):
    dependencies:
@@ -18058,6 +18084,11 @@ snapshots:
    optionalDependencies:
      '@babel/core': 7.24.9
+  styled-jsx@5.1.1(react@18.3.1):
+    dependencies:
+      client-only: 0.0.1
+      react: 18.3.1
  stylis@4.2.0: {}
  stylis@4.3.2: {}

--- a/projects/app/src/components/Markdown/CodeLight.tsx
+++ b/projects/app/src/components/Markdown/CodeLight.tsx
@@ -287,18 +287,18 @@ const codeLight: { [key: string]: React.CSSProperties } = {
 const CodeLight = ({
  children,
  className,
-  inline,
+  codeBlock,
  match
 }: {
  children: React.ReactNode & React.ReactNode[];
  className?: string;
-  inline?: boolean;
+  codeBlock?: boolean;
  match: RegExpExecArray | null;
 }) => {
  const { t } = useTranslation();
  const { copyData } = useCopyData();
-  if (!inline) {
+  if (codeBlock) {
    const codeBoxName = useMemo(() => {
      const input = match?.['input'] || '';
      if (!input) return match?.[1];
@@ -312,7 +312,6 @@ const CodeLight = ({
        my={3}
        borderRadius={'md'}
        overflow={'overlay'}
-        bg={'myGray.900'}
        boxShadow={
          '0px 0px 1px 0px rgba(19, 51, 107, 0.08), 0px 1px 2px 0px rgba(19, 51, 107, 0.05)'
        }

--- a/projects/app/src/components/Markdown/index.module.scss
+++ b/projects/app/src/components/Markdown/index.module.scss
@@ -336,7 +336,7 @@
 }
 .markdown pre code,
 .markdown pre tt {
-  background-color: transparent;
+  background-color: transparent !important;
  border: medium none;
 }
 .markdown hr {
@@ -360,13 +360,12 @@
    margin: 0;
    border: none;
    border-radius: 0;
-    background-color: #292b33 !important;
+    background-color: var(--chakra-colors-gray-900) !important;
    overflow-x: auto;
    color: #fff;
  }
  pre code {
-    background-color: #292b33 !important;
    width: 100%;
  }

--- a/projects/app/src/components/Markdown/index.tsx
+++ b/projects/app/src/components/Markdown/index.tsx
@@ -36,7 +36,7 @@ const Markdown = ({
  const components = useMemo<any>(
    () => ({
      img: Image,
-      pre: 'div',
+      pre: RewritePre,
      p: (pProps: any) => <p {...pProps} dir="auto" />,
      code: Code,
      a: A
@@ -70,8 +70,7 @@ export default React.memo(Markdown);
 /* Custom dom */
 const Code = React.memo(function Code(e: any) {
-  const { inline, className, children } = e;
+  const { className, codeBlock, children } = e;
  const match = /language-(\w+)/.exec(className || '');
  const codeType = match?.[1];
@@ -92,11 +91,11 @@ const Code = React.memo(function Code(e: any) {
    }
    return (
-      <CodeLight className={className} inline={inline} match={match}>
+      <CodeLight className={className} codeBlock={codeBlock} match={match}>
        {children}
      </CodeLight>
    );
-  }, [codeType, className, inline, match, children, strChildren]);
+  }, [codeType, className, codeBlock, match, children, strChildren]);
  return Component;
 });
@@ -149,3 +148,15 @@ const A = React.memo(function A({ children, ...props }: any) {
  return <Link {...props}>{children}</Link>;
 });
+function RewritePre({ children }: any) {
+  const modifiedChildren = React.Children.map(children, (child) => {
+    if (React.isValidElement(child)) {
+      // @ts-ignore
+      return React.cloneElement(child, { codeBlock: true });
+    }
+    return child;
+  });
+  return <>{modifiedChildren}</>;
+}
--- a/projects/app/src/pages/api/core/app/getChatLogs.ts
+++ b/projects/app/src/pages/api/core/app/getChatLogs.ts
@@ -9,6 +9,7 @@ import { authApp } from '@fastgpt/service/support/permission/app/auth';
 import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
 import { NextAPI } from '@/service/middleware/entry';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 async function handler(
  req: NextApiRequest,
@@ -39,7 +40,8 @@ async function handler(
  };
  const [data, total] = await Promise.all([
-    MongoChat.aggregate([
+    MongoChat.aggregate(
+      [
        { $match: where },
        {
          $sort: {
@@ -132,8 +134,12 @@ async function handler(
            markCount: 1
          }
        }
-    ]),
+      ],
-    MongoChat.countDocuments(where)
+      {
+        ...readFromSecondary
+      }
+    ),
+    MongoChat.countDocuments(where, { ...readFromSecondary })
  ]);
  return {

--- a/projects/app/src/pages/api/core/dataset/exportAll.ts
+++ b/projects/app/src/pages/api/core/dataset/exportAll.ts
@@ -11,6 +11,7 @@ import {
 import { NextAPI } from '@/service/middleware/entry';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  let { datasetId } = req.query as {
@@ -53,7 +54,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
      teamId,
      datasetId: { $in: datasets.map((d) => d._id) }
    },
-    'q a'
+    'q a',
+    {
+      ...readFromSecondary
+    }
  )
    .limit(50000)
    .cursor();

--- a/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts
+++ b/projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts
@@ -4,6 +4,7 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 export type getDatasetTrainingQueueResponse = {
  rebuildingCount: number;
@@ -24,8 +25,18 @@ async function handler(
  });
  const [rebuildingCount, trainingCount] = await Promise.all([
-    MongoDatasetData.countDocuments({ rebuilding: true, teamId, datasetId }),
+    MongoDatasetData.countDocuments(
-    MongoDatasetTraining.countDocuments({ teamId, datasetId })
+      { rebuilding: true, teamId, datasetId },
+      {
+        ...readFromSecondary
+      }
+    ),
+    MongoDatasetTraining.countDocuments(
+      { teamId, datasetId },
+      {
+        ...readFromSecondary
+      }
+    )
  ]);
  return {

--- a/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
+++ b/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
@@ -3,6 +3,7 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { GetTrainingQueueProps } from '@/global/core/dataset/api';
 import { NextAPI } from '@/service/middleware/entry';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 async function handler(req: NextApiRequest) {
  await authCert({ req, authToken: true });
@@ -10,7 +11,8 @@ async function handler(req: NextApiRequest) {
  // get queue data
  // 分别统计 model = vectorModel和agentModel的数量
-  const data = await MongoDatasetTraining.aggregate([
+  const data = await MongoDatasetTraining.aggregate(
+    [
      {
        $match: {
          lockTime: { $lt: new Date('2040/1/1') },
@@ -23,7 +25,11 @@ async function handler(req: NextApiRequest) {
          count: { $sum: 1 }
        }
      }
-  ]);
+    ],
+    {
+      ...readFromSecondary
+    }
+  );
  const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
  const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;