Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 23f22cda by Archer Committed by GitHub
Options

Add mongo forward read. Fix markdown code block. (#2174) 

* perf: read from mongo forward node

* fix: code block ui

* perf: markdown code css
parent 0a9a7691
Showing with 191 additions and 118 deletions
packages/service/common/file/image/controller.ts
......@@ -3,6 +3,7 @@ import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
import { MongoImage } from './schema';
import { ClientSession } from '../../../common/mongo';
import { guessBase64ImageType } from '../utils';
import { readFromSecondary } from '../../mongo/utils';
export function getMongoImgUrl(id: string, extension: string) {
return `${imageBaseUrl}${id}.${extension}`;
......@@ -44,10 +45,13 @@ export async function uploadMongoImg({
export async function readMongoImg({ id }: { id: string }) {
const formatId = id.replace(/\.[^/.]+$/, '');
const data = await MongoImage.findById(formatId);
const data = await MongoImage.findById(formatId, undefined, {
...readFromSecondary
});
if (!data) {
return Promise.reject('Image not found');
}
return {
binary: data.binary,
mime: data.metadata?.mime ?? guessBase64ImageType(data.binary.toString('base64'))
......
packages/service/common/mongo/utils.ts
import { ReadPreference } from './index';
export const readFromSecondary = {
readPreference: 'secondaryPreferred',
readConcern: 'local'
readPreference: ReadPreference.SECONDARY_PREFERRED, // primary | primaryPreferred | secondary | secondaryPreferred | nearest
readConcern: 'local' as any // local | majority | linearizable | available
};
pnpm-lock.yaml
......@@ -61,7 +61,7 @@ importers:
version: 4.0.2
next:
specifier: 14.2.5
version: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
openai:
specifier: 4.53.0
version: 4.53.0(encoding@0.1.13)
......@@ -180,7 +180,7 @@ importers:
version: 1.4.5-lts.1
next:
specifier: 14.2.5
version: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
nextjs-cors:
specifier: ^2.2.0
version: 2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))
......@@ -10036,7 +10036,7 @@ snapshots:
'@chakra-ui/react': 2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@emotion/cache': 11.11.0
'@emotion/react': 11.11.1(@types/react@18.3.1)(react@18.3.1)
next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
react: 18.3.1
'@chakra-ui/number-input@2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)':
......@@ -16652,7 +16652,7 @@ snapshots:
hoist-non-react-statics: 3.3.2
i18next: 23.11.5
i18next-fs-backend: 2.3.1
next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
react: 18.3.1
react-i18next: 14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
......@@ -16682,10 +16682,36 @@ snapshots:
- '@babel/core'
- babel-plugin-macros
next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8):
dependencies:
'@next/env': 14.2.5
'@swc/helpers': 0.5.5
busboy: 1.6.0
caniuse-lite: 1.0.30001642
graceful-fs: 4.2.11
postcss: 8.4.31
react: 18.3.1
react-dom: 18.3.1(react@18.3.1)
styled-jsx: 5.1.1(react@18.3.1)
optionalDependencies:
'@next/swc-darwin-arm64': 14.2.5
'@next/swc-darwin-x64': 14.2.5
'@next/swc-linux-arm64-gnu': 14.2.5
'@next/swc-linux-arm64-musl': 14.2.5
'@next/swc-linux-x64-gnu': 14.2.5
'@next/swc-linux-x64-musl': 14.2.5
'@next/swc-win32-arm64-msvc': 14.2.5
'@next/swc-win32-ia32-msvc': 14.2.5
'@next/swc-win32-x64-msvc': 14.2.5
sass: 1.77.8
transitivePeerDependencies:
- '@babel/core'
- babel-plugin-macros
nextjs-cors@2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)):
dependencies:
cors: 2.8.5
next: 14.2.5(@babel/core@7.24.9)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
nextjs-node-loader@1.1.5(webpack@5.92.1):
dependencies:
......@@ -18058,6 +18084,11 @@ snapshots:
optionalDependencies:
'@babel/core': 7.24.9
styled-jsx@5.1.1(react@18.3.1):
dependencies:
client-only: 0.0.1
react: 18.3.1
stylis@4.2.0: {}
stylis@4.3.2: {}
......
projects/app/src/components/Markdown/CodeLight.tsx
......@@ -287,18 +287,18 @@ const codeLight: { [key: string]: React.CSSProperties } = {
const CodeLight = ({
children,
className,
inline,
codeBlock,
match
}: {
children: React.ReactNode & React.ReactNode[];
className?: string;
inline?: boolean;
codeBlock?: boolean;
match: RegExpExecArray | null;
}) => {
const { t } = useTranslation();
const { copyData } = useCopyData();
if (!inline) {
if (codeBlock) {
const codeBoxName = useMemo(() => {
const input = match?.['input'] || '';
if (!input) return match?.[1];
......@@ -312,7 +312,6 @@ const CodeLight = ({
my={3}
borderRadius={'md'}
overflow={'overlay'}
bg={'myGray.900'}
boxShadow={
'0px 0px 1px 0px rgba(19, 51, 107, 0.08), 0px 1px 2px 0px rgba(19, 51, 107, 0.05)'
}
......
projects/app/src/components/Markdown/index.module.scss
......@@ -336,7 +336,7 @@
}
.markdown pre code,
.markdown pre tt {
background-color: transparent;
background-color: transparent !important;
border: medium none;
}
.markdown hr {
......@@ -360,13 +360,12 @@
margin: 0;
border: none;
border-radius: 0;
background-color: #292b33 !important;
background-color: var(--chakra-colors-gray-900) !important;
overflow-x: auto;
color: #fff;
}
pre code {
background-color: #292b33 !important;
width: 100%;
}
......
projects/app/src/components/Markdown/index.tsx
......@@ -36,7 +36,7 @@ const Markdown = ({
const components = useMemo<any>(
() => ({
img: Image,
pre: 'div',
pre: RewritePre,
p: (pProps: any) => <p {...pProps} dir="auto" />,
code: Code,
a: A
......@@ -70,8 +70,7 @@ export default React.memo(Markdown);
/* Custom dom */
const Code = React.memo(function Code(e: any) {
const { inline, className, children } = e;
const { className, codeBlock, children } = e;
const match = /language-(\w+)/.exec(className || '');
const codeType = match?.[1];
......@@ -92,11 +91,11 @@ const Code = React.memo(function Code(e: any) {
}
return (
<CodeLight className={className} inline={inline} match={match}>
<CodeLight className={className} codeBlock={codeBlock} match={match}>
{children}
</CodeLight>
);
}, [codeType, className, inline, match, children, strChildren]);
}, [codeType, className, codeBlock, match, children, strChildren]);
return Component;
});
......@@ -149,3 +148,15 @@ const A = React.memo(function A({ children, ...props }: any) {
return <Link {...props}>{children}</Link>;
});
function RewritePre({ children }: any) {
const modifiedChildren = React.Children.map(children, (child) => {
if (React.isValidElement(child)) {
// @ts-ignore
return React.cloneElement(child, { codeBlock: true });
}
return child;
});
return <>{modifiedChildren}</>;
}
projects/app/src/pages/api/core/app/getChatLogs.ts
......@@ -9,6 +9,7 @@ import { authApp } from '@fastgpt/service/support/permission/app/auth';
import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
async function handler(
req: NextApiRequest,
......@@ -39,101 +40,106 @@ async function handler(
};
const [data, total] = await Promise.all([
MongoChat.aggregate([
{ $match: where },
{
$sort: {
userBadFeedbackCount: -1,
userGoodFeedbackCount: -1,
customFeedbacksCount: -1,
updateTime: -1
}
},
{ $skip: (pageNum - 1) * pageSize },
{ $limit: pageSize },
{
$lookup: {
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
MongoChat.aggregate(
[
{ $match: where },
{
$sort: {
userBadFeedbackCount: -1,
userGoodFeedbackCount: -1,
customFeedbacksCount: -1,
updateTime: -1
}
},
{ $skip: (pageNum - 1) * pageSize },
{ $limit: pageSize },
{
$lookup: {
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
}
}
},
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
}
},
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
}
],
as: 'chatitems'
}
},
{
$addFields: {
userGoodFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userGoodFeedback', false] }
],
as: 'chatitems'
}
},
{
$addFields: {
userGoodFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userGoodFeedback', false] }
}
}
}
},
userBadFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userBadFeedback', false] }
},
userBadFeedbackCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.userBadFeedback', false] }
}
}
}
},
customFeedbacksCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] }
},
customFeedbacksCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $gt: [{ $size: { $ifNull: ['$$item.customFeedbacks', []] } }, 0] }
}
}
}
},
markCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.adminFeedback', false] }
},
markCount: {
$size: {
$filter: {
input: '$chatitems',
as: 'item',
cond: { $ifNull: ['$$item.adminFeedback', false] }
}
}
}
}
},
{
$project: {
_id: 1,
id: '$chatId',
title: 1,
source: 1,
time: '$updateTime',
messageCount: { $size: '$chatitems' },
userGoodFeedbackCount: 1,
userBadFeedbackCount: 1,
customFeedbacksCount: 1,
markCount: 1
}
}
},
],
{
$project: {
_id: 1,
id: '$chatId',
title: 1,
source: 1,
time: '$updateTime',
messageCount: { $size: '$chatitems' },
userGoodFeedbackCount: 1,
userBadFeedbackCount: 1,
customFeedbacksCount: 1,
markCount: 1
}
...readFromSecondary
}
]),
MongoChat.countDocuments(where)
),
MongoChat.countDocuments(where, { ...readFromSecondary })
]);
return {
......
projects/app/src/pages/api/core/dataset/exportAll.ts
......@@ -11,6 +11,7 @@ import {
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let { datasetId } = req.query as {
......@@ -53,7 +54,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
teamId,
datasetId: { $in: datasets.map((d) => d._id) }
},
'q a'
'q a',
{
...readFromSecondary
}
)
.limit(50000)
.cursor();
......
projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts
......@@ -4,6 +4,7 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
export type getDatasetTrainingQueueResponse = {
rebuildingCount: number;
......@@ -24,8 +25,18 @@ async function handler(
});
const [rebuildingCount, trainingCount] = await Promise.all([
MongoDatasetData.countDocuments({ rebuilding: true, teamId, datasetId }),
MongoDatasetTraining.countDocuments({ teamId, datasetId })
MongoDatasetData.countDocuments(
{ rebuilding: true, teamId, datasetId },
{
...readFromSecondary
}
),
MongoDatasetTraining.countDocuments(
{ teamId, datasetId },
{
...readFromSecondary
}
)
]);
return {
......
projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
......@@ -3,6 +3,7 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { GetTrainingQueueProps } from '@/global/core/dataset/api';
import { NextAPI } from '@/service/middleware/entry';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
async function handler(req: NextApiRequest) {
await authCert({ req, authToken: true });
......@@ -10,20 +11,25 @@ async function handler(req: NextApiRequest) {
// get queue data
// 分别统计 model = vectorModel和agentModel的数量
const data = await MongoDatasetTraining.aggregate([
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
const data = await MongoDatasetTraining.aggregate(
[
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
}
},
{
$group: {
_id: '$model',
count: { $sum: 1 }
}
}
},
],
{
$group: {
_id: '$model',
count: { $sum: 1 }
}
...readFromSecondary
}
]);
);
const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment