Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 5cb19653 by Archer Committed by GitHub
Options

4.8.7 fix (#2076)

parent 982325d0
Showing with 188 additions and 208 deletions
packages/service/common/mongo/index.ts
......@@ -47,7 +47,7 @@ const addCommonMiddleware = (schema: mongoose.Schema) => {
if (duration > 1000) {
addLog.warn(`Slow operation ${duration}ms`, warnLogData);
} else if (duration > 300) {
} else if (duration > 3000) {
addLog.error(`Slow operation ${duration}ms`, warnLogData);
}
}
......
packages/service/common/mongo/init.ts
import { exit } from 'process';
import { addLog } from '../system/log';
import { connectionMongo } from './index';
import type { Mongoose } from 'mongoose';
......@@ -8,19 +7,12 @@ const maxConnecting = Math.max(30, Number(process.env.DB_MAX_LINK || 20));
/**
* connect MongoDB and init data
*/
export async function connectMongo({
beforeHook,
afterHook
}: {
beforeHook?: () => any;
afterHook?: () => Promise<any>;
}): Promise<Mongoose> {
export async function connectMongo(): Promise<Mongoose> {
/* Connecting, connected will return */
if (connectionMongo.connection.readyState !== 0) {
return connectionMongo;
}
beforeHook && beforeHook();
console.log('mongo start connect');
try {
connectionMongo.set('strictQuery', true);
......@@ -56,15 +48,5 @@ export async function connectMongo({
addLog.error('mongo connect error', error);
}
try {
if (!global.systemInited) {
global.systemInited = true;
afterHook && (await afterHook());
}
} catch (error) {
addLog.error('Mongo connect after hook error', error);
exit(1);
}
return connectionMongo;
}
packages/service/type.d.ts
......@@ -26,5 +26,6 @@ declare global {
callbackMap: Record<string, (e: number) => void>;
}[];
var systemInited: boolean;
var systemLoadedGlobalVariables: boolean;
var systemLoadedGlobalConfig: boolean;
}
projects/app/package.json
......@@ -33,7 +33,7 @@
"echarts": "5.4.1",
"echarts-gl": "2.0.9",
"formidable": "^2.1.1",
"framer-motion": "^9.0.6",
"framer-motion": "9.1.7",
"hyperdown": "^2.4.29",
"immer": "^9.0.19",
"js-yaml": "^4.1.0",
......
projects/app/src/pages/api/common/system/getInitData.ts
import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { readFileSync, readdirSync } from 'fs';
import type { InitDateResponse } from '@/global/common/api/systemRes';
import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
import { getFastGPTConfigFromDB } from '@fastgpt/service/common/system/config/controller';
import { PluginTemplateType } from '@fastgpt/global/core/plugin/type';
import { readConfigData } from '@/service/common/system';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
import { initFastGPTConfig } from '@fastgpt/service/common/system/tools';
import json5 from 'json5';
import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type';
import { connectToDatabase } from '@/service/mongo';
import { jsonRes } from '@fastgpt/service/common/response';
async function handler(req: NextApiRequest, res: NextApiResponse) {
await getInitConfig();
await connectToDatabase();
jsonRes<InitDateResponse>(res, {
data: {
......@@ -37,148 +26,3 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
}
export default handler;
const defaultFeConfigs: FastGPTFeConfigsType = {
show_emptyChat: true,
show_git: true,
docUrl: 'https://doc.fastgpt.in',
openAPIDocUrl: 'https://doc.fastgpt.in/docs/development/openapi',
systemTitle: 'FastGPT',
concatMd:
'项目开源地址: [FastGPT GitHub](https://github.com/labring/FastGPT)\n交流群: ![](https://oss.laf.run/htr4n1-images/fastgpt-qr-code.jpg)',
limit: {
exportDatasetLimitMinutes: 0,
websiteSyncLimitMinuted: 0
},
scripts: [],
favicon: '/favicon.ico',
uploadFileMaxSize: 500
};
export async function getInitConfig() {
// First request
if (!global.systemInited) {
await connectToDatabase();
}
return Promise.all([
initSystemConfig(),
getSystemVersion(),
getSystemPlugin(),
// abandon
getSystemPluginV1()
]);
}
export async function initSystemConfig() {
// load config
const [dbConfig, fileConfig] = await Promise.all([
getFastGPTConfigFromDB(),
readConfigData('config.json')
]);
const fileRes = json5.parse(fileConfig) as FastGPTConfigFileType;
// get config from database
const config: FastGPTConfigFileType = {
feConfigs: {
...fileRes?.feConfigs,
...defaultFeConfigs,
...(dbConfig.feConfigs || {}),
isPlus: !!FastGPTProUrl
},
systemEnv: {
...fileRes.systemEnv,
...(dbConfig.systemEnv || {})
},
subPlans: dbConfig.subPlans || fileRes.subPlans,
llmModels: dbConfig.llmModels || fileRes.llmModels || [],
vectorModels: dbConfig.vectorModels || fileRes.vectorModels || [],
reRankModels: dbConfig.reRankModels || fileRes.reRankModels || [],
audioSpeechModels: dbConfig.audioSpeechModels || fileRes.audioSpeechModels || [],
whisperModel: dbConfig.whisperModel || fileRes.whisperModel
};
// set config
initFastGPTConfig(config);
console.log({
feConfigs: global.feConfigs,
systemEnv: global.systemEnv,
subPlans: global.subPlans,
llmModels: global.llmModels,
vectorModels: global.vectorModels,
reRankModels: global.reRankModels,
audioSpeechModels: global.audioSpeechModels,
whisperModel: global.whisperModel
});
}
export function getSystemVersion() {
if (global.systemVersion) return;
try {
if (process.env.NODE_ENV === 'development') {
global.systemVersion = process.env.npm_package_version || '0.0.0';
} else {
const packageJson = json5.parse(readFileSync('/app/package.json', 'utf-8'));
global.systemVersion = packageJson?.version;
}
console.log(`System Version: ${global.systemVersion}`);
} catch (error) {
console.log(error);
global.systemVersion = '0.0.0';
}
}
function getSystemPlugin() {
if (global.communityPlugins && global.communityPlugins.length > 0) return;
const basePath =
process.env.NODE_ENV === 'development' ? 'data/pluginTemplates' : '/app/data/pluginTemplates';
// read data/pluginTemplates directory, get all json file
const files = readdirSync(basePath);
// filter json file
const filterFiles = files.filter((item) => item.endsWith('.json'));
// read json file
const fileTemplates = filterFiles.map<SystemPluginTemplateItemType>((filename) => {
const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
return {
...json5.parse(content),
originCost: 0,
currentCost: 0,
id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
};
});
fileTemplates.sort((a, b) => (b.weight || 0) - (a.weight || 0));
global.communityPlugins = fileTemplates;
}
function getSystemPluginV1() {
if (global.communityPluginsV1 && global.communityPluginsV1.length > 0) return;
const basePath =
process.env.NODE_ENV === 'development'
? 'data/pluginTemplates/v1'
: '/app/data/pluginTemplates/v1';
// read data/pluginTemplates directory, get all json file
const files = readdirSync(basePath);
// filter json file
const filterFiles = files.filter((item) => item.endsWith('.json'));
// read json file
const fileTemplates: (PluginTemplateType & { weight: number })[] = filterFiles.map((filename) => {
const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
return {
...JSON.parse(content),
id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`,
source: PluginSourceEnum.community
};
});
fileTemplates.sort((a, b) => b.weight - a.weight);
global.communityPluginsV1 = fileTemplates;
}
projects/app/src/pages/login/provider.tsx
......@@ -13,12 +13,13 @@ import { getErrText } from '@fastgpt/global/common/error/utils';
import { useTranslation } from 'next-i18next';
import { useMount } from 'ahooks';
const provider = ({ code, state, error }: { code: string; state: string; error?: string }) => {
const provider = () => {
const { t } = useTranslation();
const { loginStore } = useSystemStore();
const { setLastChatId, setLastChatAppId } = useChatStore();
const { setUserInfo } = useUserStore();
const router = useRouter();
const { code, state, error } = router.query as { code: string; state: string; error?: string };
const { toast } = useToast();
const loginSuccess = useCallback(
......@@ -76,9 +77,7 @@ const provider = ({ code, state, error }: { code: string; state: string; error?:
[loginStore, loginSuccess, router, toast]
);
useMount(() => {
clearToken();
router.prefetch('/app/list');
useEffect(() => {
if (error) {
toast({
status: 'warning',
......@@ -87,7 +86,11 @@ const provider = ({ code, state, error }: { code: string; state: string; error?:
router.replace('/login');
return;
}
if (!code) return;
if (!code || !loginStore || !state) return;
clearToken();
router.prefetch('/app/list');
if (state !== loginStore?.state) {
toast({
......@@ -98,22 +101,12 @@ const provider = ({ code, state, error }: { code: string; state: string; error?:
router.replace('/login');
}, 1000);
return;
} else {
authCode(code);
}
authCode(code);
});
}, [code, error, loginStore, state]);
return <Loading />;
};
export async function getServerSideProps(content: any) {
return {
props: {
code: content?.query?.code || '',
state: content?.query?.state || '',
error: content?.query?.error || '',
...(await serviceSideProps(content))
}
};
}
export default provider;
projects/app/src/service/common/system/index.ts
import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
import { existsSync, readFileSync } from 'fs';
import { existsSync, readdirSync, readFileSync } from 'fs';
import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';
import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
import { getFastGPTConfigFromDB } from '@fastgpt/service/common/system/config/controller';
import { PluginTemplateType } from '@fastgpt/global/core/plugin/type';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
import { initFastGPTConfig } from '@fastgpt/service/common/system/tools';
import json5 from 'json5';
import { SystemPluginTemplateItemType } from '@fastgpt/global/core/workflow/type';
export const readConfigData = (name: string) => {
const isDev = process.env.NODE_ENV === 'development';
......@@ -25,6 +34,7 @@ export const readConfigData = (name: string) => {
return content;
};
/* Init global variables */
export function initGlobal() {
if (global.communityPlugins) return;
......@@ -33,3 +43,144 @@ export function initGlobal() {
global.vectorQueueLen = global.vectorQueueLen ?? 0;
initHttpAgent();
}
/* Init system data(Need to connected db). It only needs to run once */
export async function getInitConfig() {
return Promise.all([
initSystemConfig(),
getSystemVersion(),
// abandon
getSystemPlugin(),
getSystemPluginV1()
]);
}
const defaultFeConfigs: FastGPTFeConfigsType = {
show_emptyChat: true,
show_git: true,
docUrl: 'https://doc.fastgpt.in',
openAPIDocUrl: 'https://doc.fastgpt.in/docs/development/openapi',
systemTitle: 'FastGPT',
concatMd:
'项目开源地址: [FastGPT GitHub](https://github.com/labring/FastGPT)\n交流群: ![](https://oss.laf.run/htr4n1-images/fastgpt-qr-code.jpg)',
limit: {
exportDatasetLimitMinutes: 0,
websiteSyncLimitMinuted: 0
},
scripts: [],
favicon: '/favicon.ico',
uploadFileMaxSize: 500
};
export async function initSystemConfig() {
// load config
const [dbConfig, fileConfig] = await Promise.all([
getFastGPTConfigFromDB(),
readConfigData('config.json')
]);
const fileRes = json5.parse(fileConfig) as FastGPTConfigFileType;
// get config from database
const config: FastGPTConfigFileType = {
feConfigs: {
...fileRes?.feConfigs,
...defaultFeConfigs,
...(dbConfig.feConfigs || {}),
isPlus: !!FastGPTProUrl
},
systemEnv: {
...fileRes.systemEnv,
...(dbConfig.systemEnv || {})
},
subPlans: dbConfig.subPlans || fileRes.subPlans,
llmModels: dbConfig.llmModels || fileRes.llmModels || [],
vectorModels: dbConfig.vectorModels || fileRes.vectorModels || [],
reRankModels: dbConfig.reRankModels || fileRes.reRankModels || [],
audioSpeechModels: dbConfig.audioSpeechModels || fileRes.audioSpeechModels || [],
whisperModel: dbConfig.whisperModel || fileRes.whisperModel
};
// set config
initFastGPTConfig(config);
console.log({
feConfigs: global.feConfigs,
systemEnv: global.systemEnv,
subPlans: global.subPlans,
llmModels: global.llmModels,
vectorModels: global.vectorModels,
reRankModels: global.reRankModels,
audioSpeechModels: global.audioSpeechModels,
whisperModel: global.whisperModel
});
}
function getSystemVersion() {
if (global.systemVersion) return;
try {
if (process.env.NODE_ENV === 'development') {
global.systemVersion = process.env.npm_package_version || '0.0.0';
} else {
const packageJson = json5.parse(readFileSync('/app/package.json', 'utf-8'));
global.systemVersion = packageJson?.version;
}
console.log(`System Version: ${global.systemVersion}`);
} catch (error) {
console.log(error);
global.systemVersion = '0.0.0';
}
}
function getSystemPlugin() {
if (global.communityPlugins && global.communityPlugins.length > 0) return;
const basePath =
process.env.NODE_ENV === 'development' ? 'data/pluginTemplates' : '/app/data/pluginTemplates';
// read data/pluginTemplates directory, get all json file
const files = readdirSync(basePath);
// filter json file
const filterFiles = files.filter((item) => item.endsWith('.json'));
// read json file
const fileTemplates = filterFiles.map<SystemPluginTemplateItemType>((filename) => {
const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
return {
...json5.parse(content),
originCost: 0,
currentCost: 0,
id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`
};
});
fileTemplates.sort((a, b) => (b.weight || 0) - (a.weight || 0));
global.communityPlugins = fileTemplates;
}
function getSystemPluginV1() {
if (global.communityPluginsV1 && global.communityPluginsV1.length > 0) return;
const basePath =
process.env.NODE_ENV === 'development'
? 'data/pluginTemplates/v1'
: '/app/data/pluginTemplates/v1';
// read data/pluginTemplates directory, get all json file
const files = readdirSync(basePath);
// filter json file
const filterFiles = files.filter((item) => item.endsWith('.json'));
// read json file
const fileTemplates: (PluginTemplateType & { weight: number })[] = filterFiles.map((filename) => {
const content = readFileSync(`${basePath}/${filename}`, 'utf-8');
return {
...JSON.parse(content),
id: `${PluginSourceEnum.community}-${filename.replace('.json', '')}`,
source: PluginSourceEnum.community
};
});
fileTemplates.sort((a, b) => b.weight - a.weight);
global.communityPluginsV1 = fileTemplates;
}
projects/app/src/service/common/system/volumnMongoWatch.ts
import { initSystemConfig } from '@/pages/api/common/system/getInitData';
import { initSystemConfig } from '.';
import { createDatasetTrainingMongoWatch } from '@/service/core/dataset/training/utils';
import { MongoSystemConfigs } from '@fastgpt/service/common/system/config/schema';
......
projects/app/src/service/mongo.ts
......@@ -5,23 +5,29 @@ import { hashStr } from '@fastgpt/global/common/string/tools';
import { createDefaultTeam } from '@fastgpt/service/support/user/team/controller';
import { exit } from 'process';
import { initVectorStore } from '@fastgpt/service/common/vectorStore/controller';
import { getInitConfig } from '@/pages/api/common/system/getInitData';
import { startCron } from './common/system/cron';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { initGlobal } from './common/system';
import { initGlobal, getInitConfig } from './common/system';
import { startMongoWatch } from './common/system/volumnMongoWatch';
import { startTrainingQueue } from './core/dataset/training/utils';
import { systemStartCb } from '@fastgpt/service/common/system/tools';
import { addLog } from '@fastgpt/service/common/system/log';
/**
* This function is equivalent to the entry to the service
* connect MongoDB and init data
*/
export function connectToDatabase() {
return connectMongo({
beforeHook: () => {
initGlobal();
},
afterHook: async () => {
if (!global.systemLoadedGlobalVariables) {
global.systemLoadedGlobalVariables = true;
initGlobal();
}
return connectMongo().then(async () => {
if (global.systemLoadedGlobalConfig) return;
global.systemLoadedGlobalConfig = true;
try {
systemStartCb();
//init system config；init vector database；init root user
......@@ -33,6 +39,9 @@ export function connectToDatabase() {
// start queue
startTrainingQueue(true);
} catch (error) {
addLog.error('init error', error);
exit(1);
}
});
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment