Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 5605f1a8 by Archer Committed by GitHub
Options

4.8.6 fix (#1963) 

* feat: log store

* fix: full text search match query

* perf: mongo schema import, Avoid duplicate import
parent 88d10451
Showing with 253 additions and 203 deletions
.vscode/extensions.json deleted 100644 → 0
{
"recommendations": [
"inlang.vs-code-extension"
]
}
\ No newline at end of file
.vscode/settings.json
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
"editor.formatOnSave": true, "editor.formatOnSave": true,
"editor.mouseWheelZoom": true, "editor.mouseWheelZoom": true,
"editor.defaultFormatter": "esbenp.prettier-vscode", "editor.defaultFormatter": "esbenp.prettier-vscode",
"prettier.prettierPath": "../node_modules/prettier", "prettier.prettierPath": "node_modules/prettier",
"typescript.tsdk": "node_modules/typescript/lib", "typescript.tsdk": "node_modules/typescript/lib",
"i18n-ally.localesPaths": [ "i18n-ally.localesPaths": [
"packages/web/i18n", "packages/web/i18n",
......
files/docker/docker-compose-milvus.yml
...@@ -156,6 +156,7 @@ services: ...@@ -156,6 +156,7 @@ services:
- SANDBOX_URL=http://sandbox:3000 - SANDBOX_URL=http://sandbox:3000
# 日志等级: debug, info, warn, error # 日志等级: debug, info, warn, error
- LOG_LEVEL=info - LOG_LEVEL=info
- STORE_LOG_LEVEL=warn
volumes: volumes:
- ./config.json:/app/data/config.json - ./config.json:/app/data/config.json
......
files/docker/docker-compose-pgvector.yml
...@@ -113,6 +113,7 @@ services: ...@@ -113,6 +113,7 @@ services:
- SANDBOX_URL=http://sandbox:3000 - SANDBOX_URL=http://sandbox:3000
# 日志等级: debug, info, warn, error # 日志等级: debug, info, warn, error
- LOG_LEVEL=info - LOG_LEVEL=info
- STORE_LOG_LEVEL=warn
volumes: volumes:
- ./config.json:/app/data/config.json - ./config.json:/app/data/config.json
......
files/docker/docker-compose-zilliz.yml
...@@ -94,6 +94,7 @@ services: ...@@ -94,6 +94,7 @@ services:
- SANDBOX_URL=http://sandbox:3000 - SANDBOX_URL=http://sandbox:3000
# 日志等级: debug, info, warn, error # 日志等级: debug, info, warn, error
- LOG_LEVEL=info - LOG_LEVEL=info
- STORE_LOG_LEVEL=warn
volumes: volumes:
- ./config.json:/app/data/config.json - ./config.json:/app/data/config.json
......
packages/global/tsconfig.json
{ {
"extends":"../../tsconfig.json", "extends": "../../tsconfig.json",
"compilerOptions": { "compilerOptions": {
"baseUrl": "." "baseUrl": "."
}, },
......
packages/service/common/buffer/rawText/schema.ts
import { connectionMongo, type Model } from '../../mongo'; import { connectionMongo, getMongoModel, type Model } from '../../mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { RawTextBufferSchemaType } from './type'; import { RawTextBufferSchemaType } from './type';
...@@ -28,6 +28,7 @@ try { ...@@ -28,6 +28,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoRawTextBuffer: Model<RawTextBufferSchemaType> = export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
models[collectionName] || model(collectionName, RawTextBufferSchema); collectionName,
MongoRawTextBuffer.syncIndexes(); RawTextBufferSchema
);
packages/service/common/buffer/tts/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { TTSBufferSchemaType } from './type.d'; import { TTSBufferSchemaType } from './type.d';
...@@ -31,6 +31,4 @@ try { ...@@ -31,6 +31,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoTTSBuffer: Model<TTSBufferSchemaType> = export const MongoTTSBuffer = getMongoModel<TTSBufferSchemaType>(collectionName, TTSBufferSchema);
models[collectionName] || model(collectionName, TTSBufferSchema);
MongoTTSBuffer.syncIndexes();
packages/service/common/file/gridfs/schema.ts
import { connectionMongo, type Model } from '../../mongo'; import { connectionMongo, getMongoModel, type Model } from '../../mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
const FileSchema = new Schema({}); const FileSchema = new Schema({});
...@@ -10,6 +10,4 @@ try { ...@@ -10,6 +10,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema); export const MongoFileSchema = getMongoModel('dataset.files', FileSchema);
MongoFileSchema.syncIndexes();
packages/service/common/file/image/schema.ts
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { connectionMongo, type Model } from '../../mongo'; import { connectionMongo, getMongoModel, type Model } from '../../mongo';
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d'; import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants'; import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
...@@ -41,7 +41,4 @@ try { ...@@ -41,7 +41,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoImage: Model<MongoImageSchemaType> = export const MongoImage = getMongoModel<MongoImageSchemaType>('image', ImageSchema);
models['image'] || model('image', ImageSchema);
MongoImage.syncIndexes();
packages/service/common/mongo/index.ts
import mongoose from 'mongoose'; import { addLog } from '../../common/system/log';
import mongoose, { Model } from 'mongoose';
export default mongoose;
export * from 'mongoose';
export const connectionMongo = (() => { export const connectionMongo = (() => {
if (!global.mongodb) { if (!global.mongodb) {
...@@ -11,4 +9,68 @@ export const connectionMongo = (() => { ...@@ -11,4 +9,68 @@ export const connectionMongo = (() => {
return global.mongodb; return global.mongodb;
})(); })();
export const ReadPreference = mongoose.mongo.ReadPreference; export default mongoose;
export * from 'mongoose';
const addCommonMiddleware = (schema: mongoose.Schema) => {
const operations = [
/^find/,
'save',
'create',
/^update/,
/^delete/,
'aggregate',
'count',
'countDocuments',
'estimatedDocumentCount',
'distinct',
'insertMany'
];
operations.forEach((op: any) => {
schema.pre(op, function (this: any, next) {
this._startTime = Date.now();
this._query = this.getQuery ? this.getQuery() : null;
next();
});
schema.post(op, function (this: any, result: any, next) {
if (this._startTime) {
const duration = Date.now() - this._startTime;
const warnLogData = {
query: this._query,
op,
duration
};
if (duration > 1000) {
addLog.warn(`Slow operation ${duration}ms`, warnLogData);
} else if (duration > 300) {
addLog.error(`Slow operation ${duration}ms`, warnLogData);
}
}
next();
});
});
return schema;
};
export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
if (connectionMongo.models[name]) return connectionMongo.models[name] as Model<T>;
console.log('Load model======', name);
addCommonMiddleware(schema);
const model = connectionMongo.model<T>(name, schema);
try {
model.syncIndexes();
} catch (error) {
addLog.error('Create index error', error);
}
return model;
};
export const ReadPreference = connectionMongo.mongo.ReadPreference;
packages/service/common/system/config/schema.ts
import { SystemConfigsType } from '@fastgpt/global/common/system/config/type'; import { SystemConfigsType } from '@fastgpt/global/common/system/config/type';
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
import { SystemConfigsTypeMap } from '@fastgpt/global/common/system/config/constants'; import { SystemConfigsTypeMap } from '@fastgpt/global/common/system/config/constants';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
...@@ -27,6 +27,7 @@ try { ...@@ -27,6 +27,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoSystemConfigs: Model<SystemConfigsType> = export const MongoSystemConfigs = getMongoModel<SystemConfigsType>(
models[collectionName] || model(collectionName, systemConfigSchema); collectionName,
MongoSystemConfigs.syncIndexes(); systemConfigSchema
);
packages/service/common/system/log.ts
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import chalk from 'chalk'; import chalk from 'chalk';
import { isProduction } from './constants'; import { LogLevelEnum } from './log/constant';
// import { MongoLog } from './log/schema';
import connectionMongo from '../mongo/index';
enum LogLevelEnum {
debug = 0,
info = 1,
warn = 2,
error = 3
}
const logMap = { const logMap = {
[LogLevelEnum.debug]: { [LogLevelEnum.debug]: {
levelLog: chalk.green('[Debug]') levelLog: chalk.green('[Debug]')
...@@ -23,23 +19,26 @@ const logMap = { ...@@ -23,23 +19,26 @@ const logMap = {
} }
}; };
const envLogLevelMap: Record<string, number> = { const envLogLevelMap: Record<string, number> = {
debug: 0, debug: LogLevelEnum.debug,
info: 1, info: LogLevelEnum.info,
warn: 2, warn: LogLevelEnum.warn,
error: 3 error: LogLevelEnum.error
}; };
const logLevel = (() => { const { LOG_LEVEL, STORE_LOG_LEVEL } = (() => {
if (!isProduction) return LogLevelEnum.debug; const LOG_LEVEL = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase();
const envLogLevel = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase(); const STORE_LOG_LEVEL = (process.env.STORE_LOG_LEVEL || '').toLocaleLowerCase();
if (!envLogLevel || envLogLevelMap[envLogLevel] === undefined) return LogLevelEnum.info;
return envLogLevelMap[envLogLevel]; return {
LOG_LEVEL: envLogLevelMap[LOG_LEVEL] || LogLevelEnum.info,
STORE_LOG_LEVEL: envLogLevelMap[STORE_LOG_LEVEL] ?? 99
};
})(); })();
/* add logger */ /* add logger */
export const addLog = { export const addLog = {
log(level: LogLevelEnum, msg: string, obj: Record<string, any> = {}) { log(level: LogLevelEnum, msg: string, obj: Record<string, any> = {}) {
if (level < logLevel) return; if (level < LOG_LEVEL) return;
const stringifyObj = JSON.stringify(obj); const stringifyObj = JSON.stringify(obj);
const isEmpty = Object.keys(obj).length === 0; const isEmpty = Object.keys(obj).length === 0;
...@@ -52,35 +51,15 @@ export const addLog = { ...@@ -52,35 +51,15 @@ export const addLog = {
level === LogLevelEnum.error && console.error(obj); level === LogLevelEnum.error && console.error(obj);
const lokiUrl = process.env.LOKI_LOG_URL as string; // store
if (!lokiUrl) return; // if (level >= STORE_LOG_LEVEL && connectionMongo.connection.readyState === 1) {
// // store log
try { // MongoLog.create({
fetch(lokiUrl, { // text: msg,
method: 'POST', // level,
headers: { // metadata: obj
'Content-type': 'application/json' // });
}, // }
body: JSON.stringify({
streams: [
{
stream: {
level
},
values: [
[
`${Date.now() * 1000000}`,
JSON.stringify({
message: msg,
...obj
})
]
]
}
]
})
});
} catch (error) {}
}, },
debug(msg: string, obj?: Record<string, any>) { debug(msg: string, obj?: Record<string, any>) {
this.log(LogLevelEnum.debug, msg, obj); this.log(LogLevelEnum.debug, msg, obj);
......
packages/service/common/system/log/constant.ts 0 → 100644
export enum LogLevelEnum {
debug = 0,
info = 1,
warn = 2,
error = 3
}
export enum LogSignEnum {
slowOperation = 'slowOperation'
}
packages/service/common/system/log/schema.ts 0 → 100644
import { getMongoModel, Schema } from '../../../common/mongo';
import { SystemLogType } from './type';
import { LogLevelEnum } from './constant';
export const LogCollectionName = 'system_logs';
const SystemLogSchema = new Schema({
text: {
type: String,
required: true
},
level: {
type: String,
required: true,
enum: Object.values(LogLevelEnum)
},
time: {
type: Date,
default: () => new Date()
},
metadata: Object
});
SystemLogSchema.index({ time: 1 }, { expires: '15d' });
SystemLogSchema.index({ level: 1 });
export const MongoLog = getMongoModel<SystemLogType>(LogCollectionName, SystemLogSchema);
packages/service/common/system/log/type.d.ts 0 → 100644
import { LogLevelEnum, LogSignEnum } from './constant';
export type SystemLogType = {
_id: string;
text: string;
level: LogLevelEnum;
time: Date;
metadata?: Record<string, any>;
};
packages/service/common/system/timerLock/schema.ts
import { connectionMongo, type Model } from '../../mongo'; import { connectionMongo, getMongoModel, type Model } from '../../mongo';
import { timerIdMap } from './constants'; import { timerIdMap } from './constants';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { TimerLockSchemaType } from './type.d'; import { TimerLockSchemaType } from './type.d';
...@@ -24,6 +24,4 @@ try { ...@@ -24,6 +24,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoTimerLock: Model<TimerLockSchemaType> = export const MongoTimerLock = getMongoModel<TimerLockSchemaType>(collectionName, TimerLockSchema);
models[collectionName] || model(collectionName, TimerLockSchema);
MongoTimerLock.syncIndexes();
packages/service/core/app/schema.ts
import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { connectionMongo, type Model } from '../../common/mongo'; import { Schema, getMongoModel } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d'; import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
import { import {
TeamCollectionName, TeamCollectionName,
...@@ -21,6 +20,7 @@ export const chatConfigType = { ...@@ -21,6 +20,7 @@ export const chatConfigType = {
chatInputGuide: Object chatInputGuide: Object
}; };
// schema
const AppSchema = new Schema({ const AppSchema = new Schema({
parentId: { parentId: {
type: Schema.Types.ObjectId, type: Schema.Types.ObjectId,
...@@ -112,15 +112,8 @@ const AppSchema = new Schema({ ...@@ -112,15 +112,8 @@ const AppSchema = new Schema({
...getPermissionSchema(AppDefaultPermissionVal) ...getPermissionSchema(AppDefaultPermissionVal)
}); });
try { AppSchema.index({ updateTime: -1 });
AppSchema.index({ updateTime: -1 }); AppSchema.index({ teamId: 1, type: 1 });
AppSchema.index({ teamId: 1, type: 1 }); AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 });
} catch (error) {
console.log(error);
}
export const MongoApp: Model<AppType> = export const MongoApp = getMongoModel<AppType>(AppCollectionName, AppSchema);
models[AppCollectionName] || model(AppCollectionName, AppSchema);
MongoApp.syncIndexes();
packages/service/core/app/version/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { AppVersionSchemaType } from '@fastgpt/global/core/app/version'; import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
import { chatConfigType } from '../schema'; import { chatConfigType } from '../schema';
...@@ -34,7 +34,7 @@ try { ...@@ -34,7 +34,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoAppVersion: Model<AppVersionSchemaType> = export const MongoAppVersion = getMongoModel<AppVersionSchemaType>(
models[AppVersionCollectionName] || model(AppVersionCollectionName, AppVersionSchema); AppVersionCollectionName,
AppVersionSchema
MongoAppVersion.syncIndexes(); );
packages/service/core/chat/chatItemSchema.ts
import { connectionMongo, type Model } from '../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type'; import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type';
import { ChatRoleMap } from '@fastgpt/global/core/chat/constants'; import { ChatRoleMap } from '@fastgpt/global/core/chat/constants';
...@@ -9,7 +9,6 @@ import { ...@@ -9,7 +9,6 @@ import {
} from '@fastgpt/global/support/user/team/constant'; } from '@fastgpt/global/support/user/team/constant';
import { AppCollectionName } from '../app/schema'; import { AppCollectionName } from '../app/schema';
import { userCollectionName } from '../../support/user/schema'; import { userCollectionName } from '../../support/user/schema';
import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
export const ChatItemCollectionName = 'chatitems'; export const ChatItemCollectionName = 'chatitems';
...@@ -99,7 +98,4 @@ try { ...@@ -99,7 +98,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoChatItem: Model<ChatItemType> = export const MongoChatItem = getMongoModel<ChatItemType>(ChatItemCollectionName, ChatItemSchema);
models[ChatItemCollectionName] || model(ChatItemCollectionName, ChatItemSchema);
MongoChatItem.syncIndexes();
packages/service/core/chat/chatSchema.ts
import { connectionMongo, type Model } from '../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d'; import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
import { ChatSourceMap } from '@fastgpt/global/core/chat/constants'; import { ChatSourceMap } from '@fastgpt/global/core/chat/constants';
...@@ -98,6 +98,4 @@ try { ...@@ -98,6 +98,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoChat: Model<ChatType> = export const MongoChat = getMongoModel<ChatType>(chatCollectionName, ChatSchema);
models[chatCollectionName] || model(chatCollectionName, ChatSchema);
MongoChat.syncIndexes();
packages/service/core/chat/inputGuide/schema.ts
import { AppCollectionName } from '../../app/schema'; import { AppCollectionName } from '../../app/schema';
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d'; import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d';
...@@ -23,7 +23,7 @@ try { ...@@ -23,7 +23,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoChatInputGuide: Model<ChatInputGuideSchemaType> = export const MongoChatInputGuide = getMongoModel<ChatInputGuideSchemaType>(
models[ChatInputGuideCollectionName] || model(ChatInputGuideCollectionName, ChatInputGuideSchema); ChatInputGuideCollectionName,
ChatInputGuideSchema
MongoChatInputGuide.syncIndexes(); );
packages/service/core/dataset/collection/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants'; import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants';
...@@ -94,9 +94,6 @@ const DatasetCollectionSchema = new Schema({ ...@@ -94,9 +94,6 @@ const DatasetCollectionSchema = new Schema({
} }
}); });
export const MongoDatasetCollection: Model<DatasetCollectionSchemaType> =
models[DatasetColCollectionName] || model(DatasetColCollectionName, DatasetCollectionSchema);
try { try {
// auth file // auth file
DatasetCollectionSchema.index({ teamId: 1, fileId: 1 }); DatasetCollectionSchema.index({ teamId: 1, fileId: 1 });
...@@ -111,8 +108,11 @@ try { ...@@ -111,8 +108,11 @@ try {
// get forbid // get forbid
// DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 }); // DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 });
MongoDatasetCollection.syncIndexes({ background: true });
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} }
export const MongoDatasetCollection = getMongoModel<DatasetCollectionSchemaType>(
DatasetColCollectionName,
DatasetCollectionSchema
);
packages/service/core/dataset/data/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { import {
...@@ -77,27 +77,23 @@ const DatasetDataSchema = new Schema({ ...@@ -77,27 +77,23 @@ const DatasetDataSchema = new Schema({
rebuilding: Boolean rebuilding: Boolean
}); });
export const MongoDatasetData: Model<DatasetDataSchemaType> = // list collection and count data; list data; delete collection(relate data)
models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema); DatasetDataSchema.index({
teamId: 1,
try { datasetId: 1,
// list collection and count data; list data; delete collection(relate data) collectionId: 1,
DatasetDataSchema.index({ chunkIndex: 1,
teamId: 1, updateTime: -1
datasetId: 1, });
collectionId: 1, // full text index
chunkIndex: 1, DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
updateTime: -1 // Recall vectors after data matching
}); DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
// full text index DatasetDataSchema.index({ updateTime: 1 });
DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); // rebuild data
// Recall vectors after data matching DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
DatasetDataSchema.index({ updateTime: 1 });
// rebuild data
DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
MongoDatasetData.syncIndexes({ background: true }); export const MongoDatasetData = getMongoModel<DatasetDataSchemaType>(
} catch (error) { DatasetDataCollectionName,
console.log(error); DatasetDataSchema
} );
packages/service/core/dataset/schema.ts
import { connectionMongo, type Model } from '../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { import {
...@@ -11,7 +11,6 @@ import { ...@@ -11,7 +11,6 @@ import {
TeamCollectionName, TeamCollectionName,
TeamMemberCollectionName TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant'; } from '@fastgpt/global/support/user/team/constant';
import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant'; import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
export const DatasetCollectionName = 'datasets'; export const DatasetCollectionName = 'datasets';
...@@ -99,6 +98,4 @@ try { ...@@ -99,6 +98,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoDataset: Model<DatasetSchemaType> = export const MongoDataset = getMongoModel<DatasetSchemaType>(DatasetCollectionName, DatasetSchema);
models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
MongoDataset.syncIndexes();
packages/service/core/dataset/search/controller.ts
...@@ -212,7 +212,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { ...@@ -212,7 +212,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
{ {
$match: { $match: {
$expr: { $eq: ['$_id', '$$collectionId'] }, $expr: { $eq: ['$_id', '$$collectionId'] },
forbid: { $eq: false } // 直接在lookup阶段过滤 forbid: { $eq: true } // 匹配被禁用的数据
} }
}, },
{ {
...@@ -226,7 +226,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { ...@@ -226,7 +226,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
}, },
{ {
$match: { $match: {
collection: { $ne: [] } collection: { $eq: [] } // 没有 forbid=true 的数据
} }
}, },
{ {
......
packages/service/core/dataset/training/schema.ts
/* 模型的知识库 */ /* 模型的知识库 */
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type'; import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
...@@ -103,7 +103,7 @@ try { ...@@ -103,7 +103,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> = export const MongoDatasetTraining = getMongoModel<DatasetTrainingSchemaType>(
models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema); DatasetTrainingCollectionName,
TrainingDataSchema
MongoDatasetTraining.syncIndexes(); );
packages/service/core/workflow/dispatch/agent/extract.ts
...@@ -198,7 +198,7 @@ ${description ? `- ${description}` : ''} ...@@ -198,7 +198,7 @@ ${description ? `- ${description}` : ''}
required: [] required: []
} }
}; };
console.log(properties);
return { return {
filterMessages, filterMessages,
agentFunction agentFunction
......
packages/service/support/activity/promotion/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { PromotionRecordSchema as PromotionRecordType } from '@fastgpt/global/support/activity/type.d'; import { PromotionRecordSchema as PromotionRecordType } from '@fastgpt/global/support/activity/type.d';
...@@ -29,6 +29,7 @@ const PromotionRecordSchema = new Schema({ ...@@ -29,6 +29,7 @@ const PromotionRecordSchema = new Schema({
} }
}); });
export const MongoPromotionRecord: Model<PromotionRecordType> = export const MongoPromotionRecord = getMongoModel<PromotionRecordType>(
models['promotionRecord'] || model('promotionRecord', PromotionRecordSchema); 'promotionRecord',
MongoPromotionRecord.syncIndexes(); PromotionRecordSchema
);
packages/service/support/openapi/schema.ts
import { connectionMongo, type Model } from '../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type'; import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
import { import {
...@@ -64,6 +64,4 @@ try { ...@@ -64,6 +64,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoOpenApi: Model<OpenApiSchema> = export const MongoOpenApi = getMongoModel<OpenApiSchema>('openapi', OpenApiSchema);
models['openapi'] || model('openapi', OpenApiSchema);
MongoOpenApi.syncIndexes();
packages/service/support/outLink/schema.ts
import { connectionMongo, type Model } from '../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type'; import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
import { import {
...@@ -90,7 +90,4 @@ try { ...@@ -90,7 +90,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoOutLink: Model<SchemaType> = export const MongoOutLink = getMongoModel<SchemaType>('outlinks', OutLinkSchema);
models['outlinks'] || model('outlinks', OutLinkSchema);
MongoOutLink.syncIndexes();
packages/service/support/permission/schema.ts
...@@ -2,7 +2,7 @@ import { ...@@ -2,7 +2,7 @@ import {
TeamCollectionName, TeamCollectionName,
TeamMemberCollectionName TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant'; } from '@fastgpt/global/support/user/team/constant';
import { Model, connectionMongo } from '../../common/mongo'; import { Model, connectionMongo, getMongoModel } from '../../common/mongo';
import type { ResourcePermissionType } from '@fastgpt/global/support/permission/type'; import type { ResourcePermissionType } from '@fastgpt/global/support/permission/type';
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
...@@ -54,8 +54,7 @@ try { ...@@ -54,8 +54,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoResourcePermission: Model<ResourcePermissionType> = export const MongoResourcePermission = getMongoModel<ResourcePermissionType>(
models[ResourcePermissionCollectionName] || ResourcePermissionCollectionName,
model(ResourcePermissionCollectionName, ResourcePermissionSchema); ResourcePermissionSchema
);
MongoResourcePermission.syncIndexes();
packages/service/support/user/schema.ts
import { connectionMongo, type Model } from '../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { hashStr } from '@fastgpt/global/common/string/tools'; import { hashStr } from '@fastgpt/global/common/string/tools';
import type { UserModelSchema } from '@fastgpt/global/support/user/type'; import type { UserModelSchema } from '@fastgpt/global/support/user/type';
...@@ -74,6 +74,4 @@ try { ...@@ -74,6 +74,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoUser: Model<UserModelSchema> = export const MongoUser = getMongoModel<UserModelSchema>(userCollectionName, UserSchema);
models[userCollectionName] || model(userCollectionName, UserSchema);
MongoUser.syncIndexes();
packages/service/support/user/team/teamMemberSchema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { TeamMemberSchema as TeamMemberType } from '@fastgpt/global/support/user/team/type.d'; import { TeamMemberSchema as TeamMemberType } from '@fastgpt/global/support/user/team/type.d';
import { userCollectionName } from '../../user/schema'; import { userCollectionName } from '../../user/schema';
...@@ -49,5 +49,7 @@ try { ...@@ -49,5 +49,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoTeamMember: Model<TeamMemberType> = export const MongoTeamMember = getMongoModel<TeamMemberType>(
models[TeamMemberCollectionName] || model(TeamMemberCollectionName, TeamMemberSchema); TeamMemberCollectionName,
TeamMemberSchema
);
packages/service/support/user/team/teamSchema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { TeamSchema as TeamType } from '@fastgpt/global/support/user/team/type.d'; import { TeamSchema as TeamType } from '@fastgpt/global/support/user/team/type.d';
import { userCollectionName } from '../../user/schema'; import { userCollectionName } from '../../user/schema';
...@@ -61,5 +61,4 @@ try { ...@@ -61,5 +61,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoTeam: Model<TeamType> = export const MongoTeam = getMongoModel<TeamType>(TeamCollectionName, TeamSchema);
models[TeamCollectionName] || model(TeamCollectionName, TeamSchema);
packages/service/support/user/team/teamTagsSchema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d'; import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
import { import {
...@@ -32,5 +32,7 @@ try { ...@@ -32,5 +32,7 @@ try {
console.log(error); console.log(error);
} }
export const MongoTeamTags: Model<TeamTagsSchemaType> = export const MongoTeamTags = getMongoModel<TeamTagsSchemaType>(
models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema); TeamTagsCollectionName,
TeamTagSchema
);
packages/service/support/wallet/sub/schema.ts
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
1. type=standard: There will only be 1, and each team will have one 1. type=standard: There will only be 1, and each team will have one
2. type=extraDatasetSize/extraPoints: Can buy multiple 2. type=extraDatasetSize/extraPoints: Can buy multiple
*/ */
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { import {
...@@ -93,5 +93,4 @@ try { ...@@ -93,5 +93,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoTeamSub: Model<TeamSubSchema> = export const MongoTeamSub = getMongoModel<TeamSubSchema>(subCollectionName, SubSchema);
models[subCollectionName] || model(subCollectionName, SubSchema);
packages/service/support/wallet/usage/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo'; import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo; const { Schema, model, models } = connectionMongo;
import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type'; import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type';
import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants'; import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants';
...@@ -70,6 +70,4 @@ try { ...@@ -70,6 +70,4 @@ try {
console.log(error); console.log(error);
} }
export const MongoUsage: Model<UsageSchemaType> = export const MongoUsage = getMongoModel<UsageSchemaType>(UsageCollectionName, UsageSchema);
models[UsageCollectionName] || model(UsageCollectionName, UsageSchema);
MongoUsage.syncIndexes();
projects/app/.env.template
...@@ -33,6 +33,6 @@ PRO_URL= ...@@ -33,6 +33,6 @@ PRO_URL=
# 首页路径 # 首页路径
HOME_URL=/ HOME_URL=/
# 日志等级: debug, info, warn, error # 日志等级: debug, info, warn, error
LOG_LEVEL=info LOG_LEVEL=debug
# Loki Log Path STORE_LOG_LEVEL=warn
# LOKI_LOG_URL= # Loki Log Path
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment