Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit b9161838 by Archer Committed by GitHub
Options

4.6.3-alpha1 (#529)

parent 007fce2d
Showing with 326 additions and 133 deletions
packages/global/common/string/time.ts 0 → 100644
import dayjs from 'dayjs';
export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm');
packages/global/core/app/utils.ts
......@@ -77,7 +77,7 @@ export const appModules2Form = ({
);
defaultAppForm.aiSettings.quotePrompt = findInputValueByKey(
module.inputs,
ModuleInputKeyEnum.aiChatQuoteTemplate
ModuleInputKeyEnum.aiChatQuotePrompt
);
} else if (module.flowType === FlowNodeTypeEnum.datasetSearchNode) {
defaultAppForm.dataset.datasets = findInputValueByKey(
......
packages/global/core/chat/type.d.ts
......@@ -4,6 +4,7 @@ import { ChatRoleEnum, ChatSourceEnum } from './constants';
import { FlowNodeTypeEnum } from '../module/node/constant';
import { ModuleOutputKeyEnum } from '../module/constants';
import { AppSchema } from '../app/type';
import { DatasetSearchModeEnum } from '../dataset/constant';
export type ChatSchema = {
_id: string;
......@@ -94,6 +95,7 @@ export type moduleDispatchResType = {
// dataset search
similarity?: number;
limit?: number;
searchMode?: `${DatasetSearchModeEnum}`;
// cq
cqList?: ClassifyQuestionAgentItemType[];
......
packages/global/core/dataset/constant.ts
export const PgDatasetTableName = 'modeldata';
/* ------------ dataset -------------- */
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset'
......@@ -14,28 +15,45 @@ export const DatasetTypeMap = {
}
};
/* ------------ collection -------------- */
export enum DatasetCollectionTypeEnum {
file = 'file',
folder = 'folder',
file = 'file',
link = 'link',
virtual = 'virtual'
}
export const DatasetCollectionTypeMap = {
[DatasetCollectionTypeEnum.file]: {
name: 'dataset.file'
},
[DatasetCollectionTypeEnum.folder]: {
name: 'dataset.folder'
name: 'core.dataset.folder'
},
[DatasetCollectionTypeEnum.file]: {
name: 'core.dataset.file'
},
[DatasetCollectionTypeEnum.link]: {
name: 'dataset.link'
name: 'core.dataset.link'
},
[DatasetCollectionTypeEnum.virtual]: {
name: 'dataset.Virtual File'
name: 'core.dataset.Virtual File'
}
};
export enum DatasetCollectionTrainingModeEnum {
manual = 'manual',
chunk = 'chunk',
qa = 'qa'
}
export const DatasetCollectionTrainingTypeMap = {
[DatasetCollectionTrainingModeEnum.manual]: {
label: 'core.dataset.collection.training.type manual'
},
[DatasetCollectionTrainingModeEnum.chunk]: {
label: 'core.dataset.collection.training.type chunk'
},
[DatasetCollectionTrainingModeEnum.qa]: {
label: 'core.dataset.collection.training.type qa'
}
};
/* ------------ data -------------- */
export enum DatasetDataIndexTypeEnum {
chunk = 'chunk',
qa = 'qa',
......@@ -61,31 +79,22 @@ export const DatasetDataIndexTypeMap = {
}
};
/* ------------ training -------------- */
export enum TrainingModeEnum {
'chunk' = 'chunk',
'qa' = 'qa'
// 'hypothetical' = 'hypothetical',
// 'summary' = 'summary',
// 'multipleIndex' = 'multipleIndex'
chunk = 'chunk',
qa = 'qa'
}
export const TrainingTypeMap = {
[TrainingModeEnum.chunk]: {
name: 'chunk'
label: 'core.dataset.training.type chunk'
},
[TrainingModeEnum.qa]: {
name: 'qa'
label: 'core.dataset.training.type qa'
}
// [TrainingModeEnum.hypothetical]: {
// name: 'hypothetical'
// },
// [TrainingModeEnum.summary]: {
// name: 'summary'
// },
// [TrainingModeEnum.multipleIndex]: {
// name: 'multipleIndex'
// }
};
/* ------------ search -------------- */
export enum DatasetSearchModeEnum {
embedding = 'embedding',
embeddingReRank = 'embeddingReRank',
......
packages/global/core/dataset/controller.d.ts
......@@ -5,6 +5,7 @@ export type CreateDatasetDataProps = {
tmbId: string;
datasetId: string;
collectionId: string;
chunkIndex?: number;
q: string;
a?: string;
indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
......
packages/global/core/dataset/type.d.ts
......@@ -27,19 +27,18 @@ export type DatasetSchemaType = {
export type DatasetCollectionSchemaType = {
_id: string;
userId: string;
teamId: string;
tmbId: string;
datasetId: string;
parentId?: string;
name: string;
type: `${DatasetCollectionTypeEnum}`;
createTime: Date;
updateTime: Date;
metadata: {
fileId?: string;
rawLink?: string;
pgCollectionId?: string;
};
trainingType: `${TrainingModeEnum}`;
chunkSize: number;
fileId?: string;
rawLink?: string;
};
export type DatasetDataIndexItemType = {
......@@ -57,6 +56,8 @@ export type DatasetDataSchemaType = {
collectionId: string;
datasetId: string;
collectionId: string;
chunkIndex: number;
updateTime: Date;
q: string; // large chunks or question
a: string; // answer or custom content
fullTextToken: string;
......@@ -78,6 +79,7 @@ export type DatasetTrainingSchemaType = {
prompt: string;
q: string;
a: string;
chunkIndex: number;
indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
};
......@@ -101,6 +103,7 @@ export type DatasetCollectionItemType = CollectionWithDatasetType & {
canWrite: boolean;
sourceName: string;
sourceId?: string;
file?: DatasetFileSchema;
};
/* ================= data ===================== */
......
packages/service/core/dataset/collection/schema.ts
import { connectionMongo, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant';
import {
DatasetCollectionTrainingTypeMap,
DatasetCollectionTypeMap
} from '@fastgpt/global/core/dataset/constant';
import { DatasetCollectionName } from '../schema';
import {
TeamCollectionName,
......@@ -45,24 +48,32 @@ const DatasetCollectionSchema = new Schema({
enum: Object.keys(DatasetCollectionTypeMap),
required: true
},
createTime: {
type: Date,
default: () => new Date()
},
updateTime: {
type: Date,
default: () => new Date()
},
trainingType: {
type: String,
enum: Object.keys(DatasetCollectionTrainingTypeMap),
required: true
},
chunkSize: {
type: Number,
required: true
},
fileId: {
type: Schema.Types.ObjectId,
ref: 'dataset.files'
},
rawLink: {
type: String
},
metadata: {
type: {
fileId: {
type: Schema.Types.ObjectId,
ref: 'dataset.files'
},
rawLink: {
type: String
},
// 451 初始化
pgCollectionId: {
type: String
}
},
type: Object,
default: {}
}
});
......
packages/service/core/dataset/data/schema.ts
......@@ -70,6 +70,15 @@ const DatasetDataSchema = new Schema({
}
],
default: []
},
// metadata
updateTime: {
type: Date,
default: () => new Date()
},
chunkIndex: {
type: Number,
default: 0
}
});
......
packages/service/core/dataset/training/schema.ts
......@@ -75,6 +75,10 @@ const TrainingDataSchema = new Schema({
type: String,
default: ''
},
chunkIndex: {
type: Number,
default: 0
},
indexes: {
type: [
{
......
projects/app/package.json
{
"name": "app",
"version": "4.6.2",
"version": "4.6.3",
"private": false,
"scripts": {
"dev": "next dev",
......
projects/app/public/locales/en/common.json
......@@ -266,15 +266,39 @@
"Search Top K": "Top K",
"Set Empty Result Tip": ",Response empty text",
"Similarity": "Similarity",
"Sync Time": "Update Time",
"Virtual File": "Virtual File",
"collection": {
"metadata": {
"Chunk Size": "Chunk Size",
"Createtime": "Create Time",
"Read Metadata": "Read Metadata",
"Training Type": "Training Type",
"Updatetime": "Update Time",
"metadata": "Metadata",
"read source": "Read Source",
"source": "Source",
"source name": "Source Name",
"source size": "Source Size"
},
"training": {
"type chunk": "Chunk",
"type manual": "Manual",
"type qa": "QA"
}
},
"data": {
"Edit": "Edit Data",
"data is deleted": "Data is deleted",
"id": "Data ID"
},
"file": "File",
"folder": "Folder",
"import": {
"Ideal chunk length": "Ideal chunk length",
"Ideal chunk length Tips": "Segment by end symbol. We recommend that your document should be properly punctuated to ensure that each complete sentence length does not exceed this value \n Chinese document recommended 400~1000\n English document recommended 600~1200"
},
"link": "Link",
"search": {
"Empty result response": "Empty Response",
"Empty result response Tips": "If you fill in the content, if no suitable content is found, you will directly reply to the content.",
......@@ -289,7 +313,8 @@
"embedding desc": "Direct vector topk correlation query ",
"embeddingReRank": "Enhanced semantic retrieval ",
"embeddingReRank desc": "Sort using Rerank after overperforming vector topk queries "
}
},
"search mode": "Search Mode"
},
"test": {
"Test": "Start",
......@@ -300,6 +325,10 @@
"test history": "Test History",
"test result placeholder": "The test results will be presented here",
"test result tip": "The contents of the knowledge base are sorted according to their similarity to the test text, and you can adjust the corresponding text according to the test results. Note: The data in the test record may have been modified, clicking on a test data will show the latest data."
},
"training": {
"type chunk": "Chunk",
"type qa": "QA"
}
},
"module": {
......@@ -693,9 +722,9 @@
"wallet": {
"bill": {
"Audio Speech": "Audio Speech",
"ReRank": "ReRank",
"Whisper": "Whisper",
"bill username": "User",
"ReRank": "ReRank"
"bill username": "User"
}
}
}
projects/app/public/locales/zh/common.json
......@@ -266,15 +266,39 @@
"Search Top K": "单次搜索数量",
"Set Empty Result Tip": ",未搜索到内容时回复指定内容",
"Similarity": "相似度",
"Sync Time": "最后更新时间",
"Virtual File": "虚拟文件",
"collection": {
"metadata": {
"Chunk Size": "分割大小",
"Createtime": "创建时间",
"Read Metadata": "查看元数据",
"Training Type": "训练模式",
"Updatetime": "更新时间",
"metadata": "元数据",
"read source": "查看原始内容",
"source": "数据来源",
"source name": "来源名",
"source size": "来源大小"
},
"training": {
"type manual": "手动",
"type chunk": "直接分段",
"type qa": "问答拆分"
}
},
"data": {
"Edit": "编辑数据",
"data is deleted": "该数据已被删除",
"id": "数据ID"
},
"file": "文件",
"folder": "目录",
"import": {
"Ideal chunk length": "理想分块长度",
"Ideal chunk length Tips": "按结束符号进行分段。我们建议您的文档应合理的使用标点符号，以确保每个完整的句子长度不要超过该值\n中文文档建议400~1000\n英文文档建议600~1200"
},
"link": "链接",
"search": {
"Empty result response": "空搜索回复",
"Empty result response Tips": "若填写该内容，没有搜索到合适内容时，将直接回复填写的内容。",
......@@ -289,7 +313,8 @@
"embedding desc": "直接进行向量 topk 相关性查询",
"embeddingReRank": "增强语义检索",
"embeddingReRank desc": "超额进行向量 topk 查询后再使用 Rerank 进行排序"
}
},
"search mode": "检索模式"
},
"test": {
"Test": "测试",
......@@ -300,6 +325,10 @@
"test history": "测试历史",
"test result placeholder": "测试结果将在这里展示",
"test result tip": "根据知识库内容与测试文本的相似度进行排序，你可以根据测试结果调整对应的文本。\n注意：测试记录中的数据可能已经被修改过，点击某条测试数据后将展示最新的数据。"
},
"training": {
"type chunk": "直接分段",
"type qa": "问答拆分"
}
},
"module": {
......@@ -693,9 +722,9 @@
"wallet": {
"bill": {
"Audio Speech": "语音播报",
"ReRank": "结果重排",
"Whisper": "语音输入",
"bill username": "用户",
"ReRank": "结果重排"
"bill username": "用户"
}
}
}
projects/app/src/components/ChatBox/WholeResponseModal.tsx
......@@ -10,6 +10,7 @@ import MyTooltip from '../MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import Markdown from '../Markdown';
import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
function Row({ label, value }: { label: string; value?: string | number }) {
const theme = useTheme();
......@@ -127,6 +128,13 @@ const WholeResponseModal = ({
)}
{/* dataset search */}
{activeModule?.searchMode && (
<Row
label={t('core.dataset.search.search mode')}
// @ts-ignore
value={t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
/>
)}
<Row label={t('chat.response.module similarity')} value={activeModule?.similarity} />
<Row label={t('chat.response.module limit')} value={activeModule?.limit} />
......
projects/app/src/components/common/MyRadio/index.tsx
......@@ -36,16 +36,17 @@ const MyRadio = ({
border={theme.borders.sm}
borderWidth={'1.5px'}
borderRadius={'md'}
bg={'myWhite.300'}
position={'relative'}
{...(value === item.value
? {
borderColor: 'myBlue.700'
borderColor: 'myBlue.500',
bg: 'myBlue.100'
}
: {
bg: 'myWhite.300',
_hover: {
bg: 'myBlue.100',
borderColor: 'myBlue.600'
bg: '#f5f8ff',
borderColor: '#b2ccff'
}
})}
_after={{
......
projects/app/src/components/core/module/Flow/ModuleTemplateList.tsx
import React, { useCallback, useMemo } from 'react';
import {
Box,
Flex,
Accordion,
AccordionItem,
AccordionButton,
AccordionPanel,
AccordionIcon,
useTheme
} from '@chakra-ui/react';
import { Box, Flex } from '@chakra-ui/react';
import type {
FlowModuleTemplateType,
moduleTemplateListType
......
projects/app/src/constants/dataset.ts
......@@ -24,7 +24,6 @@ export const defaultDatasetDetail: DatasetItemType = {
export const defaultCollectionDetail: DatasetCollectionItemType = {
_id: '',
userId: '',
teamId: '',
tmbId: '',
datasetId: {
......@@ -46,8 +45,10 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
name: '',
type: 'file',
updateTime: new Date(),
metadata: {},
canWrite: false,
sourceName: '',
sourceId: ''
sourceId: '',
createTime: new Date(),
trainingType: 'chunk',
chunkSize: 0
};
projects/app/src/global/core/api/datasetReq.d.ts
import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
DatasetCollectionTrainingModeEnum,
DatasetCollectionTypeEnum,
DatasetTypeEnum
} from '@fastgpt/global/core/dataset/constant';
import type { RequestPaging } from '@/types';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import type { SearchTestItemType } from '@/types/core/dataset';
......@@ -31,14 +35,15 @@ export type CreateDatasetCollectionParams = {
parentId?: string;
name: string;
type: `${DatasetCollectionTypeEnum}`;
metadata?: DatasetCollectionSchemaType['metadata'];
updateTime?: string;
trainingType?: `${DatasetCollectionTrainingModeEnum}`;
chunkSize?: number;
fileId?: string;
rawLink?: string;
};
export type UpdateDatasetCollectionParams = {
id: string;
parentId?: string;
name?: string;
metadata?: DatasetCollectionSchemaType['metadata'];
};
/* ==== data ===== */
......
projects/app/src/global/core/dataset/type.d.ts
......@@ -16,7 +16,8 @@ export type DatasetCollectionsListItemType = {
updateTime: Date;
dataAmount: number;
trainingAmount: number;
metadata: DatasetCollectionSchemaType['metadata'];
fileId?: string;
rawLink?: string;
canWrite: boolean;
};
......
projects/app/src/pages/api/admin/initv463.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
await MongoDatasetCollection.updateMany({}, [
{
$set: {
createTime: '$updateTime',
trainingType: {
$cond: {
if: { $ifNull: ['$a', false] },
then: TrainingModeEnum.qa,
else: TrainingModeEnum.chunk
}
},
chunkSize: 0,
fileId: '$metadata.fileId',
rawLink: '$metadata.rawLink'
}
}
]);
await MongoDatasetData.updateMany(
{},
{
chunkIndex: 0,
updateTime: new Date()
}
);
jsonRes(res, {
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
projects/app/src/pages/api/core/dataset/collection/create.ts
......@@ -6,8 +6,11 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum,
DatasetCollectionTrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
......@@ -45,7 +48,10 @@ export async function createOneCollection({
parentId,
datasetId,
type,
metadata = {},
trainingType = DatasetCollectionTrainingModeEnum.manual,
chunkSize = 0,
fileId,
rawLink,
teamId,
tmbId
}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
......@@ -56,8 +62,10 @@ export async function createOneCollection({
datasetId,
parentId: parentId || null,
type,
metadata,
updateTime: getCollectionUpdateTime({ name })
trainingType,
chunkSize,
fileId,
rawLink
});
// create default collection
......@@ -94,7 +102,8 @@ export function createDefaultCollection({
datasetId,
parentId,
type: DatasetCollectionTypeEnum.virtual,
updateTime: new Date('2099'),
metadata: {}
trainingType: DatasetCollectionTrainingModeEnum.manual,
chunkSize: 0,
updateTime: new Date('2099')
});
}
projects/app/src/pages/api/core/dataset/collection/delById.ts
......@@ -42,10 +42,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// delete file
await Promise.all(
collections.map((collection) => {
if (!collection.metadata?.fileId) return;
if (!collection?.fileId) return;
return delFileById({
bucketName: BucketNameEnum.dataset,
fileId: collection.metadata.fileId
fileId: collection.fileId
});
})
);
......
projects/app/src/pages/api/core/dataset/collection/detail.ts
......@@ -6,6 +6,8 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getFileById } from '@fastgpt/service/common/file/gridfs/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......@@ -24,12 +26,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
per: 'r'
});
// get file
const file = collection?.fileId
? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
: undefined;
jsonRes<DatasetCollectionItemType>(res, {
data: {
...collection,
canWrite,
sourceName: collection?.name,
sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink
sourceId: collection?.fileId || collection?.rawLink,
file
}
});
} catch (err) {
......
projects/app/src/pages/api/core/dataset/collection/list.ts
......@@ -115,9 +115,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
name: 1,
type: 1,
updateTime: 1,
trainingAmount: { $size: '$trainings' },
dataAmount: { $size: '$datas' },
metadata: 1
trainingAmount: { $size: '$trainings' },
fileId: 1,
rawLink: 1
}
},
{
......
projects/app/src/pages/api/core/dataset/collection/update.ts
......@@ -9,7 +9,7 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name, metadata = {} } = req.body as UpdateDatasetCollectionParams;
const { id, parentId, name } = req.body as UpdateDatasetCollectionParams;
if (!id) {
throw new Error('缺少参数');
......@@ -23,11 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
...(name && { name, updateTime: getCollectionUpdateTime({ name }) })
};
// 将metadata的每个字段添加到updateFields中
for (const [key, value] of Object.entries(metadata)) {
updateFields[`metadata.${key}`] = value;
}
await MongoDatasetCollection.findByIdAndUpdate(id, {
$set: updateFields
});
......
projects/app/src/pages/api/core/dataset/data/insertData.ts
......@@ -76,6 +76,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
collectionId,
q: formatQ,
a: formatA,
chunkIndex: 0,
model: vectorModelData.model,
indexes: formatIndexes
});
......
projects/app/src/pages/api/core/dataset/data/list.ts
......@@ -32,8 +32,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
};
const [data, total] = await Promise.all([
MongoDatasetData.find(match, '_id datasetId collectionId q a indexes')
.sort({ _id: -1 })
MongoDatasetData.find(match, '_id datasetId collectionId q a indexes')
.sort({ chunkIndex: 1, updateTime: -1 })
.skip((pageNum - 1) * pageSize)
.limit(pageSize)
.lean(),
......
projects/app/src/pages/api/core/dataset/data/pushData.ts
......@@ -125,7 +125,7 @@ export async function pushDataToDatasetCollection({
// 插入记录
const insertRes = await MongoDatasetTraining.insertMany(
filterResult.success.map((item) => ({
filterResult.success.map((item, i) => ({
teamId,
tmbId,
datasetId,
......@@ -136,6 +136,7 @@ export async function pushDataToDatasetCollection({
model,
q: item.q,
a: item.a,
chunkIndex: i,
indexes: item.indexes
}))
);
......
projects/app/src/pages/api/core/dataset/delete.ts
......@@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { delDatasetFiles } from '@fastgpt/service/core/dataset/file/controller';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { delDataByDatasetId } from '@/service/core/dataset/data/controller';
......
projects/app/src/pages/api/core/plugin/templates.ts
......@@ -16,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const [userPlugins, plusPlugins] = await Promise.all([
MongoPlugin.find({ teamId }).lean(),
GET<PluginTemplateType[]>('/core/plugin/getTemplates')
global.systemEnv.pluginBaseUrl ? GET<PluginTemplateType[]>('/core/plugin/getTemplates') : []
]);
const data: FlowModuleTemplateType[] = [
......
projects/app/src/pages/api/v1/chat/completions.ts
......@@ -142,7 +142,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
responseDetail: detail,
apikey,
authType,
canWrite: false
canWrite: true
};
}
......
projects/app/src/pages/app/detail/components/SimpleEdit/index.tsx
......@@ -144,6 +144,7 @@ function ConfigForm({
templateId: appDetail.simpleTemplateId,
modules: appDetail.modules
});
reset(formVal);
setTimeout(() => {
setRefresh((state) => !state);
......
projects/app/src/pages/dataset/detail/components/CollectionCard.tsx
......@@ -36,7 +36,11 @@ import MyMenu from '@/components/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import EmptyTip from '@/components/EmptyTip';
import { FolderAvatarSrc, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
FolderAvatarSrc,
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
import { TabEnum } from '..';
......@@ -347,7 +351,7 @@ const CollectionCard = () => {
<Th>#</Th>
<Th>{t('common.Name')}</Th>
<Th>{t('dataset.collections.Data Amount')}</Th>
<Th>{t('common.Time')}</Th>
<Th>{t('core.dataset.Sync Time')}</Th>
<Th>{t('common.Status')}</Th>
<Th />
</Tr>
......
projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
......@@ -19,7 +19,6 @@ import { customAlphabet } from 'nanoid';
import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import type { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
......@@ -40,7 +39,8 @@ export type FileItemType = {
icon: string;
tokens: number; // total tokens
type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
metadata: DatasetCollectionSchemaType['metadata'];
fileId?: string;
rawLink?: string;
};
export interface Props extends BoxProps {
......@@ -157,9 +157,7 @@ const FileSelect = ({
.join('\n')}`,
chunks: filterData,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId
}
fileId
};
onPushFiles([fileItem]);
......@@ -195,9 +193,7 @@ const FileSelect = ({
text,
tokens: splitRes.tokens,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId
},
fileId,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
......@@ -220,7 +216,7 @@ const FileSelect = ({
// link fetch
const onUrlFetch = useCallback(
(e: FetchResultItem[]) => {
const result: FileItemType[] = e.map(({ url, content }) => {
const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
const splitRes = splitText2Chunks({
text: content,
chunkLen,
......@@ -233,9 +229,7 @@ const FileSelect = ({
text: content,
tokens: splitRes.tokens,
type: DatasetCollectionTypeEnum.link,
metadata: {
rawLink: url
},
rawLink: url,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
......@@ -277,9 +271,7 @@ const FileSelect = ({
text: content,
tokens: splitRes.tokens,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId: fileIds[0]
},
fileId: fileIds[0],
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: ''
......
projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
......@@ -11,7 +11,10 @@ import MyModal from '@/components/MyModal';
import Provider from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { qaModelList } from '@/web/common/system/staticData';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import {
DatasetCollectionTrainingModeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
export enum ImportTypeEnum {
chunk = 'chunk',
......@@ -43,19 +46,22 @@ const ImportData = ({
defaultChunkLen: vectorModel?.defaultToken || 500,
chunkOverlapRatio: 0.2,
unitPrice: vectorModel?.price || 0.2,
mode: TrainingModeEnum.chunk
mode: TrainingModeEnum.chunk,
collectionTrainingType: DatasetCollectionTrainingModeEnum.chunk
},
[ImportTypeEnum.qa]: {
defaultChunkLen: agentModel?.maxContext * 0.6 || 8000,
chunkOverlapRatio: 0,
unitPrice: agentModel?.price || 3,
mode: TrainingModeEnum.qa
mode: TrainingModeEnum.qa,
collectionTrainingType: DatasetCollectionTrainingModeEnum.qa
},
[ImportTypeEnum.csv]: {
defaultChunkLen: vectorModel?.defaultToken || 500,
defaultChunkLen: 0,
chunkOverlapRatio: 0,
unitPrice: vectorModel?.price || 0.2,
mode: TrainingModeEnum.chunk
mode: TrainingModeEnum.chunk,
collectionTrainingType: DatasetCollectionTrainingModeEnum.manual
}
};
return map[importType];
......
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
......@@ -15,7 +15,10 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import {
DatasetCollectionTrainingModeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constant';
import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
import { CloseIcon } from '@chakra-ui/icons';
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
......@@ -92,6 +95,7 @@ const Provider = ({
parentId,
unitPrice,
mode,
collectionTrainingType,
vectorModel,
agentModel,
defaultChunkLen = 500,
......@@ -104,6 +108,7 @@ const Provider = ({
parentId: string;
unitPrice: number;
mode: `${TrainingModeEnum}`;
collectionTrainingType: `${DatasetCollectionTrainingModeEnum}`;
vectorModel: string;
agentModel: string;
defaultChunkLen: number;
......@@ -150,7 +155,10 @@ const Provider = ({
parentId,
name: file.filename,
type: file.type,
metadata: file.metadata
fileId: file.fileId,
rawLink: file.rawLink,
chunkSize: chunkLen,
trainingType: collectionTrainingType
});
// upload data
......
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
......@@ -427,7 +427,11 @@ export function RawSourceText({
{...props}
>
<Image src={icon} alt="" w={['14px', '16px']} mr={2} />
<Box maxW={['200px', '300px']} className={props.className ?? 'textEllipsis'}>
<Box
maxW={['200px', '300px']}
className={props.className ?? 'textEllipsis'}
wordBreak={'break-all'}
>
{sourceName || t('common.UnKnow Source')}
</Box>
</Box>
......
projects/app/src/service/core/dataset/data/controller.ts
......@@ -22,6 +22,7 @@ export async function insertData2Dataset({
collectionId,
q,
a = '',
chunkIndex = 0,
indexes,
model
}: CreateDatasetDataProps & {
......@@ -73,6 +74,7 @@ export async function insertData2Dataset({
q,
a,
fullTextToken: jiebaSplit({ text: qaStr }),
chunkIndex,
indexes: indexes.map((item, i) => ({
...item,
dataId: result[i].insertId
......
projects/app/src/service/core/dataset/data/pg.ts
......@@ -270,7 +270,7 @@ export async function embeddingRecall({
{
_id: { $in: filterRows.map((item) => item.collection_id) }
},
'name metadata'
'name fileId rawLink'
).lean(),
MongoDatasetData.find(
{
......@@ -297,7 +297,7 @@ export async function embeddingRecall({
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
sourceName: collection.name || '',
sourceId: collection.metadata?.fileId || collection.metadata?.rawLink,
sourceId: collection?.fileId || collection?.rawLink,
score: item.score
};
})
......@@ -352,7 +352,7 @@ export async function fullTextRecall({ text, limit, datasetIds = [] }: SearchPro
{
_id: { $in: searchResults.map((item) => item.collectionId) }
},
'_id name metadata'
'_id name fileId rawLink'
);
return {
......@@ -363,7 +363,7 @@ export async function fullTextRecall({ text, limit, datasetIds = [] }: SearchPro
datasetId: String(item.datasetId),
collectionId: String(item.collectionId),
sourceName: collection?.name || '',
sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink,
sourceId: collection?.fileId || collection?.rawLink,
q: item.q,
a: item.a,
indexes: item.indexes,
......
projects/app/src/service/events/generateVector.ts
......@@ -50,6 +50,7 @@ export async function generateVector(): Promise<any> {
collectionId: 1,
q: 1,
a: 1,
chunkIndex: 1,
indexes: 1,
model: 1,
billId: 1
......@@ -134,6 +135,7 @@ export async function generateVector(): Promise<any> {
collectionId: data.collectionId,
q: dataItem.q,
a: dataItem.a,
chunkIndex: data.chunkIndex,
indexes: dataItem.indexes,
model: data.model
});
......
projects/app/src/service/moduleDispatch/dataset/search.ts
......@@ -65,7 +65,8 @@ export async function dispatchDatasetSearch(
model: vectorModel.name,
tokens: tokenLen,
similarity,
limit
limit,
searchMode
}
};
}
projects/app/src/service/support/permission/auth/dataset.ts
......@@ -31,7 +31,7 @@ export async function authDatasetData({
datasetId: String(datasetData.datasetId),
collectionId: String(datasetData.collectionId),
sourceName: result.collection.name || '',
sourceId: result.collection.metadata?.fileId || result.collection.metadata?.rawLink,
sourceId: result.collection?.fileId || result.collection?.rawLink,
isOwner: String(datasetData.tmbId) === result.tmbId,
canWrite: result.canWrite
};
......
projects/app/src/web/core/dataset/api.ts
......@@ -16,10 +16,7 @@ import type {
InsertOneDatasetDataProps
} from '@/global/core/dataset/api.d';
import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
import type {
DatasetCollectionItemType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment