Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit cd876251 by Archer Committed by GitHub
Options

External dataset (#1485) 

* fix: revert version

* feat: external collection

* import context

* external ui

* doc

* fix: ts

* clear invalid data

* feat: rename sub name

* fix: node if else edge remove

* fix: init

* api size

* fix: if else node refresh
parent fb04889a
Showing with 1461 additions and 934 deletions
.vscode/nextapi.code-snippets
......@@ -40,18 +40,11 @@
"",
"type ContextType = {$1};",
"",
"type ContextValueType = {};",
"",
"export const Context = createContext<ContextType>({});",
"",
"export const ContextProvider = ({",
" children,",
" value",
"}: {",
" children: ReactNode;",
" value: ContextValueType;",
"}) => {",
" return <Context.Provider value={value}>{children}</Context.Provider>;",
"export const ContextProvider = ({ children }: { children: ReactNode }) => {",
" const contextValue: ContextType = {};",
" return <Context.Provider value={contextValue}>{children}</Context.Provider>;",
"};",
],
"description": "FastGPT usecontext template"
......
Dockerfile
......@@ -36,6 +36,8 @@ COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modu
RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
ENV NODE_OPTIONS="--max-old-space-size=4096"
RUN pnpm --filter=$name build
# --------- runner -----------
......
docSite/content/docs/development/faq.md
......@@ -118,4 +118,5 @@ OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并
### bad_response_status_code bad response status code 503
1. 模型服务不可用
2. ....
\ No newline at end of file
2. 模型接口参数异常（温度、max token等可能不适配）
3. ....
\ No newline at end of file
docSite/content/docs/development/upgrading/481.md
......@@ -35,4 +35,5 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \
## V4.8.1 更新说明
1. 新增 - 知识库重新选择向量模型重建
2. 修复 - 定时器清理脏数据任务
\ No newline at end of file
2. 修复 - 工作流删除节点的动态输入和输出时候，没有正确的删除连接线，导致可能出现逻辑异常。
3. 修复 - 定时器清理脏数据任务
\ No newline at end of file
packages/global/core/dataset/api.d.ts
......@@ -11,14 +11,16 @@ export type DatasetUpdateBody = {
intro?: string;
permission?: DatasetSchemaType['permission'];
agentModel?: LLMModelItemType;
websiteConfig?: DatasetSchemaType['websiteConfig'];
status?: DatasetSchemaType['status'];
websiteConfig?: DatasetSchemaType['websiteConfig'];
externalReadUrl?: DatasetSchemaType['externalReadUrl'];
};
/* ================= collection ===================== */
export type DatasetCollectionChunkMetadataType = {
parentId?: string;
trainingType?: `${TrainingModeEnum}`;
trainingType?: TrainingModeEnum;
chunkSize?: number;
chunkSplitter?: string;
qaPrompt?: string;
......@@ -78,7 +80,7 @@ export type PostWebsiteSyncParams = {
export type PushDatasetDataProps = {
collectionId: string;
data: PushDatasetDataChunkProps[];
trainingMode: `${TrainingModeEnum}`;
trainingMode: TrainingModeEnum;
prompt?: string;
billId?: string;
};
......
packages/global/core/dataset/collection/constants.ts 0 → 100644
/* sourceId = prefix-id; id=fileId;link url;externalId */
export enum CollectionSourcePrefixEnum {
local = 'local',
link = 'link',
external = 'external'
}
packages/global/core/dataset/constants.ts
......@@ -2,23 +2,29 @@
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset',
websiteDataset = 'websiteDataset' // depp link
websiteDataset = 'websiteDataset', // depp link
externalFile = 'externalFile'
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
icon: 'common/folderFill',
label: 'core.dataset.Folder Dataset',
label: 'Folder Dataset',
collectionLabel: 'common.Folder'
},
[DatasetTypeEnum.dataset]: {
icon: 'core/dataset/commonDataset',
label: 'core.dataset.Common Dataset',
label: 'Common Dataset',
collectionLabel: 'common.File'
},
[DatasetTypeEnum.websiteDataset]: {
icon: 'core/dataset/websiteDataset',
label: 'core.dataset.Website Dataset',
label: 'Website Dataset',
collectionLabel: 'common.Website'
},
[DatasetTypeEnum.externalFile]: {
icon: 'core/dataset/commonDataset',
label: 'External File',
collectionLabel: 'common.File'
}
};
......@@ -77,7 +83,8 @@ export enum ImportDataSourceEnum {
fileLocal = 'fileLocal',
fileLink = 'fileLink',
fileCustom = 'fileCustom',
csvTable = 'csvTable'
csvTable = 'csvTable',
externalFile = 'externalFile'
}
export enum TrainingModeEnum {
......
packages/global/core/dataset/type.d.ts
......@@ -22,13 +22,16 @@ export type DatasetSchemaType = {
vectorModel: string;
agentModel: string;
intro: string;
type: `${DatasetTypeEnum}`;
type: DatasetTypeEnum;
status: `${DatasetStatusEnum}`;
permission: `${PermissionTypeEnum}`;
// metadata
websiteConfig?: {
url: string;
selector: string;
};
externalReadUrl?: string;
};
export type DatasetCollectionSchemaType = {
......@@ -42,16 +45,18 @@ export type DatasetCollectionSchemaType = {
createTime: Date;
updateTime: Date;
trainingType: `${TrainingModeEnum}`;
trainingType: TrainingModeEnum;
chunkSize: number;
chunkSplitter?: string;
qaPrompt?: string;
fileId?: string;
rawLink?: string;
sourceId?: string; // relate CollectionSourcePrefixEnum
fileId?: string; // local file id
rawLink?: string; // link url
rawTextLength?: number;
hashRawText?: string;
externalSourceUrl?: string; // external import url
metadata?: {
webPageSelector?: string;
relatedImgId?: string; // The id of the associated image collections
......@@ -93,7 +98,7 @@ export type DatasetTrainingSchemaType = {
billId: string;
expireAt: Date;
lockTime: Date;
mode: `${TrainingModeEnum}`;
mode: TrainingModeEnum;
model: string;
prompt: string;
dataId?: string;
......@@ -112,13 +117,19 @@ export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collect
};
/* ================= dataset ===================== */
export type DatasetSimpleItemType = {
_id: string;
avatar: string;
name: string;
vectorModel: VectorModelItemType;
};
export type DatasetListItemType = {
_id: string;
parentId: string;
avatar: string;
name: string;
intro: string;
type: `${DatasetTypeEnum}`;
type: DatasetTypeEnum;
isOwner: boolean;
canWrite: boolean;
permission: `${PermissionTypeEnum}`;
......
packages/global/core/dataset/utils.ts
......@@ -46,7 +46,7 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin
};
}
export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => {
export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
if (mode === TrainingModeEnum.qa) return data.length * 20;
if (mode === TrainingModeEnum.auto) return data.length * 5;
return data.length;
......
packages/global/core/workflow/template/system/assignedAnswer.ts
......@@ -18,6 +18,7 @@ export const AssignedAnswerModule: FlowNodeTemplateType = {
intro:
'该模块可以直接回复一段指定的内容。常用于引导、提示。非字符串内容传入时，会转成字符串进行输出。',
version: '481',
isTool: true,
inputs: [
{
key: NodeInputKeyEnum.answerText,
......
packages/service/core/dataset/collection/schema.ts
......@@ -16,11 +16,6 @@ const DatasetCollectionSchema = new Schema({
ref: DatasetColCollectionName,
default: null
},
userId: {
// abandoned
type: Schema.Types.ObjectId,
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
......@@ -54,6 +49,7 @@ const DatasetCollectionSchema = new Schema({
default: () => new Date()
},
// chunk filed
trainingType: {
type: String,
enum: Object.keys(TrainingTypeMap),
......@@ -70,20 +66,21 @@ const DatasetCollectionSchema = new Schema({
type: String
},
sourceId: String,
// local file collection
fileId: {
type: Schema.Types.ObjectId,
ref: 'dataset.files'
},
rawLink: {
type: String
},
// web link collection
rawLink: String,
rawTextLength: {
type: Number
},
hashRawText: {
type: String
},
// external collection
// metadata
rawTextLength: Number,
hashRawText: String,
externalSourceUrl: String, // external import url
metadata: {
type: Object,
default: {}
......
packages/service/core/dataset/schema.ts
......@@ -89,7 +89,8 @@ const DatasetSchema = new Schema({
default: 'body'
}
}
}
},
externalReadUrl: String
});
try {
......
packages/service/support/wallet/sub/schema.ts
......@@ -14,7 +14,7 @@ import {
} from '@fastgpt/global/support/wallet/sub/constants';
import type { TeamSubSchema } from '@fastgpt/global/support/wallet/sub/type';
export const subCollectionName = 'team.subscriptions';
export const subCollectionName = 'team_subscriptions';
const SubSchema = new Schema({
teamId: {
......
packages/web/components/common/DndDrag/DragIcon.tsx
import { DragHandleIcon } from '@chakra-ui/icons';
import { Box } from '@chakra-ui/react';
import { Box, BoxProps } from '@chakra-ui/react';
import React from 'react';
import { DraggableProvided } from 'react-beautiful-dnd';
const DragIcon = ({ provided }: { provided: DraggableProvided }) => {
const DragIcon = ({ provided, ...props }: { provided: DraggableProvided } & BoxProps) => {
return (
<Box {...provided.dragHandleProps}>
<Box {...provided.dragHandleProps} {...props}>
<DragHandleIcon color={'myGray.500'} _hover={{ color: 'primary.600' }} />
</Box>
);
......
projects/app/.eslintrc.json
{
"parser": "@typescript-eslint/parser", // 确保使用了 TypeScript 解析器
"plugins": ["@typescript-eslint"], // 引入 TypeScript 插件
"extends": "next/core-web-vitals",
"rules": {
"react-hooks/rules-of-hooks": 0,
"@typescript-eslint/consistent-type-imports": "warn" // 或者 "error" 来强制执行
"react-hooks/rules-of-hooks": 0
}
}
projects/app/i18n/en/common.json
{
"Add new": "Add new",
"App": "App",
"Export": "Export",
"Folder": "Folder",
......@@ -509,18 +510,14 @@
"Choose Dataset": "Associate dataset",
"Chunk amount": "Number of chunks",
"Collection": "Dataset",
"Common Dataset": "Common dataset",
"Common Dataset Desc": "Can be built by importing files, web links, or manual entry",
"Create dataset": "Create a dataset",
"Dataset": "Dataset",
"Dataset ID": "Dataset ID",
"Dataset Type": "Dataset type",
"Delete Confirm": "Confirm to delete this dataset? Data cannot be recovered after deletion, please confirm!",
"Delete Website Tips": "Confirm to delete this site?",
"Empty Dataset": "",
"Empty Dataset Tips": "No datasets yet, go create one!",
"File collection": "File dataset",
"Folder Dataset": "Folder",
"Folder placeholder": "This is a directory",
"Go Dataset": "Go to dataset",
"Intro Placeholder": "This dataset has no introduction~",
......@@ -540,8 +537,6 @@
"Table collection": "Table dataset",
"Text collection": "Text dataset",
"Total chunks": "Total chunks: {{total}}",
"Website Dataset": "Web site synchronization",
"Website Dataset Desc": "Web site synchronization allows you to use a web page link to build a dataset",
"collection": {
"Click top config website": "Click to configure website",
"Collection name": "Dataset name",
......
projects/app/i18n/en/dataset.json
{
"Common Dataset": "Common dataset",
"Common Dataset Desc": "Can be built by importing files, web links, or manual entry",
"Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index? Switching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.",
"External file": "External file",
"External file Dataset Desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice",
"External id": "File id",
"External read url": "External read url",
"External url": "File read url",
"Folder Dataset": "Folder",
"Rebuild embedding start tip": "The task of switching index models has begun",
"Rebuilding index count": "Rebuilding count: {{count}}",
"The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt"
"The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt",
"Website Dataset": "Web site",
"Website Dataset Desc": "Web site synchronization allows you to use a web page link to build a dataset",
"filename": "filename"
}
projects/app/i18n/zh/common.json
{
"Add new": "新增",
"App": "应用",
"Export": "导出",
"Folder": "文件夹",
......@@ -509,8 +510,6 @@
"Choose Dataset": "关联知识库",
"Chunk amount": "分段数",
"Collection": "数据集",
"Common Dataset": "通用知识库",
"Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库",
"Create dataset": "创建一个知识库",
"Dataset": "知识库",
"Dataset ID": "知识库 ID",
......@@ -520,7 +519,6 @@
"Empty Dataset": "",
"Empty Dataset Tips": "还没有知识库，快去创建一个吧！",
"File collection": "文件数据集",
"Folder Dataset": "文件夹",
"Folder placeholder": "这是一个目录",
"Go Dataset": "前往知识库",
"Intro Placeholder": "这个知识库还没有介绍~",
......@@ -540,8 +538,6 @@
"Table collection": "表格数据集",
"Text collection": "文本数据集",
"Total chunks": "总分段: {{total}}",
"Website Dataset": "Web 站点同步",
"Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
"collection": {
"Click top config website": "点击配置网站",
"Collection name": "数据集名称",
......
projects/app/i18n/zh/dataset.json
{
"Common Dataset": "通用知识库",
"Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库",
"Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。",
"External File": "外部文件库",
"External file Dataset Desc": "可以从外部文件库导入文件构建知识库，文件不会进行二次存储",
"External id": "文件阅读ID",
"External read url": "外部预览地址",
"External url": "文件访问URL",
"Folder Dataset": "文件夹",
"Rebuild embedding start tip": "切换索引模型任务已开始",
"Rebuilding index count": "重建中索引数量: {{count}}",
"The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引"
"The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引",
"Website Dataset": "Web 站点同步",
"Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
"filename": "文件名"
}
projects/app/src/components/common/NextHead/index.tsx 0 → 100644
import Head from 'next/head';
import React from 'react';
const NextHead = ({ title, icon, desc }: { title?: string; icon?: string; desc?: string }) => {
return (
<Head>
<title>{title}</title>
{desc && <meta name="description" content={desc} />}
{icon && <link rel="icon" href={icon} />}
</Head>
);
};
export default NextHead;
projects/app/src/components/core/dataset/DatasetTypeTag.tsx
import { Box, Flex, FlexProps } from '@chakra-ui/react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import React from 'react';
import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import { useI18n } from '@/web/context/I18n';
const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => {
const { t } = useTranslation();
const DatasetTypeTag = ({ type, ...props }: { type: DatasetTypeEnum } & FlexProps) => {
const { datasetT } = useI18n();
const item = DatasetTypeMap[type] || DatasetTypeMap['dataset'];
......@@ -22,7 +22,8 @@ const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & Fle
{...props}
>
<MyIcon name={item.icon as any} w={'16px'} mr={2} color={'myGray.400'} />
<Box>{t(item.label)}</Box>
{/* @ts-ignore */}
<Box>{datasetT(item.label)}</Box>
</Flex>
);
};
......
projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx
......@@ -50,16 +50,11 @@ const ListItem = ({
}) => {
const { t } = useTranslation();
const { getZoom } = useReactFlow();
const onDelEdge = useContextSelector(WorkflowContext, (v) => v.onDelEdge);
const handleId = getHandleId(nodeId, 'source', getElseIFLabel(conditionIndex));
return (
<Box
ref={provided.innerRef}
{...provided.draggableProps}
style={{
...provided.draggableProps.style,
opacity: snapshot.isDragging ? 0.8 : 1
}}
>
const Render = useMemo(() => {
return (
<Flex
alignItems={'center'}
position={'relative'}
......@@ -68,7 +63,10 @@ const ListItem = ({
>
<Container w={snapshot.isDragging ? '' : 'full'} className="nodrag">
<Flex mb={4} alignItems={'center'}>
{ifElseList.length > 1 && <DragIcon provided={provided} />}
<DragIcon
visibility={ifElseList.length > 1 ? 'visible' : 'hidden'}
provided={provided}
/>
<Box color={'black'} fontSize={'lg'} ml={2}>
{getElseIFLabel(conditionIndex)}
</Box>
......@@ -109,6 +107,10 @@ const ListItem = ({
color={'myGray.400'}
onClick={() => {
onUpdateIfElseList(ifElseList.filter((_, index) => index !== conditionIndex));
onDelEdge({
nodeId,
sourceHandle: handleId
});
}}
/>
)}
......@@ -185,21 +187,21 @@ const ListItem = ({
onChange={(e) => {
onUpdateIfElseList(
ifElseList.map((ifElse, index) => {
if (index === conditionIndex) {
return {
...ifElse,
list: ifElse.list.map((item, index) => {
if (index === i) {
return {
...item,
value: e
};
}
return item;
})
};
}
return ifElse;
return {
...ifElse,
list:
index === conditionIndex
? ifElse.list.map((item, index) => {
if (index === i) {
return {
...item,
value: e
};
}
return item;
})
: ifElse.list
};
})
);
}}
......@@ -263,12 +265,38 @@ const ListItem = ({
{!snapshot.isDragging && (
<SourceHandle
nodeId={nodeId}
handleId={getHandleId(nodeId, 'source', getElseIFLabel(conditionIndex))}
handleId={handleId}
position={Position.Right}
translate={[18, 0]}
/>
)}
</Flex>
);
}, [
conditionIndex,
conditionItem.condition,
conditionItem.list,
getZoom,
handleId,
ifElseList,
nodeId,
onDelEdge,
onUpdateIfElseList,
provided,
snapshot.isDragging,
t
]);
return (
<Box
ref={provided.innerRef}
{...provided.draggableProps}
style={{
...provided.draggableProps.style,
opacity: snapshot.isDragging ? 0.8 : 1
}}
>
{Render}
</Box>
);
};
......@@ -387,35 +415,39 @@ const ConditionValueInput = ({
return output.valueType;
}, [nodeList, variable]);
if (valueType === WorkflowIOValueTypeEnum.boolean) {
return (
<MySelect
list={[
{ label: 'True', value: 'true' },
{ label: 'False', value: 'false' }
]}
onchange={onChange}
value={value}
placeholder={'选择值'}
isDisabled={
condition === VariableConditionEnum.isEmpty ||
condition === VariableConditionEnum.isNotEmpty
}
/>
);
} else {
return (
<MyInput
value={value}
placeholder={'输入值'}
w={'100%'}
bg={'white'}
isDisabled={
condition === VariableConditionEnum.isEmpty ||
condition === VariableConditionEnum.isNotEmpty
}
onChange={(e) => onChange(e.target.value)}
/>
);
}
const Render = useMemo(() => {
if (valueType === WorkflowIOValueTypeEnum.boolean) {
return (
<MySelect
list={[
{ label: 'True', value: 'true' },
{ label: 'False', value: 'false' }
]}
onchange={onChange}
value={value}
placeholder={'选择值'}
isDisabled={
condition === VariableConditionEnum.isEmpty ||
condition === VariableConditionEnum.isNotEmpty
}
/>
);
} else {
return (
<MyInput
value={value}
placeholder={'输入值'}
w={'100%'}
bg={'white'}
isDisabled={
condition === VariableConditionEnum.isEmpty ||
condition === VariableConditionEnum.isNotEmpty
}
onChange={(e) => onChange(e.target.value)}
/>
);
}
}, [condition, onChange, value, valueType]);
return Render;
};
projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx
import React, { useCallback, useMemo, useState } from 'react';
import React, { useCallback, useMemo } from 'react';
import NodeCard from '../render/NodeCard';
import { useTranslation } from 'next-i18next';
import { Box, Button, Flex } from '@chakra-ui/react';
......@@ -9,7 +9,7 @@ import { IfElseListItemType } from '@fastgpt/global/core/workflow/template/syste
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '../../../context';
import Container from '../../components/Container';
import DndDrag, { Draggable, DropResult } from '@fastgpt/web/components/common/DndDrag/index';
import DndDrag, { Draggable } from '@fastgpt/web/components/common/DndDrag/index';
import { SourceHandle } from '../render/Handle';
import { getHandleId } from '@fastgpt/global/core/workflow/utils';
import ListItem from './ListItem';
......@@ -19,6 +19,7 @@ const NodeIfElse = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
const { t } = useTranslation();
const { nodeId, inputs = [] } = data;
const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
const elseHandleId = getHandleId(nodeId, 'source', IfElseResultEnum.ELSE);
const ifElseList = useMemo(
() =>
......@@ -49,7 +50,7 @@ const NodeIfElse = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
<NodeCard selected={selected} maxW={'1000px'} {...data}>
<Box px={4} cursor={'default'}>
<DndDrag<IfElseListItemType>
onDragEndCb={(list) => onUpdateIfElseList(list)}
onDragEndCb={(list: IfElseListItemType[]) => onUpdateIfElseList(list)}
dataList={ifElseList}
renderClone={(provided, snapshot, rubric) => (
<ListItem
......@@ -95,7 +96,7 @@ const NodeIfElse = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
</Box>
<SourceHandle
nodeId={nodeId}
handleId={getHandleId(nodeId, 'source', IfElseResultEnum.ELSE)}
handleId={elseHandleId}
position={Position.Right}
translate={[26, 0]}
/>
......
projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx
import React, { useState } from 'react';
import React, { useCallback, useState } from 'react';
import { getPublishList, postRevertVersion } from '@/web/core/app/versionApi';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import CustomRightDrawer from '@fastgpt/web/components/common/MyDrawer/CustomRightDrawer';
......@@ -14,6 +14,8 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type';
import { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
const PublishHistoriesSlider = () => {
const { t } = useTranslation();
......@@ -45,29 +47,29 @@ const PublishHistoriesSlider = () => {
setIsShowVersionHistories(false);
});
const onPreview = useMemoizedFn((data: AppVersionSchemaType) => {
const onPreview = useCallback((data: AppVersionSchemaType) => {
setSelectedHistoryId(data._id);
initData({
nodes: data.nodes,
edges: data.edges
});
});
const onCloseSlider = useMemoizedFn(() => {
setSelectedHistoryId(undefined);
initData({
nodes: appDetail.modules,
edges: appDetail.edges
});
onClose();
});
}, []);
const onCloseSlider = useCallback(
(data: { nodes: StoreNodeItemType[]; edges: StoreEdgeItemType[] }) => {
setSelectedHistoryId(undefined);
initData(data);
onClose();
},
[appDetail]
);
const { mutate: onRevert, isLoading: isReverting } = useRequest({
mutationFn: async (data: AppVersionSchemaType) => {
if (!appId) return;
await postRevertVersion(appId, {
versionId: data._id,
editNodes: appDetail.modules,
editNodes: appDetail.modules, // old workflow
editEdges: appDetail.edges
});
......@@ -77,7 +79,7 @@ const PublishHistoriesSlider = () => {
edges: data.edges
});
onCloseSlider();
onCloseSlider(data);
}
});
......@@ -86,7 +88,12 @@ const PublishHistoriesSlider = () => {
return (
<>
<CustomRightDrawer
onClose={onCloseSlider}
onClose={() =>
onCloseSlider({
nodes: appDetail.modules,
edges: appDetail.edges
})
}
iconSrc="core/workflow/versionHistories"
title={t('core.workflow.publish.histories')}
maxW={'300px'}
......
projects/app/src/components/core/workflow/context.tsx
......@@ -430,8 +430,8 @@ const WorkflowContextProvider = ({
const initData = useMemoizedFn(
async (e: { nodes: StoreNodeItemType[]; edges: StoreEdgeItemType[] }) => {
setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item })));
setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })));
setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item })) || []);
setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || []);
}
);
......
projects/app/src/global/core/dataset/api.d.ts
......@@ -14,7 +14,7 @@ import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
/* ================= dataset ===================== */
export type CreateDatasetParams = {
parentId?: string;
type: `${DatasetTypeEnum}`;
type: DatasetTypeEnum;
name: string;
intro: string;
avatar: string;
......@@ -76,7 +76,7 @@ export type SearchTestResponse = {
/* =========== training =========== */
export type PostPreviewFilesChunksProps = {
type: `${ImportDataSourceEnum}`;
type: ImportDataSourceEnum;
sourceId: string;
chunkSize: number;
overlapRatio: number;
......
projects/app/src/pages/_app.tsx
......@@ -10,28 +10,22 @@ import I18nContextProvider from '@/web/context/I18n';
import { useInitApp } from '@/web/context/useInitApp';
import '@/web/styles/reset.scss';
import NextHead from '@/components/common/NextHead';
function App({ Component, pageProps }: AppProps) {
const { feConfigs, scripts, title } = useInitApp();
return (
<>
<Head>
<title>{title}</title>
<meta
name="description"
content={
feConfigs?.systemDescription ||
process.env.SYSTEM_DESCRIPTION ||
`${title} 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！`
}
/>
<meta
name="viewport"
content="width=device-width,initial-scale=1.0,maximum-scale=1.0,minimum-scale=1.0,user-scalable=no, viewport-fit=cover"
/>
<link rel="icon" href={feConfigs.favicon || process.env.SYSTEM_FAVICON} />
</Head>
<NextHead
title={title}
desc={
feConfigs?.systemDescription ||
process.env.SYSTEM_DESCRIPTION ||
`${title} 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！`
}
icon={feConfigs?.favicon || process.env.SYSTEM_FAVICON}
/>
{scripts?.map((item, i) => <Script key={i} strategy="lazyOnload" {...item}></Script>)}
<QueryClientContext>
......
projects/app/src/pages/api/admin/clearInvalidData.ts
......@@ -58,17 +58,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
(async () => {
try {
console.log('执行脏数据清理任务');
// 360天 ~ 2小时前
const end = addHours(new Date(), -2);
const start = addHours(new Date(), -360 * 24);
await checkInvalidDatasetFiles(start, end);
await checkInvalidImg(start, end);
await checkInvalidDatasetData(start, end);
await checkInvalidVector(start, end);
const endTime = addHours(new Date(), start);
const startTime = addHours(new Date(), end);
await checkInvalidDatasetFiles(startTime, endTime);
await checkInvalidImg(startTime, endTime);
await checkInvalidDatasetData(startTime, endTime);
await checkInvalidVector(startTime, endTime);
console.log('执行脏数据清理任务完毕');
} catch (error) {
console.log('执行脏数据清理任务出错了');
......
projects/app/src/pages/api/admin/initv481.ts
......@@ -141,11 +141,18 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
const collections = await connectionMongo.connection.db
.listCollections({ name: 'team.members' })
.toArray();
if (collections.length > 0) {
const sourceCol = connectionMongo.connection.db.collection('team.members');
const targetCol = connectionMongo.connection.db.collection('team_members');
await sourceCol.rename('team_members', { dropTarget: true });
console.log('success rename team.members -> team_members');
if ((await targetCol.countDocuments()) > 1) {
// 除了root
console.log('team_members 中有数据，无法自动将 buffer.tts 迁移到 team_members，请手动操作');
} else {
await sourceCol.rename('team_members', { dropTarget: true });
console.log('success rename team.members -> team_members');
}
}
} catch (error) {
console.log('error： rename team.members -> team_members', error);
......@@ -170,6 +177,27 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
console.log('error： rename team.tags -> team_tags', error);
}
try {
const collections = await connectionMongo.connection.db
.listCollections({ name: 'team.subscriptions' })
.toArray();
if (collections.length > 0) {
const sourceCol = connectionMongo.connection.db.collection('team.subscriptions');
const targetCol = connectionMongo.connection.db.collection('team_subscriptions');
if ((await targetCol.countDocuments()) > 0) {
console.log(
'team_subscriptions 中有数据，无法自动将 team.subscriptions 迁移到 team_subscriptions，请手动操作'
);
} else {
await sourceCol.rename('team_subscriptions', { dropTarget: true });
console.log('success rename team.subscriptions -> team_subscriptions');
}
}
} catch (error) {
console.log('error： rename team.subscriptions -> team_subscriptions', error);
}
jsonRes(res, {
message: 'success'
});
......
projects/app/src/pages/api/common/file/uploadImage.ts
......@@ -28,7 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
export const config = {
api: {
sizeLimit: '10mb',
bodyParser: {
sizeLimit: '16mb'
}
......
projects/app/src/pages/api/core/dataset/allDataset.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { NextAPI } from '@/service/middle/entry';
/* get all dataset by teamId or tmbId */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true });
async function handler(
req: NextApiRequest,
res: NextApiResponse<any>
): Promise<DatasetSimpleItemType[]> {
// 凭证校验
const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true });
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
type: { $ne: DatasetTypeEnum.folder }
}).lean();
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
type: { $ne: DatasetTypeEnum.folder }
}).lean();
const data = datasets.map((item) => ({
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
vectorModel: getVectorModel(item.vectorModel),
canWrite: String(item.tmbId) === tmbId,
isOwner: teamOwner || String(item.tmbId) === tmbId
}));
jsonRes<DatasetListItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return datasets.map((item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
vectorModel: getVectorModel(item.vectorModel)
}));
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/list.ts
......@@ -9,7 +9,7 @@ import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { NextAPI } from '@/service/middle/entry';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum };
// 凭证校验
const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({
req,
......
projects/app/src/pages/api/core/dataset/update.ts
......@@ -8,8 +8,18 @@ import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name, avatar, intro, permission, agentModel, websiteConfig, status } =
req.body as DatasetUpdateBody;
const {
id,
parentId,
name,
avatar,
intro,
permission,
agentModel,
websiteConfig,
externalReadUrl,
status
} = req.body as DatasetUpdateBody;
if (!id) {
throw new Error('缺少参数');
......@@ -33,7 +43,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
...(agentModel && { agentModel: agentModel.model }),
...(websiteConfig && { websiteConfig }),
...(status && { status }),
...(intro && { intro })
...(intro && { intro }),
...(externalReadUrl && { externalReadUrl })
}
);
......
projects/app/src/pages/api/core/plugin/list.ts
......@@ -9,7 +9,7 @@ import { PluginListItemType } from '@fastgpt/global/core/plugin/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum };
const { teamId } = await authCert({ req, authToken: true });
......
projects/app/src/pages/api/core/workflow/debug.ts
......@@ -82,7 +82,7 @@ export default NextAPI(handler);
export const config = {
api: {
bodyParser: {
sizeLimit: '10mb'
sizeLimit: '20mb'
},
responseLimit: '20mb'
}
......
projects/app/src/pages/api/plugins/textEditor/v2/index.ts
......@@ -43,6 +43,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
export const config = {
api: {
bodyParser: {
sizeLimit: '16mb'
},
responseLimit: '16mb'
}
};
projects/app/src/pages/api/v1/chat/completions.ts
......@@ -523,6 +523,9 @@ const authHeaderRequest = async ({
export const config = {
api: {
bodyParser: {
sizeLimit: '20mb'
},
responseLimit: '20mb'
}
};
projects/app/src/pages/chat/share.tsx
......@@ -269,139 +269,141 @@ const OutLink = ({
}, []);
return (
<PageContainer
{...(isEmbed
? { p: '0 !important', insertProps: { borderRadius: '0', boxShadow: 'none' } }
: { p: [0, 5] })}
>
<>
<Head>
<title>{appName || chatData.app?.name}</title>
<meta name="description" content={appIntro} />
<link rel="icon" href={appAvatar || chatData.app?.avatar} />
</Head>
<MyBox
isLoading={isFetching}
h={'100%'}
display={'flex'}
flexDirection={['column', 'row']}
bg={'white'}
<PageContainer
{...(isEmbed
? { p: '0 !important', insertProps: { borderRadius: '0', boxShadow: 'none' } }
: { p: [0, 5] })}
>
{showHistory === '1'
? ((children: React.ReactNode) => {
return isPc ? (
<SideBar>{children}</SideBar>
) : (
<Drawer
isOpen={isOpenSlider}
placement="left"
autoFocus={false}
size={'xs'}
<MyBox
isLoading={isFetching}
h={'100%'}
display={'flex'}
flexDirection={['column', 'row']}
bg={'white'}
>
{showHistory === '1'
? ((children: React.ReactNode) => {
return isPc ? (
<SideBar>{children}</SideBar>
) : (
<Drawer
isOpen={isOpenSlider}
placement="left"
autoFocus={false}
size={'xs'}
onClose={onCloseSlider}
>
<DrawerOverlay backgroundColor={'rgba(255,255,255,0.5)'} />
<DrawerContent maxWidth={'250px'} boxShadow={'2px 0 10px rgba(0,0,0,0.15)'}>
{children}
</DrawerContent>
</Drawer>
);
})(
<ChatHistorySlider
appName={chatData.app.name}
appAvatar={chatData.app.avatar}
confirmClearText={t('core.chat.Confirm to clear share chat history')}
activeChatId={chatId}
history={histories.map((item) => ({
id: item.chatId,
title: item.title,
customTitle: item.customTitle,
top: item.top
}))}
onClose={onCloseSlider}
>
<DrawerOverlay backgroundColor={'rgba(255,255,255,0.5)'} />
<DrawerContent maxWidth={'250px'} boxShadow={'2px 0 10px rgba(0,0,0,0.15)'}>
{children}
</DrawerContent>
</Drawer>
);
})(
<ChatHistorySlider
appName={chatData.app.name}
appAvatar={chatData.app.avatar}
confirmClearText={t('core.chat.Confirm to clear share chat history')}
activeChatId={chatId}
history={histories.map((item) => ({
id: item.chatId,
title: item.title,
customTitle: item.customTitle,
top: item.top
}))}
onClose={onCloseSlider}
onChangeChat={(chatId) => {
router.replace({
query: {
...router.query,
chatId: chatId || ''
onChangeChat={(chatId) => {
router.replace({
query: {
...router.query,
chatId: chatId || ''
}
});
if (!isPc) {
onCloseSlider();
}
});
if (!isPc) {
onCloseSlider();
}}
onDelHistory={({ chatId }) =>
delOneHistory({ appId: chatData.appId, chatId, shareId, outLinkUid })
}
}}
onDelHistory={({ chatId }) =>
delOneHistory({ appId: chatData.appId, chatId, shareId, outLinkUid })
}
onClearHistory={() => {
clearHistories({ shareId, outLinkUid });
router.replace({
query: {
...router.query,
chatId: ''
}
});
}}
onSetHistoryTop={(e) => {
updateHistory({
...e,
appId: chatData.appId,
shareId,
outLinkUid
});
}}
onSetCustomTitle={async (e) => {
updateHistory({
appId: chatData.appId,
chatId: e.chatId,
title: e.title,
customTitle: e.title,
shareId,
outLinkUid
});
}}
/>
)
: null}
onClearHistory={() => {
clearHistories({ shareId, outLinkUid });
router.replace({
query: {
...router.query,
chatId: ''
}
});
}}
onSetHistoryTop={(e) => {
updateHistory({
...e,
appId: chatData.appId,
shareId,
outLinkUid
});
}}
onSetCustomTitle={async (e) => {
updateHistory({
appId: chatData.appId,
chatId: e.chatId,
title: e.title,
customTitle: e.title,
shareId,
outLinkUid
});
}}
/>
)
: null}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
<ChatHeader
appAvatar={chatData.app.avatar}
appName={chatData.app.name}
history={chatData.history}
showHistory={showHistory === '1'}
onOpenSlider={onOpenSlider}
/>
{/* chat box */}
<Box flex={1}>
<ChatBox
active={!!chatData.app.name}
ref={ChatBoxRef}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
<ChatHeader
appAvatar={chatData.app.avatar}
userAvatar={chatData.userAvatar}
userGuideModule={chatData.app?.userGuideModule}
showFileSelector={checkChatSupportSelectFileByChatModels(chatData.app.chatModels)}
feedbackType={'user'}
onUpdateVariable={(e) => {}}
onStartChat={startChat}
onDelMessage={(e) =>
delOneHistoryItem({ ...e, appId: chatData.appId, chatId, shareId, outLinkUid })
}
appId={chatData.appId}
chatId={chatId}
shareId={shareId}
outLinkUid={outLinkUid}
appName={chatData.app.name}
history={chatData.history}
showHistory={showHistory === '1'}
onOpenSlider={onOpenSlider}
/>
</Box>
</Flex>
</MyBox>
</PageContainer>
{/* chat box */}
<Box flex={1}>
<ChatBox
active={!!chatData.app.name}
ref={ChatBoxRef}
appAvatar={chatData.app.avatar}
userAvatar={chatData.userAvatar}
userGuideModule={chatData.app?.userGuideModule}
showFileSelector={checkChatSupportSelectFileByChatModels(chatData.app.chatModels)}
feedbackType={'user'}
onUpdateVariable={(e) => {}}
onStartChat={startChat}
onDelMessage={(e) =>
delOneHistoryItem({ ...e, appId: chatData.appId, chatId, shareId, outLinkUid })
}
appId={chatData.appId}
chatId={chatId}
shareId={shareId}
outLinkUid={outLinkUid}
/>
</Box>
</Flex>
</MyBox>
</PageContainer>
</>
);
};
......
projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx 0 → 100644
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext, useContextSelector } from 'use-context-selector';
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { useDisclosure } from '@chakra-ui/react';
import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api';
import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api';
import dynamic from 'next/dynamic';
import { usePagination } from '@fastgpt/web/hooks/usePagination';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
import { useRouter } from 'next/router';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig'));
type CollectionPageContextType = {
openWebSyncConfirm: () => void;
onOpenWebsiteModal: () => void;
collections: DatasetCollectionsListItemType[];
Pagination: () => JSX.Element;
total: number;
getData: (e: number) => void;
isGetting: boolean;
pageNum: number;
pageSize: number;
searchText: string;
setSearchText: Dispatch<SetStateAction<string>>;
};
export const CollectionPageContext = createContext<CollectionPageContextType>({
openWebSyncConfirm: function (): () => void {
throw new Error('Function not implemented.');
},
onOpenWebsiteModal: function (): void {
throw new Error('Function not implemented.');
},
collections: [],
Pagination: function (): JSX.Element {
throw new Error('Function not implemented.');
},
total: 0,
getData: function (e: number): void {
throw new Error('Function not implemented.');
},
isGetting: false,
pageNum: 0,
pageSize: 0,
searchText: '',
setSearchText: function (value: SetStateAction<string>): void {
throw new Error('Function not implemented.');
}
});
const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => {
const { t } = useTranslation();
const router = useRouter();
const { parentId = '' } = router.query as { parentId: string };
const { datasetDetail, datasetId, updateDataset } = useContextSelector(
DatasetPageContext,
(v) => v
);
// website config
const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({
content: t('core.dataset.collection.Start Sync Tip')
});
const {
isOpen: isOpenWebsiteModal,
onOpen: onOpenWebsiteModal,
onClose: onCloseWebsiteModal
} = useDisclosure();
const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
onCloseWebsiteModal();
await checkTeamWebSyncLimit();
const billId = await postCreateTrainingUsage({
name: t('core.dataset.training.Website Sync'),
datasetId: datasetId
});
await postWebsiteSync({ datasetId: datasetId, billId });
await updateDataset({
id: datasetId,
websiteConfig,
status: DatasetStatusEnum.syncing
});
return;
},
errorToast: t('common.Update Failed')
});
// collection list
const [searchText, setSearchText] = useState('');
const {
data: collections,
Pagination,
total,
getData,
isLoading: isGetting,
pageNum,
pageSize
} = usePagination<DatasetCollectionsListItemType>({
api: getDatasetCollections,
pageSize: 20,
params: {
datasetId,
parentId,
searchText
},
defaultRequest: false
});
useEffect(() => {
getData(1);
}, [parentId]);
const contextValue: CollectionPageContextType = {
openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig),
onOpenWebsiteModal,
searchText,
setSearchText,
collections,
Pagination,
total,
getData,
isGetting,
pageNum,
pageSize
};
return (
<CollectionPageContext.Provider value={contextValue}>
{children}
{datasetDetail.type === DatasetTypeEnum.websiteDataset && (
<>
{isOpenWebsiteModal && (
<WebSiteConfigModal
onClose={onCloseWebsiteModal}
onSuccess={onUpdateDatasetWebsiteConfig}
defaultValue={{
url: datasetDetail?.websiteConfig?.url,
selector: datasetDetail?.websiteConfig?.selector
}}
/>
)}
<ConfirmWebSyncModal />
</>
)}
</CollectionPageContext.Provider>
);
};
export default CollectionPageContextProvider;
projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx 0 → 100644
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import React from 'react';
import { useTranslation } from 'next-i18next';
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Box, Flex } from '@chakra-ui/react';
import { useContextSelector } from 'use-context-selector';
import { CollectionPageContext } from './Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
const EmptyCollectionTip = () => {
const { t } = useTranslation();
const onOpenWebsiteModal = useContextSelector(CollectionPageContext, (v) => v.onOpenWebsiteModal);
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
return (
<>
{(datasetDetail.type === DatasetTypeEnum.dataset ||
datasetDetail.type === DatasetTypeEnum.externalFile) && (
<EmptyTip text={t('core.dataset.collection.Empty Tip')} />
)}
{datasetDetail.type === DatasetTypeEnum.websiteDataset && (
<EmptyTip
text={
<Flex>
{datasetDetail.status === DatasetStatusEnum.syncing && (
<>{t('core.dataset.status.syncing')}</>
)}
{datasetDetail.status === DatasetStatusEnum.active && (
<>
{!datasetDetail?.websiteConfig?.url ? (
<>
{t('core.dataset.collection.Website Empty Tip')}
{', '}
<Box
textDecoration={'underline'}
cursor={'pointer'}
onClick={onOpenWebsiteModal}
>
{t('core.dataset.collection.Click top config website')}
</Box>
</>
) : (
<>{t('core.dataset.website.UnValid Website Tip')}</>
)}
</>
)}
</Flex>
}
/>
)}
</>
);
};
export default EmptyCollectionTip;
projects/app/src/pages/dataset/detail/components/Import/Context.tsx 0 → 100644
import { useRouter } from 'next/router';
import { SetStateAction, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext, useContextSelector } from 'use-context-selector';
import { ImportDataSourceEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { useMyStep } from '@fastgpt/web/hooks/useStep';
import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { TabEnum } from '../Slider';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import { UseFormReturn, useForm } from 'react-hook-form';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
type TrainingFiledType = {
chunkOverlapRatio: number;
maxChunkSize: number;
minChunkSize: number;
autoChunkSize: number;
chunkSize: number;
showChunkInput: boolean;
showPromptInput: boolean;
charsPointsPrice: number;
priceTip: string;
uploadRate: number;
chunkSizeField?: ChunkSizeFieldType;
};
type DatasetImportContextType = {
importSource: ImportDataSourceEnum;
parentId: string | undefined;
activeStep: number;
goToNext: () => void;
processParamsForm: UseFormReturn<ImportFormType, any>;
sources: ImportSourceItemType[];
setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
} & TrainingFiledType;
type ChunkSizeFieldType = 'embeddingChunkSize';
export type ImportFormType = {
mode: TrainingModeEnum;
way: ImportProcessWayEnum;
embeddingChunkSize: number;
customSplitChar: string;
qaPrompt: string;
webSelector: string;
};
export const DatasetImportContext = createContext<DatasetImportContextType>({
importSource: ImportDataSourceEnum.fileLocal,
goToNext: function (): void {
throw new Error('Function not implemented.');
},
activeStep: 0,
parentId: undefined,
maxChunkSize: 0,
minChunkSize: 0,
showChunkInput: false,
showPromptInput: false,
sources: [],
setSources: function (value: SetStateAction<ImportSourceItemType[]>): void {
throw new Error('Function not implemented.');
},
chunkSize: 0,
chunkOverlapRatio: 0,
uploadRate: 0,
//@ts-ignore
processParamsForm: undefined,
autoChunkSize: 0,
charsPointsPrice: 0,
priceTip: ''
});
const DatasetImportContextProvider = ({ children }: { children: React.ReactNode }) => {
const { t } = useTranslation();
const router = useRouter();
const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as {
source: ImportDataSourceEnum;
parentId?: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
// step
const modeSteps: Record<ImportDataSourceEnum, { title: string }[]> = {
[ImportDataSourceEnum.fileLocal]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileLink]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileCustom]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.csvTable]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.externalFile]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
]
};
const steps = modeSteps[source];
const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({
defaultStep: 0,
steps
});
// -----
const vectorModel = datasetDetail.vectorModel;
const agentModel = datasetDetail.agentModel;
const processParamsForm = useForm<ImportFormType>({
defaultValues: {
mode: TrainingModeEnum.chunk,
way: ImportProcessWayEnum.auto,
embeddingChunkSize: vectorModel?.defaultToken || 512,
customSplitChar: '',
qaPrompt: Prompt_AgentQA.description,
webSelector: ''
}
});
const [sources, setSources] = useState<ImportSourceItemType[]>([]);
// watch form
const mode = processParamsForm.watch('mode');
const way = processParamsForm.watch('way');
const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
const customSplitChar = processParamsForm.watch('customSplitChar');
const modeStaticParams: Record<TrainingModeEnum, TrainingFiledType> = {
[TrainingModeEnum.auto]: {
chunkOverlapRatio: 0.2,
maxChunkSize: 2048,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
showChunkInput: false,
showPromptInput: false,
charsPointsPrice: agentModel.charsPointsPrice,
priceTip: t('core.dataset.import.Auto mode Estimated Price Tips', {
price: agentModel.charsPointsPrice
}),
uploadRate: 100
},
[TrainingModeEnum.chunk]: {
chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType,
chunkOverlapRatio: 0.2,
maxChunkSize: vectorModel?.maxToken || 512,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken || 512,
chunkSize: embeddingChunkSize,
showChunkInput: true,
showPromptInput: false,
charsPointsPrice: vectorModel.charsPointsPrice,
priceTip: t('core.dataset.import.Embedding Estimated Price Tips', {
price: vectorModel.charsPointsPrice
}),
uploadRate: 150
},
[TrainingModeEnum.qa]: {
chunkOverlapRatio: 0,
maxChunkSize: 8000,
minChunkSize: 3000,
autoChunkSize: agentModel.maxContext * 0.55 || 6000,
chunkSize: agentModel.maxContext * 0.55 || 6000,
showChunkInput: false,
showPromptInput: true,
charsPointsPrice: agentModel.charsPointsPrice,
priceTip: t('core.dataset.import.QA Estimated Price Tips', {
price: agentModel?.charsPointsPrice
}),
uploadRate: 30
}
};
const selectModelStaticParam = modeStaticParams[mode];
const wayStaticPrams = {
[ImportProcessWayEnum.auto]: {
chunkSize: selectModelStaticParam.autoChunkSize,
customSplitChar: ''
},
[ImportProcessWayEnum.custom]: {
chunkSize: modeStaticParams[mode].chunkSize,
customSplitChar
}
};
const chunkSize = wayStaticPrams[way].chunkSize;
const contextValue = {
importSource: source,
parentId,
activeStep,
goToNext,
processParamsForm,
...selectModelStaticParam,
sources,
setSources,
chunkSize
};
return (
<DatasetImportContext.Provider value={contextValue}>
<Flex>
{activeStep === 0 ? (
<Flex alignItems={'center'}>
<IconButton
icon={<MyIcon name={'common/backFill'} w={'14px'} />}
aria-label={''}
size={'smSquare'}
w={'26px'}
h={'26px'}
borderRadius={'50%'}
variant={'whiteBase'}
mr={2}
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.collectionCard
}
})
}
/>
{t('common.Exit')}
</Flex>
) : (
<Button
variant={'whiteBase'}
leftIcon={<MyIcon name={'common/backFill'} w={'14px'} />}
onClick={goToPrevious}
>
{t('common.Last Step')}
</Button>
)}
<Box flex={1} />
</Flex>
{/* step */}
<Box
mt={4}
mb={5}
px={3}
py={[2, 4]}
bg={'myGray.50'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
borderRadius={'md'}
>
<Box maxW={['100%', '900px']} mx={'auto'}>
<MyStep />
</Box>
</Box>
{children}
</DatasetImportContext.Provider>
);
};
export default DatasetImportContextProvider;
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx deleted 100644 → 0
import React, { useContext, createContext, useState, useMemo, useEffect } from 'react';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import { UseFormReturn, useForm } from 'react-hook-form';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
type ChunkSizeFieldType = 'embeddingChunkSize';
export type FormType = {
mode: `${TrainingModeEnum}`;
way: `${ImportProcessWayEnum}`;
embeddingChunkSize: number;
customSplitChar: string;
qaPrompt: string;
webSelector: string;
};
type useImportStoreType = {
parentId?: string;
processParamsForm: UseFormReturn<FormType, any>;
chunkSizeField?: ChunkSizeFieldType;
maxChunkSize: number;
minChunkSize: number;
showChunkInput: boolean;
showPromptInput: boolean;
sources: ImportSourceItemType[];
setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
chunkSize: number;
chunkOverlapRatio: number;
priceTip: string;
uploadRate: number;
importSource: `${ImportDataSourceEnum}`;
};
const StateContext = createContext<useImportStoreType>({
processParamsForm: {} as any,
sources: [],
setSources: function (value: React.SetStateAction<ImportSourceItemType[]>): void {
throw new Error('Function not implemented.');
},
maxChunkSize: 0,
minChunkSize: 0,
showChunkInput: false,
showPromptInput: false,
chunkSizeField: 'embeddingChunkSize',
chunkSize: 0,
chunkOverlapRatio: 0,
priceTip: '',
uploadRate: 50,
importSource: ImportDataSourceEnum.fileLocal
});
export const useImportStore = () => useContext(StateContext);
const Provider = ({
importSource,
dataset,
parentId,
children
}: {
importSource: `${ImportDataSourceEnum}`;
dataset: DatasetItemType;
parentId?: string;
children: React.ReactNode;
}) => {
const vectorModel = dataset.vectorModel;
const agentModel = dataset.agentModel;
const processParamsForm = useForm<FormType>({
defaultValues: {
mode: TrainingModeEnum.chunk,
way: ImportProcessWayEnum.auto,
embeddingChunkSize: vectorModel?.defaultToken || 512,
customSplitChar: '',
qaPrompt: Prompt_AgentQA.description,
webSelector: ''
}
});
const { t } = useTranslation();
const [sources, setSources] = useState<ImportSourceItemType[]>([]);
// watch form
const mode = processParamsForm.watch('mode');
const way = processParamsForm.watch('way');
const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
const customSplitChar = processParamsForm.watch('customSplitChar');
const modeStaticParams = {
[TrainingModeEnum.auto]: {
chunkOverlapRatio: 0.2,
maxChunkSize: 2048,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
showChunkInput: false,
showPromptInput: false,
charsPointsPrice: agentModel.charsPointsPrice,
priceTip: t('core.dataset.import.Auto mode Estimated Price Tips', {
price: agentModel.charsPointsPrice
}),
uploadRate: 100
},
[TrainingModeEnum.chunk]: {
chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType,
chunkOverlapRatio: 0.2,
maxChunkSize: vectorModel?.maxToken || 512,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken || 512,
chunkSize: embeddingChunkSize,
showChunkInput: true,
showPromptInput: false,
charsPointsPrice: vectorModel.charsPointsPrice,
priceTip: t('core.dataset.import.Embedding Estimated Price Tips', {
price: vectorModel.charsPointsPrice
}),
uploadRate: 150
},
[TrainingModeEnum.qa]: {
chunkOverlapRatio: 0,
maxChunkSize: 8000,
minChunkSize: 3000,
autoChunkSize: agentModel.maxContext * 0.55 || 6000,
chunkSize: agentModel.maxContext * 0.55 || 6000,
showChunkInput: false,
showPromptInput: true,
charsPointsPrice: agentModel.charsPointsPrice,
priceTip: t('core.dataset.import.QA Estimated Price Tips', {
price: agentModel?.charsPointsPrice
}),
uploadRate: 30
}
};
const selectModelStaticParam = useMemo(() => modeStaticParams[mode], [mode]);
const wayStaticPrams = {
[ImportProcessWayEnum.auto]: {
chunkSize: selectModelStaticParam.autoChunkSize,
customSplitChar: ''
},
[ImportProcessWayEnum.custom]: {
chunkSize: modeStaticParams[mode].chunkSize,
customSplitChar
}
};
const chunkSize = wayStaticPrams[way].chunkSize;
const value: useImportStoreType = {
parentId,
processParamsForm,
...selectModelStaticParam,
sources,
setSources,
chunkSize,
importSource
};
return <StateContext.Provider value={value}>{children}</StateContext.Provider>;
};
export default React.memo(Provider);
projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx
......@@ -20,23 +20,20 @@ import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import MyTooltip from '@/components/MyTooltip';
import { useImportStore } from '../Provider';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import Preview from '../components/Preview';
import Tag from '@fastgpt/web/components/common/Tag/index';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
function DataProcess({
showPreviewChunks = true,
goToNext
}: {
showPreviewChunks: boolean;
goToNext: () => void;
}) {
function DataProcess({ showPreviewChunks = true }: { showPreviewChunks: boolean }) {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const {
goToNext,
processParamsForm,
chunkSizeField,
minChunkSize,
......@@ -44,7 +41,7 @@ function DataProcess({
showPromptInput,
maxChunkSize,
priceTip
} = useImportStore();
} = useContextSelector(DatasetImportContext, (v) => v);
const { getValues, setValue, register } = processParamsForm;
const [refresh, setRefresh] = useState(false);
......
projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx
......@@ -2,15 +2,12 @@ import React from 'react';
import Preview from '../components/Preview';
import { Box, Button, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewData = ({
showPreviewChunks,
goToNext
}: {
showPreviewChunks: boolean;
goToNext: () => void;
}) => {
const PreviewData = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
const { t } = useTranslation();
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
return (
<Flex flexDirection={'column'} h={'100%'}>
......
projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx
......@@ -11,7 +11,6 @@ import {
Flex,
Button
} from '@chakra-ui/react';
import { useImportStore, type FormType } from '../Provider';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
......@@ -28,20 +27,23 @@ import {
} from '@/web/core/dataset/api';
import Tag from '@fastgpt/web/components/common/Tag/index';
import { useI18n } from '@/web/context/I18n';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { DatasetImportContext, type ImportFormType } from '../Context';
const Upload = () => {
const { t } = useTranslation();
const { fileT } = useI18n();
const { toast } = useToast();
const router = useRouter();
const { datasetDetail } = useDatasetStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { importSource, parentId, sources, setSources, processParamsForm, chunkSize } =
useImportStore();
useContextSelector(DatasetImportContext, (v) => v);
const { handleSubmit } = processParamsForm;
const { mutate: startUpload, isLoading } = useRequest({
mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: FormType) => {
mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: ImportFormType) => {
if (sources.length === 0) return;
const filterWaitingSources = sources.filter((item) => item.createStatus === 'waiting');
......
projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx
......@@ -10,7 +10,7 @@ import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
const FileModeSelector = ({ onClose }: { onClose: () => void }) => {
const { t } = useTranslation();
const router = useRouter();
const [value, setValue] = useState<`${ImportDataSourceEnum}`>(ImportDataSourceEnum.fileLocal);
const [value, setValue] = useState<ImportDataSourceEnum>(ImportDataSourceEnum.fileLocal);
return (
<MyModal
......
projects/app/src/pages/dataset/detail/components/Import/components/Preview.tsx
......@@ -3,17 +3,18 @@ import { Box, Flex, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import { useImportStore } from '../Provider';
import MyMenu from '@fastgpt/web/components/common/MyMenu';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import dynamic from 'next/dynamic';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewRawText = dynamic(() => import('./PreviewRawText'));
const PreviewChunks = dynamic(() => import('./PreviewChunks'));
const Preview = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
const { t } = useTranslation();
const { sources } = useImportStore();
const { sources } = useContextSelector(DatasetImportContext, (v) => v);
const [previewRawTextSource, setPreviewRawTextSource] = useState<ImportSourceItemType>();
const [previewChunkSource, setPreviewChunkSource] = useState<ImportSourceItemType>();
......
projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx
......@@ -4,11 +4,12 @@ import { ImportSourceItemType } from '@/web/core/dataset/type';
import { useQuery } from '@tanstack/react-query';
import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
import { getPreviewChunks } from '@/web/core/dataset/api';
import { useImportStore } from '../Provider';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewChunks = ({
previewSource,
......@@ -18,7 +19,10 @@ const PreviewChunks = ({
onClose: () => void;
}) => {
const { toast } = useToast();
const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useImportStore();
const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useContextSelector(
DatasetImportContext,
(v) => v
);
const { data = [], isLoading } = useQuery(
['previewSource'],
......
projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx
......@@ -4,10 +4,11 @@ import { ImportSourceItemType } from '@/web/core/dataset/type';
import { useQuery } from '@tanstack/react-query';
import { getPreviewFileContent } from '@/web/common/file/api';
import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
import { useImportStore } from '../Provider';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewRawText = ({
previewSource,
......@@ -17,7 +18,7 @@ const PreviewRawText = ({
onClose: () => void;
}) => {
const { toast } = useToast();
const { importSource } = useImportStore();
const { importSource } = useContextSelector(DatasetImportContext, (v) => v);
const { data, isLoading } = useQuery(
['previewSource', previewSource?.dbFileId],
......
projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx 0 → 100644
import React, { useEffect } from 'react';
import dynamic from 'next/dynamic';
import { useTranslation } from 'next-i18next';
import { useFieldArray, useForm } from 'react-hook-form';
import {
Box,
Button,
Flex,
Table,
Thead,
Tbody,
Tr,
Th,
Td,
TableContainer,
Input
} from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { useI18n } from '@/web/context/I18n';
import { SmallAddIcon } from '@chakra-ui/icons';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const ExternalFileCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomLinkInput />}
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(ExternalFileCollection);
const CustomLinkInput = () => {
const { t } = useTranslation();
const { datasetT, commonT } = useI18n();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const { register, reset, handleSubmit, control } = useForm<{
list: {
sourceName: string;
sourceUrl: string;
externalId: string;
}[];
}>({
defaultValues: {
list: [
{
sourceName: '',
sourceUrl: '',
externalId: ''
}
]
}
});
const {
fields: list,
append,
remove,
update
} = useFieldArray({
control,
name: 'list'
});
useEffect(() => {
if (sources.length > 0) {
reset({
list: sources.map((item) => ({
sourceName: item.sourceName,
sourceUrl: item.sourceUrl || '',
externalId: item.externalId || ''
}))
});
}
}, []);
return (
<Box>
<TableContainer>
<Table bg={'white'}>
<Thead>
<Tr bg={'myGray.50'}>
<Th>{datasetT('External url')}</Th>
<Th>{datasetT('External id')}</Th>
<Th>{datasetT('filename')}</Th>
<Th></Th>
</Tr>
</Thead>
<Tbody>
{list.map((item, index) => (
<Tr key={item.id}>
<Td>
<Input
{...register(`list.${index}.sourceUrl`, {
required: index !== list.length - 1,
onBlur(e) {
const val = (e.target.value || '') as string;
if (val.includes('.') && !list[index]?.sourceName) {
const sourceName = val.split('/').pop() || '';
update(index, {
...list[index],
sourceUrl: val,
sourceName: decodeURIComponent(sourceName)
});
}
if (val && index === list.length - 1) {
append({
sourceName: '',
sourceUrl: '',
externalId: ''
});
}
}
})}
/>
</Td>
<Td>
<Input {...register(`list.${index}.externalId`)} />
</Td>
<Td>
<Input {...register(`list.${index}.sourceName`)} />
</Td>
<Td>
<MyIcon
name={'delete'}
w={'16px'}
cursor={'pointer'}
_hover={{ color: 'red.600' }}
onClick={() => remove(index)}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<Flex mt={5} justifyContent={'space-between'}>
<Button
variant={'whitePrimary'}
leftIcon={<SmallAddIcon />}
onClick={() => {
append({
sourceName: '',
sourceUrl: '',
externalId: ''
});
}}
>
{commonT('Add new')}
</Button>
<Button
isDisabled={list.length === 0}
onClick={handleSubmit((data) => {
setSources(
data.list
.filter((item) => !!item.sourceUrl)
.map((item) => ({
id: getNanoid(32),
createStatus: 'waiting',
sourceName: item.sourceName || item.sourceUrl,
icon: getFileIcon(item.sourceUrl),
externalId: item.externalId,
sourceUrl: item.sourceUrl
}))
);
goToNext();
})}
>
{t('common.Next Step')}
</Button>
</Flex>
</Box>
);
};
projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx
import React, { useCallback, useEffect } from 'react';
import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
import dynamic from 'next/dynamic';
import { useImportStore } from '../Provider';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => {
const CustomTet = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomTextInput goToNext={goToNext} />}
{activeStep === 1 && <DataProcess showPreviewChunks goToNext={goToNext} />}
{activeStep === 0 && <CustomTextInput />}
{activeStep === 1 && <DataProcess showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
......@@ -26,9 +27,9 @@ const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => {
export default React.memo(CustomTet);
const CustomTextInput = ({ goToNext }: { goToNext: () => void }) => {
const CustomTextInput = () => {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
const { sources, goToNext, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const { register, reset, handleSubmit } = useForm({
defaultValues: {
name: '',
......
projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx
import React, { useEffect } from 'react';
import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
import dynamic from 'next/dynamic';
import { useImportStore } from '../Provider';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react';
......@@ -12,17 +9,21 @@ import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { getDocPath } from '@/web/common/system/doc';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => {
const LinkCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomLinkImport goToNext={goToNext} />}
{activeStep === 1 && <DataProcess showPreviewChunks={false} goToNext={goToNext} />}
{activeStep === 0 && <CustomLinkImport />}
{activeStep === 1 && <DataProcess showPreviewChunks={false} />}
{activeStep === 2 && <Upload />}
</>
);
......@@ -30,10 +31,13 @@ const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => {
export default React.memo(LinkCollection);
const CustomLinkImport = ({ goToNext }: { goToNext: () => void }) => {
const CustomLinkImport = () => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { sources, setSources, processParamsForm } = useImportStore();
const { goToNext, sources, setSources, processParamsForm } = useContextSelector(
DatasetImportContext,
(v) => v
);
const { register, reset, handleSubmit, watch } = useForm({
defaultValues: {
link: ''
......
projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { ImportDataComponentProps, ImportSourceItemType } from '@/web/core/dataset/type.d';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import { Box, Button } from '@chakra-ui/react';
import FileSelector from '../components/FileSelector';
import { useTranslation } from 'next-i18next';
import { useImportStore } from '../Provider';
import dynamic from 'next/dynamic';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { RenderUploadFiles } from '../components/RenderFiles';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
......@@ -16,11 +17,13 @@ const Upload = dynamic(() => import('../commonProgress/Upload'));
const fileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
const FileLocal = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <SelectFile goToNext={goToNext} />}
{activeStep === 1 && <DataProcess showPreviewChunks goToNext={goToNext} />}
{activeStep === 0 && <SelectFile />}
{activeStep === 1 && <DataProcess showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
......@@ -28,9 +31,9 @@ const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
export default React.memo(FileLocal);
const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) {
const SelectFile = React.memo(function SelectFile() {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
sources.map((source) => ({
isUploading: false,
......
projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx
import React, { useEffect, useMemo, useState } from 'react';
import { ImportDataComponentProps, ImportSourceItemType } from '@/web/core/dataset/type.d';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import { Box, Button } from '@chakra-ui/react';
import FileSelector from '../components/FileSelector';
import { useTranslation } from 'next-i18next';
import { useImportStore } from '../Provider';
import dynamic from 'next/dynamic';
import { fileDownload } from '@/web/common/file/utils';
import { RenderUploadFiles } from '../components/RenderFiles';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const Upload = dynamic(() => import('../commonProgress/Upload'));
const fileType = '.csv';
const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
const FileLocal = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <SelectFile goToNext={goToNext} />}
{activeStep === 1 && <PreviewData showPreviewChunks goToNext={goToNext} />}
{activeStep === 0 && <SelectFile />}
{activeStep === 1 && <PreviewData showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
......@@ -32,9 +35,9 @@ const csvTemplate = `"第一列内容","第二列内容"
"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) {
const SelectFile = React.memo(function SelectFile() {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
sources.map((source) => ({
isUploading: false,
......
projects/app/src/pages/dataset/detail/components/Import/index.tsx
import React, { useMemo } from 'react';
import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import { TabEnum } from '../../index';
import { useMyStep } from '@fastgpt/web/hooks/useStep';
import { Box, Flex } from '@chakra-ui/react';
import dynamic from 'next/dynamic';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import Provider from './Provider';
import { useContextSelector } from 'use-context-selector';
import DatasetImportContextProvider, { DatasetImportContext } from './Context';
const FileLocal = dynamic(() => import('./diffSource/FileLocal'));
const FileLink = dynamic(() => import('./diffSource/FileLink'));
const FileCustomText = dynamic(() => import('./diffSource/FileCustomText'));
const TableLocal = dynamic(() => import('./diffSource/TableLocal'));
const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile'));
const ImportDataset = () => {
const { t } = useTranslation();
const router = useRouter();
const { datasetDetail } = useDatasetStore();
const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as {
source: `${ImportDataSourceEnum}`;
parentId?: string;
};
const modeSteps: Record<`${ImportDataSourceEnum}`, { title: string }[]> = {
[ImportDataSourceEnum.fileLocal]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileLink]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileCustom]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.csvTable]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
]
};
const steps = modeSteps[source];
const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({
defaultStep: 0,
steps
});
const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource);
const ImportComponent = useMemo(() => {
if (source === ImportDataSourceEnum.fileLocal) return FileLocal;
if (source === ImportDataSourceEnum.fileLink) return FileLink;
if (source === ImportDataSourceEnum.fileCustom) return FileCustomText;
if (source === ImportDataSourceEnum.csvTable) return TableLocal;
}, [source]);
if (importSource === ImportDataSourceEnum.fileLocal) return FileLocal;
if (importSource === ImportDataSourceEnum.fileLink) return FileLink;
if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText;
if (importSource === ImportDataSourceEnum.csvTable) return TableLocal;
if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection;
}, [importSource]);
return ImportComponent ? (
<Box flex={'1 0 0'} overflow={'auto'} position={'relative'}>
<ImportComponent />
</Box>
) : null;
};
const Render = () => {
return (
<Flex flexDirection={'column'} bg={'white'} h={'100%'} px={[2, 9]} py={[2, 5]}>
<Flex>
{activeStep === 0 ? (
<Flex alignItems={'center'}>
<IconButton
icon={<MyIcon name={'common/backFill'} w={'14px'} />}
aria-label={''}
size={'smSquare'}
w={'26px'}
h={'26px'}
borderRadius={'50%'}
variant={'whiteBase'}
mr={2}
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.collectionCard
}
})
}
/>
{t('common.Exit')}
</Flex>
) : (
<Button
variant={'whiteBase'}
leftIcon={<MyIcon name={'common/backFill'} w={'14px'} />}
onClick={goToPrevious}
>
{t('common.Last Step')}
</Button>
)}
<Box flex={1} />
</Flex>
{/* step */}
<Box
mt={4}
mb={5}
px={3}
py={[2, 4]}
bg={'myGray.50'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
borderRadius={'md'}
>
<Box maxW={['100%', '900px']} mx={'auto'}>
<MyStep />
</Box>
</Box>
<Provider dataset={datasetDetail} parentId={parentId} importSource={source}>
<Box flex={'1 0 0'} overflow={'auto'} position={'relative'}>
<ImportComponent activeStep={activeStep} goToNext={goToNext} />
</Box>
</Provider>
<DatasetImportContextProvider>
<ImportDataset />
</DatasetImportContextProvider>
</Flex>
) : null;
);
};
export default React.memo(ImportDataset);
export default React.memo(Render);
projects/app/src/pages/dataset/detail/components/Info.tsx
......@@ -23,13 +23,14 @@ import type { VectorModelItemType } from '@fastgpt/global/core/ai/model.d';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import MyDivider from '@fastgpt/web/components/common/MyDivider/index';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
const Info = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const { datasetT } = useI18n();
const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore();
const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount);
const trainingCount = useContextSelector(DatasetPageContext, (v) => v.trainingCount);
const { datasetDetail, loadDatasetDetail, updateDataset, rebuildingCount, trainingCount } =
useContextSelector(DatasetPageContext, (v) => v);
const refetchDatasetTraining = useContextSelector(
DatasetPageContext,
(v) => v.refetchDatasetTraining
......@@ -82,9 +83,6 @@ const Info = ({ datasetId }: { datasetId: string }) => {
...data
});
},
onSuccess() {
loadDatasets();
},
successToast: t('common.Update Success'),
errorToast: t('common.Update Failed')
});
......@@ -117,7 +115,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
},
onSuccess() {
refetchDatasetTraining();
loadDatasetDetail(datasetId, true);
loadDatasetDetail(datasetId);
},
successToast: datasetT('Rebuild embedding start tip'),
errorToast: t('common.Update Failed')
......@@ -128,16 +126,16 @@ const Info = ({ datasetId }: { datasetId: string }) => {
return (
<Box py={5} px={[5, 10]}>
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.dataset.Dataset ID')}
</Box>
<Box flex={1}>{datasetDetail._id}</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'} flexWrap={'wrap'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Vector Model')}
</Box>
<Box flex={[1, '0 0 300px']}>
<Box flex={[1, '0 0 320px']}>
<AIModelSelector
w={'100%'}
value={vectorModel.model}
......@@ -162,16 +160,16 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.Max Token')}
</Box>
<Box flex={[1, '0 0 300px']}>{vectorModel.maxToken}</Box>
<Box flex={[1, '0 0 320px']}>{vectorModel.maxToken}</Box>
</Flex>
<Flex mt={6} alignItems={'center'} flexWrap={'wrap'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Dataset Agent Model')}
</Box>
<Box flex={[1, '0 0 300px']}>
<Box flex={[1, '0 0 320px']}>
<AIModelSelector
w={'100%'}
value={agentModel.model}
......@@ -188,13 +186,29 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
</Flex>
<MyDivider my={4} h={'2px'} maxW={'500px'} />
<MyDivider my={6} h={'2px'} maxW={'500px'} />
{datasetDetail.type === DatasetTypeEnum.externalFile && (
<>
<Flex w={'100%'} alignItems={'center'}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{datasetT('External read url')}
</Box>
<Input
flex={[1, '0 0 320px']}
placeholder="https://test.com/read?fileId={{fileId}}"
{...register('externalReadUrl')}
/>
</Flex>
<MyDivider my={6} h={'2px'} maxW={'500px'} />
</>
)}
<Flex mt={5} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.dataset.Avatar')}
</Box>
<Box flex={[1, '0 0 300px']}>
<Box flex={[1, '0 0 320px']}>
<MyTooltip label={t('common.avatar.Select Avatar')}>
<Avatar
m={'auto'}
......@@ -208,18 +222,20 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.dataset.Name')}
</Box>
<Input flex={[1, '0 0 300px']} maxLength={30} {...register('name')} />
<Input flex={[1, '0 0 320px']} maxLength={30} {...register('name')} />
</Flex>
<Flex mt={8} alignItems={'center'} w={'100%'}>
<Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
<Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']}>
{t('common.Intro')}
</Box>
<Textarea flex={[1, '0 0 320px']} {...register('intro')} placeholder={t('common.Intro')} />
</Flex>
{datasetDetail.isOwner && (
<Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}>
{t('user.Permission')}
</Box>
<Box>
......@@ -234,7 +250,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
)}
<Flex mt={5} w={'100%'} alignItems={'flex-end'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}></Box>
<Box fontSize={['sm', 'md']} flex={['0 0 90px', '0 0 160px']} w={0}></Box>
<Button
isLoading={btnLoading}
mr={4}
......
projects/app/src/pages/dataset/detail/components/Slider.tsx
......@@ -5,17 +5,10 @@ import { useUserStore } from '@/web/support/user/useUserStore';
import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import Avatar from '@/components/Avatar';
import {
DatasetStatusEnum,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constants';
import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import SideTabs from '@/components/SideTabs';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useRouter } from 'next/router';
import Tabs from '@/components/Tabs';
import { useContextSelector } from 'use-context-selector';
......@@ -36,12 +29,10 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
const { datasetT } = useI18n();
const router = useRouter();
const query = router.query;
const { datasetDetail, startWebsiteSync } = useDatasetStore();
const { userInfo } = useUserStore();
const { isPc, setLoading } = useSystemStore();
const vectorTrainingMap = useContextSelector(DatasetPageContext, (v) => v.vectorTrainingMap);
const agentTrainingMap = useContextSelector(DatasetPageContext, (v) => v.agentTrainingMap);
const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount);
const { isPc } = useSystemStore();
const { datasetDetail, vectorTrainingMap, agentTrainingMap, rebuildingCount } =
useContextSelector(DatasetPageContext, (v) => v);
const tabList = [
{
......@@ -67,20 +58,6 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
[query, router]
);
const { ConfirmModal: ConfirmSyncModal, openConfirm: openConfirmSync } = useConfirm({
type: 'common'
});
const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
mutationFn: () => {
setLoading(true);
return startWebsiteSync();
},
onSettled() {
setLoading(false);
},
errorToast: t('common.Update Failed')
});
return (
<>
{isPc ? (
......@@ -101,25 +78,6 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
{DatasetTypeMap[datasetDetail.type] && (
<Flex alignItems={'center'} pl={2} justifyContent={'space-between'}>
<DatasetTypeTag type={datasetDetail.type} />
{datasetDetail.type === DatasetTypeEnum.websiteDataset &&
datasetDetail.status === DatasetStatusEnum.active && (
<MyTooltip label={t('core.dataset.website.Start Sync')}>
<MyIcon
mt={1}
name={'common/refreshLight'}
w={'12px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={() =>
openConfirmSync(
onUpdateDatasetWebsiteConfig,
undefined,
t('core.dataset.website.Confirm Create Tips')
)()
}
/>
</MyTooltip>
)}
</Flex>
)}
</Box>
......@@ -206,8 +164,6 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
/>
</Box>
)}
<ConfirmSyncModal />
</>
);
};
......
projects/app/src/pages/dataset/detail/components/Test.tsx
......@@ -25,6 +25,8 @@ import { fileDownload } from '@/web/common/file/utils';
import QuoteItem from '@/components/core/dataset/QuoteItem';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
......@@ -48,7 +50,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
const theme = useTheme();
const { toast } = useToast();
const { llmModelList } = useSystemStore();
const { datasetDetail } = useDatasetStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { pushDatasetTestItem } = useSearchTestStore();
const [inputType, setInputType] = useState<'text' | 'file'>('text');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
......
projects/app/src/pages/dataset/detail/index.tsx
import React, { useCallback, useMemo } from 'react';
import React from 'react';
import { useRouter } from 'next/router';
import { Box } from '@chakra-ui/react';
import { useToast } from '@fastgpt/web/hooks/useToast';
......@@ -9,15 +9,17 @@ import PageContainer from '@/components/PageContainer';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useTranslation } from 'next-i18next';
import CollectionCard from './components/CollectionCard';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import Head from 'next/head';
import Slider from './components/Slider';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { DatasetPageContextProvider } from '@/web/core/dataset/context/datasetPageContext';
import {
DatasetPageContext,
DatasetPageContextProvider
} from '@/web/core/dataset/context/datasetPageContext';
import CollectionPageContextProvider from './components/CollectionCard/Context';
import { useContextSelector } from 'use-context-selector';
import NextHead from '@/components/common/NextHead';
const CollectionCard = dynamic(() => import('./components/CollectionCard/index'));
const DataCard = dynamic(() => import('./components/DataCard'));
const Test = dynamic(() => import('./components/Test'));
const Info = dynamic(() => import('./components/Info'));
......@@ -30,16 +32,14 @@ export enum TabEnum {
info = 'info',
import = 'import'
}
type Props = { datasetId: string; currentTab: TabEnum };
const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: TabEnum }) => {
const Detail = ({ datasetId, currentTab }: Props) => {
const { t } = useTranslation();
const { toast } = useToast();
const router = useRouter();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const { ConfirmModal: ConfirmSyncModal, openConfirm: openConfirmSync } = useConfirm({
type: 'common'
});
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const loadDatasetDetail = useContextSelector(DatasetPageContext, (v) => v.loadDatasetDetail);
useQuery([datasetId], () => loadDatasetDetail(datasetId), {
onError(err: any) {
......@@ -53,36 +53,37 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: TabE
return (
<>
<Head>
<title>{datasetDetail?.name}</title>
</Head>
<DatasetPageContextProvider
value={{
datasetId
}}
>
<PageContainer>
<MyBox display={'flex'} flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
<Slider currentTab={currentTab} />
<NextHead title={datasetDetail?.name} icon={datasetDetail?.avatar} />
<PageContainer>
<MyBox display={'flex'} flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
<Slider currentTab={currentTab} />
{!!datasetDetail._id && (
<Box flex={'1 0 0'} pb={0}>
{currentTab === TabEnum.collectionCard && <CollectionCard />}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && <Info datasetId={datasetId} />}
{currentTab === TabEnum.import && <Import />}
</Box>
)}
</MyBox>
</PageContainer>
</DatasetPageContextProvider>
<ConfirmSyncModal />
{!!datasetDetail._id && (
<Box flex={'1 0 0'} pb={0}>
{currentTab === TabEnum.collectionCard && (
<CollectionPageContextProvider>
<CollectionCard />
</CollectionPageContextProvider>
)}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && <Info datasetId={datasetId} />}
{currentTab === TabEnum.import && <Import />}
</Box>
)}
</MyBox>
</PageContainer>
</>
);
};
const Render = (data: Props) => (
<DatasetPageContextProvider datasetId={data.datasetId}>
<Detail {...data} />
</DatasetPageContextProvider>
);
export default Render;
export async function getServerSideProps(context: any) {
const currentTab = context?.query?.currentTab || TabEnum.collectionCard;
const datasetId = context?.query?.datasetId;
......@@ -91,5 +92,3 @@ export async function getServerSideProps(context: any) {
props: { currentTab, datasetId, ...(await serviceSideProps(context, ['dataset', 'file'])) }
};
}
export default React.memo(Detail);
projects/app/src/pages/dataset/list/component/CreateModal.tsx
......@@ -20,9 +20,11 @@ import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import MySelect from '@fastgpt/web/components/common/MySelect';
import AIModelSelector from '@/components/Select/AIModelSelector';
import { useI18n } from '@/web/context/I18n';
const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const { t } = useTranslation();
const { datasetT } = useI18n();
const [refresh, setRefresh] = useState(false);
const { toast } = useToast();
const router = useRouter();
......@@ -102,25 +104,31 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
gridTemplateColumns={'repeat(1,1fr)'}
list={[
{
title: t('core.dataset.Common Dataset'),
title: datasetT('Common Dataset'),
value: DatasetTypeEnum.dataset,
icon: 'core/dataset/commonDataset',
desc: t('core.dataset.Common Dataset Desc')
desc: datasetT('Common Dataset Desc')
},
...(feConfigs.isPlus
? [
{
title: t('core.dataset.Website Dataset'),
title: datasetT('Website Dataset'),
value: DatasetTypeEnum.websiteDataset,
icon: 'core/dataset/websiteDataset',
desc: t('core.dataset.Website Dataset Desc')
desc: datasetT('Website Dataset Desc')
}
// {
// title: datasetT('External File'),
// value: DatasetTypeEnum.externalFile,
// icon: 'core/dataset/websiteDataset',
// desc: datasetT('External file Dataset Desc')
// }
]
: [])
]}
value={getValues('type')}
onChange={(e) => {
setValue('type', e as `${DatasetTypeEnum}`);
setValue('type', e as DatasetTypeEnum);
setRefresh(!refresh);
}}
/>
......
projects/app/src/pages/dataset/list/component/MoveModal.tsx
......@@ -35,7 +35,10 @@ const MoveModal = ({
const [parentId, setParentId] = useState<string>('');
const { data } = useQuery(['getDatasets', parentId], () => {
return Promise.all([getDatasets({ parentId, type: 'folder' }), getDatasetPaths(parentId)]);
return Promise.all([
getDatasets({ parentId, type: DatasetTypeEnum.folder }),
getDatasetPaths(parentId)
]);
});
const paths = useMemo(
() => [
......
projects/app/src/pages/dataset/list/index.tsx
import React, { useMemo, useRef } from 'react';
import React, { useMemo, useRef, useState } from 'react';
import { Box, Flex, Grid, useDisclosure, Image, Button } from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import PageContainer from '@/components/PageContainer';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { AddIcon } from '@chakra-ui/icons';
......@@ -35,28 +34,30 @@ import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { xmlDownloadFetch } from '@/web/common/api/xmlFetch';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
const MoveModal = dynamic(() => import('./component/MoveModal'), { ssr: false });
const Kb = () => {
const Dataset = () => {
const { t } = useTranslation();
const { toast } = useToast();
const router = useRouter();
const { parentId } = router.query as { parentId: string };
const { setLoading } = useSystemStore();
const { userInfo } = useUserStore();
const { myDatasets, loadMyDatasets, setMyDatasets } = useDatasetStore();
const DeleteTipsMap = useRef({
[DatasetTypeEnum.folder]: t('dataset.deleteFolderTips'),
[DatasetTypeEnum.dataset]: t('core.dataset.Delete Confirm'),
[DatasetTypeEnum.websiteDataset]: t('core.dataset.Delete Confirm')
[DatasetTypeEnum.websiteDataset]: t('core.dataset.Delete Confirm'),
[DatasetTypeEnum.externalFile]: t('core.dataset.Delete Confirm')
});
const { openConfirm, ConfirmModal } = useConfirm({
type: 'delete'
});
const { myDatasets, loadDatasets, setDatasets, updateDataset } = useDatasetStore();
const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('Rename')
});
......@@ -78,7 +79,7 @@ const Kb = () => {
return id;
},
onSuccess(id: string) {
setDatasets(myDatasets.filter((item) => item._id !== id));
setMyDatasets(myDatasets.filter((item) => item._id !== id));
},
onSettled() {
setLoading(false);
......@@ -112,7 +113,7 @@ const Kb = () => {
const { data, refetch, isFetching } = useQuery(
['loadDataset', parentId],
() => {
return Promise.all([loadDatasets(parentId), getDatasetPaths(parentId)]);
return Promise.all([loadMyDatasets(parentId), getDatasetPaths(parentId)]);
},
{
onError(err) {
......@@ -139,7 +140,10 @@ const Kb = () => {
);
return (
<PageContainer isLoading={isFetching} insertProps={{ px: [5, '48px'] }}>
<PageContainer
isLoading={myDatasets.length === 0 && isFetching}
insertProps={{ px: [5, '48px'] }}
>
<Flex pt={[4, '30px']} alignItems={'center'} justifyContent={'space-between'}>
{/* url path */}
<ParentPaths
......@@ -317,7 +321,10 @@ const Kb = () => {
defaultVal: dataset.name,
onSuccess: (val) => {
if (val === dataset.name || !val) return;
updateDataset({ id: dataset._id, name: val });
putDatasetById({
id: dataset._id,
name: val
});
}
})
},
......@@ -351,7 +358,7 @@ const Kb = () => {
</Flex>
),
onClick: () => {
updateDataset({
putDatasetById({
id: dataset._id,
permission: PermissionTypeEnum.public
});
......@@ -371,7 +378,7 @@ const Kb = () => {
</Flex>
),
onClick: () => {
updateDataset({
putDatasetById({
id: dataset._id,
permission: PermissionTypeEnum.private
});
......@@ -476,9 +483,9 @@ const Kb = () => {
export async function getServerSideProps(content: any) {
return {
props: {
...(await serviceSideProps(content))
...(await serviceSideProps(content, ['dataset']))
}
};
}
export default Kb;
export default Dataset;
projects/app/src/service/support/wallet/usage/push.ts
......@@ -66,7 +66,7 @@ export const pushQAUsage = async ({
modelType: ModelTypeEnum.llm,
tokens
});
console.log(tokens, '----');
concatUsage({
billId,
teamId,
......
projects/app/src/web/core/dataset/api.ts
import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d';
import type { DatasetItemType, DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import type {
DatasetItemType,
DatasetListItemType,
DatasetSimpleItemType
} from '@fastgpt/global/core/dataset/type.d';
import type {
GetDatasetCollectionsProps,
GetDatasetDataListProps,
......@@ -39,13 +43,13 @@ import type { getDatasetTrainingQueueResponse } from '@/pages/api/core/dataset/t
import type { rebuildEmbeddingBody } from '@/pages/api/core/dataset/training/rebuildEmbedding';
/* ======================== dataset ======================= */
export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
export const getDatasets = (data: { parentId?: string; type?: DatasetTypeEnum }) =>
GET<DatasetListItemType[]>(`/core/dataset/list`, data);
/**
* get type=dataset list
*/
export const getAllDataset = () => GET<DatasetListItemType[]>(`/core/dataset/allDataset`);
export const getAllDataset = () => GET<DatasetSimpleItemType[]>(`/core/dataset/allDataset`);
export const getDatasetPaths = (parentId?: string) =>
GET<ParentTreePathItemType[]>('/core/dataset/paths', { parentId });
......
projects/app/src/web/core/dataset/components/SelectCollections.tsx
......@@ -11,6 +11,8 @@ import { useQuery } from '@tanstack/react-query';
import React, { useMemo, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { useLoading } from '@fastgpt/web/hooks/useLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '../context/datasetPageContext';
const SelectCollections = ({
datasetId,
......@@ -37,7 +39,8 @@ const SelectCollections = ({
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { datasetDetail, loadDatasetDetail } = useDatasetStore();
const { loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
const { Loading } = useLoading();
const [selectedDatasetCollectionIds, setSelectedDatasetCollectionIds] =
useState<string[]>(defaultSelectedId);
......
projects/app/src/web/core/dataset/constants.ts
import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/model';
import { DatasetTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetCollectionItemType,
DatasetItemType
......@@ -11,7 +12,7 @@ export const defaultDatasetDetail: DatasetItemType = {
teamId: '',
tmbId: '',
updateTime: new Date(),
type: 'dataset',
type: DatasetTypeEnum.dataset,
avatar: '/icon/logo.svg',
name: '',
intro: '',
......@@ -34,7 +35,7 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
teamId: '',
tmbId: '',
updateTime: new Date(),
type: 'dataset',
type: DatasetTypeEnum.dataset,
avatar: '/icon/logo.svg',
name: '',
intro: '',
......@@ -51,7 +52,7 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
sourceName: '',
sourceId: '',
createTime: new Date(),
trainingType: 'chunk',
trainingType: TrainingModeEnum.chunk,
chunkSize: 0
};
......
projects/app/src/web/core/dataset/context/datasetPageContext.tsx
import { useQuery } from '@tanstack/react-query';
import { ReactNode, useMemo } from 'react';
import { ReactNode, useMemo, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext } from 'use-context-selector';
import { getDatasetTrainingQueue, getTrainingQueueLen } from '../api';
import { useDatasetStore } from '../store/dataset';
import {
getDatasetById,
getDatasetTrainingQueue,
getTrainingQueueLen,
putDatasetById
} from '../api';
import { defaultDatasetDetail } from '../constants';
import { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
type DatasetPageContextType = {
datasetId: string;
datasetDetail: DatasetItemType;
loadDatasetDetail: (id: string) => Promise<DatasetItemType>;
updateDataset: (data: DatasetUpdateBody) => Promise<void>;
vectorTrainingMap: {
colorSchema: string;
tip: string;
......@@ -19,10 +31,6 @@ type DatasetPageContextType = {
refetchDatasetTraining: () => void;
};
type DatasetPageContextValueType = {
datasetId: string;
};
export const DatasetPageContext = createContext<DatasetPageContextType>({
vectorTrainingMap: {
colorSchema: '',
......@@ -36,19 +44,46 @@ export const DatasetPageContext = createContext<DatasetPageContextType>({
trainingCount: 0,
refetchDatasetTraining: function (): void {
throw new Error('Function not implemented.');
},
datasetId: '',
datasetDetail: defaultDatasetDetail,
loadDatasetDetail: function (id: string): Promise<DatasetItemType> {
throw new Error('Function not implemented.');
},
updateDataset: function (data: DatasetUpdateBody): Promise<void> {
throw new Error('Function not implemented.');
}
});
export const DatasetPageContextProvider = ({
children,
value
datasetId
}: {
children: ReactNode;
value: DatasetPageContextValueType;
datasetId: string;
}) => {
const { t } = useTranslation();
const { datasetId } = value;
const { datasetDetail } = useDatasetStore();
// dataset detail
const [datasetDetail, setDatasetDetail] = useState(defaultDatasetDetail);
const loadDatasetDetail = async (id: string) => {
const data = await getDatasetById(id);
setDatasetDetail(data);
return data;
};
const updateDataset = async (data: DatasetUpdateBody) => {
await putDatasetById(data);
if (datasetId === data.id) {
setDatasetDetail((state) => ({
...state,
...data
}));
}
};
// global queue
const { data: { vectorTrainingCount = 0, agentTrainingCount = 0 } = {} } = useQuery(
......@@ -108,6 +143,11 @@ export const DatasetPageContextProvider = ({
});
const contextValue: DatasetPageContextType = {
datasetId,
datasetDetail,
loadDatasetDetail,
updateDataset,
vectorTrainingMap,
agentTrainingMap,
rebuildingCount,
......
projects/app/src/web/core/dataset/context/datasetsContext.tsx 0 → 100644
import { ReactNode } from 'react';
import { createContext } from 'use-context-selector';
type ContextType = {};
export const Context = createContext<ContextType>({});
export const ContextProvider = ({ children }: { children: ReactNode }) => {
const contextValue: ContextType = {};
return <Context.Provider value={contextValue}>{children}</Context.Provider>;
};
projects/app/src/web/core/dataset/context/useDatasetContext.tsx deleted 100644 → 0
import { ReactNode } from 'react';
import { createContext } from 'use-context-selector';
type DatasetContextType = {};
type DatasetContextValueType = {};
export const DatasetContext = createContext<DatasetContextType>({});
export const DatasetContextProvider = ({
children,
value
}: {
children: ReactNode;
value: DatasetContextValueType;
}) => {
return <DatasetContext.Provider value={value}>{children}</DatasetContext.Provider>;
};
projects/app/src/web/core/dataset/store/dataset.ts
import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import type { DatasetItemType, DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import {
getAllDataset,
getDatasets,
getDatasetById,
putDatasetById,
postWebsiteSync
} from '@/web/core/dataset/api';
import { defaultDatasetDetail } from '../constants';
import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants';
import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api';
import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
import type {
DatasetListItemType,
DatasetSimpleItemType
} from '@fastgpt/global/core/dataset/type.d';
import { getAllDataset, getDatasets } from '@/web/core/dataset/api';
type State = {
allDatasets: DatasetListItemType[];
loadAllDatasets: () => Promise<DatasetListItemType[]>;
allDatasets: DatasetSimpleItemType[];
loadAllDatasets: () => Promise<DatasetSimpleItemType[]>;
myDatasets: DatasetListItemType[];
loadDatasets: (parentId?: string) => Promise<any>;
setDatasets(val: DatasetListItemType[]): void;
datasetDetail: DatasetItemType;
loadDatasetDetail: (id: string, init?: boolean) => Promise<DatasetItemType>;
updateDataset: (data: DatasetUpdateBody) => Promise<any>;
startWebsiteSync: () => Promise<any>;
loadMyDatasets: (parentId?: string) => Promise<any>;
setMyDatasets(val: DatasetListItemType[]): void;
};
export const useDatasetStore = create<State>()(
......@@ -40,66 +28,17 @@ export const useDatasetStore = create<State>()(
return res;
},
myDatasets: [],
async loadDatasets(parentId = '') {
async loadMyDatasets(parentId = '') {
const res = await getDatasets({ parentId });
set((state) => {
state.myDatasets = res;
});
return res;
},
setDatasets(val) {
setMyDatasets(val) {
set((state) => {
state.myDatasets = val;
});
},
datasetDetail: defaultDatasetDetail,
async loadDatasetDetail(id: string, init = false) {
if (!id || (id === get().datasetDetail._id && !init)) return get().datasetDetail;
const data = await getDatasetById(id);
set((state) => {
state.datasetDetail = data;
});
return data;
},
async updateDataset(data) {
await putDatasetById(data);
if (get().datasetDetail._id === data.id) {
set((state) => {
state.datasetDetail = {
...get().datasetDetail,
...data
};
});
}
set((state) => {
state.myDatasets = state.myDatasets = state.myDatasets.map((item) =>
item._id === data.id
? {
...item,
...data
}
: item
);
});
},
async startWebsiteSync() {
await checkTeamWebSyncLimit();
const billId = await postCreateTrainingUsage({
name: 'core.dataset.training.Website Sync',
datasetId: get().datasetDetail._id
});
return postWebsiteSync({ datasetId: get().datasetDetail._id, billId }).then(() => {
get().updateDataset({
id: get().datasetDetail._id,
status: DatasetStatusEnum.syncing
});
});
}
})),
{
......
projects/app/src/web/core/dataset/type.d.ts
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { ImportProcessWayEnum, ImportSourceTypeEnum } from './constants';
import { ImportProcessWayEnum } from './constants';
import { UseFormReturn } from 'react-hook-form';
export type ImportDataComponentProps = {
activeStep: number;
goToNext: () => void;
};
export type ImportSourceItemType = {
id: string;
......@@ -17,10 +12,10 @@ export type ImportSourceItemType = {
// source
sourceName: string;
sourceSize?: string;
icon: string;
// file
sourceSize?: string;
isUploading?: boolean;
uploadedFileRate?: number;
dbFileId?: string; // 存储在数据库里的文件Id，这个 ID 还是图片和集合的 metadata 中 relateId
......@@ -31,6 +26,10 @@ export type ImportSourceItemType = {
// custom text
rawText?: string;
// external file
sourceUrl?: string;
externalId?: string;
};
export type ImportSourceParamsType = UseFormReturn<
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment