Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 1fcdd7cb by Archer Committed by GitHub
Options

feat: url fetch and create file (#199) 

* docs

* docs

* feat: url fetch and create file
parent 4054eb9d
Showing with 534 additions and 309 deletions
README.md
...@@ -119,3 +119,4 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开 ...@@ -119,3 +119,4 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
1. 允许作为后台服务直接商用，但不允许直接使用 saas 服务商用。 1. 允许作为后台服务直接商用，但不允许直接使用 saas 服务商用。
2. 需保留相关版权信息。 2. 需保留相关版权信息。
3. 完整请查看 [FstGPT Open Source License](./LICENSE) 3. 完整请查看 [FstGPT Open Source License](./LICENSE)
4. 联系方式：yujinlong@sealos.io, [点击查看定价策略](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
client/public/imgs/files/url.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1692418843591" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4084" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M511.5 82c-236.6 0-429 192.4-429 429 0 236.5 192.5 429 429 429 236.6 0 429-192.4 429-429 0-236.5-192.4-429-429-429z m377.6 403.8H734.3c-4-139.9-41.4-259.9-97.5-331.9C776.5 203 879 332 889.1 485.8z m-402.8-349v349h-147c5.5-175.5 68.6-322.6 147-349z m0 399.4v349c-78.4-26.4-141.4-173.5-147-349h147z m50.5 349v-349h147c-5.6 175.5-68.6 322.6-147 349z m0-399.4v-349c78.4 26.4 141.4 173.5 147 349h-147zM386.3 153.9c-56.1 72-93.5 192-97.5 331.9H133.9C144.1 332 246.5 203 386.3 153.9zM133.9 536.2h154.8c4 139.9 41.4 259.9 97.5 331.9C246.5 819 144.1 690 133.9 536.2z m502.8 331.9c56.1-72 93.5-192 97.5-331.9H889C879 690 776.5 819 636.7 868.1z" fill="#5F9BEB" p-id="4085"></path></svg>
\ No newline at end of file
client/public/locales/en/common.json
...@@ -54,9 +54,14 @@ ...@@ -54,9 +54,14 @@
}, },
"file": { "file": {
"Click to download CSV template": "Click to download CSV template", "Click to download CSV template": "Click to download CSV template",
"Drag and drop": "Drag and drop files here, or click", "Create File": "Create File",
"Create file": "Create file",
"Drag and drop": "Drag and drop files here",
"Fetch Url": "Fetch Url",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
"Release the mouse to upload the file": "Release the mouse to upload the file", "Release the mouse to upload the file": "Release the mouse to upload the file",
"Select a maximum of 10 files": "Select a maximum of 10 files",
"max 10": "Max 10 files",
"select a document": "select a document", "select a document": "select a document",
"support": "support {{fileExtension}} file", "support": "support {{fileExtension}} file",
"upload error description": "Only upload multiple files or one folder at a time" "upload error description": "Only upload multiple files or one folder at a time"
......
client/public/locales/zh/common.json
...@@ -54,9 +54,14 @@ ...@@ -54,9 +54,14 @@
}, },
"file": { "file": {
"Click to download CSV template": "点击下载 CSV 模板", "Click to download CSV template": "点击下载 CSV 模板",
"Drag and drop": "拖拽文件至此，或点击", "Create File": "创建新文件",
"Create file": "创建文件",
"Drag and drop": "拖拽文件至此",
"Fetch Url": "链接读取",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
"Release the mouse to upload the file": "松开鼠标上传文件", "Release the mouse to upload the file": "松开鼠标上传文件",
"Select a maximum of 10 files": "最多选择10个文件",
"max 10": "最多选择 10 个文件",
"select a document": "选择文件", "select a document": "选择文件",
"support": "支持 {{fileExtension}} 文件", "support": "支持 {{fileExtension}} 文件",
"upload error description": "单次只支持上传多个文件或者一个文件夹" "upload error description": "单次只支持上传多个文件或者一个文件夹"
......
client/src/api/plugins/common.ts 0 → 100644
import { GET, POST, PUT, DELETE } from '../request';
import type { FetchResultItem } from '@/types/plugin';
export const fetchUrls = (urlList: string[]) =>
POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });
client/src/constants/flow/ModuleTemplate.ts
...@@ -182,7 +182,7 @@ export const ChatModule: FlowModuleTemplateType = { ...@@ -182,7 +182,7 @@ export const ChatModule: FlowModuleTemplateType = {
{ {
key: TaskResponseKeyEnum.answerText, key: TaskResponseKeyEnum.answerText,
label: '模型回复', label: '模型回复',
description: '如果外接了内容，会在回复结束时自动添加\n\n', description: '将在 stream 回复完毕后触发',
valueType: FlowValueTypeEnum.string, valueType: FlowValueTypeEnum.string,
type: FlowOutputItemTypeEnum.source, type: FlowOutputItemTypeEnum.source,
targets: [] targets: []
......
client/src/hooks/useSelectFile.tsx
import React, { useRef, useCallback } from 'react'; import React, { useRef, useCallback } from 'react';
import { Box } from '@chakra-ui/react'; import { Box } from '@chakra-ui/react';
import { useToast } from './useToast';
import { useTranslation } from 'react-i18next';
export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => { export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
const { t } = useTranslation();
const { fileType = '*', multiple = false } = props || {}; const { fileType = '*', multiple = false } = props || {};
const { toast } = useToast();
const SelectFileDom = useRef<HTMLInputElement>(null); const SelectFileDom = useRef<HTMLInputElement>(null);
const File = useCallback( const File = useCallback(
...@@ -15,12 +19,18 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) ...@@ -15,12 +19,18 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean })
multiple={multiple} multiple={multiple}
onChange={(e) => { onChange={(e) => {
if (!e.target.files || e.target.files?.length === 0) return; if (!e.target.files || e.target.files?.length === 0) return;
if (e.target.files.length > 10) {
return toast({
status: 'warning',
title: t('file.Select a maximum of 10 files')
});
}
onSelect(Array.from(e.target.files)); onSelect(Array.from(e.target.files));
}} }}
/> />
</Box> </Box>
), ),
[fileType, multiple] [fileType, multiple, t, toast]
); );
const onOpen = useCallback(() => { const onOpen = useCallback(() => {
......
client/src/pages/api/plugins/urlFetch.ts
...@@ -5,12 +5,9 @@ import { JSDOM } from 'jsdom'; ...@@ -5,12 +5,9 @@ import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability'; import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import type { FetchResultItem } from '@/types/plugin';
import { simpleText } from '@/utils/file';
type FetchResultItem = {
url: string;
title: string;
content: string;
};
export type UrlFetchResponse = FetchResultItem[]; export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => { const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
...@@ -38,10 +35,11 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => { ...@@ -38,10 +35,11 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
const reader = new Readability(dom.window.document); const reader = new Readability(dom.window.document);
const article = reader.parse(); const article = reader.parse();
const content = article?.textContent || '';
return { return {
url, url,
title: article?.title || '', content: simpleText(`${article?.title}\n${content}`)
content: article?.textContent || ''
}; };
}) })
) )
......
client/src/pages/kb/detail/components/DataCard.tsx
...@@ -217,7 +217,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -217,7 +217,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
<Box color={'myGray.600'}>{item.a}</Box> <Box color={'myGray.600'}>{item.a}</Box>
</Box> </Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}> <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1}> <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
{item.source?.trim()} {item.source?.trim()}
</Box> </Box>
<IconButton <IconButton
......
client/src/pages/kb/detail/components/Import/CreateFileModal.tsx 0 → 100644
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
const CreateFileModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: { filename: string; content: string }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
filename: '',
content: ''
}
});
return (
<MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
<ModalBody>
<Box mb={1} fontSize={'sm'}>
文件名
</Box>
<Input
mb={5}
{...register('filename', {
required: '文件名不能为空'
})}
/>
<Box mb={1} fontSize={'sm'}>
文件内容
</Box>
<Textarea
{...register('content', {
required: '文件内容不能为空'
})}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
取消
</Button>
<Button
onClick={() => {
handleSubmit(onSuccess)();
onClose();
}}
>
确认
</Button>
</ModalFooter>
</MyModal>
);
};
export default CreateFileModal;
client/src/pages/kb/detail/components/Import/Csv.tsx
...@@ -8,28 +8,18 @@ import { getErrText } from '@/utils/tools'; ...@@ -8,28 +8,18 @@ import { getErrText } from '@/utils/tools';
import { vectorModelList } from '@/store/static'; import { vectorModelList } from '@/store/static';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete'; import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import { customAlphabet } from 'nanoid';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
import { readCsvContent } from '@/utils/file';
const fileExtension = '.csv'; const fileExtension = '.csv';
type FileItemType = {
id: string;
filename: string;
chunks: { q: string; a: string; source?: string }[];
};
const CsvImport = ({ kbId }: { kbId: string }) => { const CsvImport = ({ kbId }: { kbId: string }) => {
const model = vectorModelList[0]?.model; const model = vectorModelList[0]?.model;
const theme = useTheme(); const theme = useTheme();
const router = useRouter(); const router = useRouter();
const { toast } = useToast(); const { toast } = useToast();
const [selecting, setSelecting] = useState(false);
const [files, setFiles] = useState<FileItemType[]>([]); const [files, setFiles] = useState<FileItemType[]>([]);
const [successChunks, setSuccessChunks] = useState(0); const [successChunks, setSuccessChunks] = useState(0);
...@@ -43,58 +33,9 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -43,58 +33,9 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。` content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。`
}); });
const onSelectFile = useCallback(
async (files: File[]) => {
setSelecting(true);
try {
let promise = Promise.resolve();
files
.filter((file) => /csv$/.test(file.name))
.forEach((file) => {
promise = promise.then(async () => {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'question' || header[1] !== 'answer') {
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
}
setFiles((state) => [
{
id: nanoid(),
filename: file.name,
chunks: data.map((item) => ({
q: item[0],
a: item[1],
source: item[2]
}))
},
...state
]);
});
});
await promise;
} catch (error: any) {
console.log(error);
toast({
title: getErrText(error, '解析文件失败'),
status: 'error'
});
}
setSelecting(false);
},
[toast]
);
const { mutate: onclickUpload, isLoading: uploading } = useMutation({ const { mutate: onclickUpload, isLoading: uploading } = useMutation({
mutationFn: async () => { mutationFn: async () => {
const chunks: { a: string; q: string; source: string }[] = []; const chunks = files.map((file) => file.chunks).flat();
files.forEach((file) =>
file.chunks.forEach((chunk) => {
chunks.push({
...chunk,
source: chunk.source || file.filename
});
})
);
// subsection import // subsection import
let success = 0; let success = 0;
...@@ -131,6 +72,10 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -131,6 +72,10 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
} }
}); });
const filenameStyles = {
className: 'textEllipsis',
maxW: '400px'
};
return ( return (
<Box display={['block', 'flex']} h={['auto', '100%']}> <Box display={['block', 'flex']} h={['auto', '100%']}>
<Flex <Flex
...@@ -143,11 +88,12 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -143,11 +88,12 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
> >
<FileSelect <FileSelect
fileExtension={fileExtension} fileExtension={fileExtension}
onSelectFile={onSelectFile}
isLoading={selecting}
tipText={ tipText={
'file.If the imported file is garbled, please convert CSV to UTF-8 encoding format' 'file.If the imported file is garbled, please convert CSV to UTF-8 encoding format'
} }
onPushFiles={(files) => setFiles((state) => files.concat(state))}
showUrlFetch={false}
showCreateFile={false}
py={emptyFiles ? '100px' : 5} py={emptyFiles ? '100px' : 5}
isCsv isCsv
/> />
...@@ -169,7 +115,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -169,7 +115,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
_hover={{ ...hoverDeleteStyles }} _hover={{ ...hoverDeleteStyles }}
> >
<Image src={'/imgs/files/csv.svg'} w={'16px'} alt={''} /> <Image src={'/imgs/files/csv.svg'} w={'16px'} alt={''} />
<Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis"> <Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
{item.filename} {item.filename}
</Box> </Box>
<MyIcon <MyIcon
...@@ -203,9 +149,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -203,9 +149,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
</Flex> </Flex>
{!emptyFiles && ( {!emptyFiles && (
<Box flex={'2 0 0'} w={['100%', 0]} h={'100%'} pt={[4, 8]} overflow={'overlay'}> <Box flex={'2 0 0'} w={['100%', 0]} h={'100%'} pt={[4, 8]} overflow={'overlay'}>
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}> <Flex px={[4, 8]} alignItems={'center'}>
数据预览({totalChunk}组) <Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
分段预览({totalChunk}组)
</Box>
{totalChunk > 100 && (
<Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
仅展示部分
</Box> </Box>
)}
</Flex>
<Box px={[4, 8]} overflow={'overlay'}> <Box px={[4, 8]} overflow={'overlay'}>
{files.map((file) => {files.map((file) =>
file.chunks.slice(0, 100).map((item, i) => ( file.chunks.slice(0, 100).map((item, i) => (
......
client/src/pages/kb/detail/components/Import/FileSelect.tsx
...@@ -2,30 +2,53 @@ import MyIcon from '@/components/Icon'; ...@@ -2,30 +2,53 @@ import MyIcon from '@/components/Icon';
import { useLoading } from '@/hooks/useLoading'; import { useLoading } from '@/hooks/useLoading';
import { useSelectFile } from '@/hooks/useSelectFile'; import { useSelectFile } from '@/hooks/useSelectFile';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import { fileDownload } from '@/utils/file'; import { fileDownload, readCsvContent, simpleText, splitText2Chunks } from '@/utils/file';
import { Box, Flex, Text, type BoxProps } from '@chakra-ui/react'; import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
import { fileImgs } from '@/constants/common';
import { DragEvent, useCallback, useState } from 'react'; import { DragEvent, useCallback, useState } from 'react';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { customAlphabet } from 'nanoid';
import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import { FetchResultItem } from '@/types/plugin';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
export type FileItemType = {
id: string;
filename: string;
chunks: { q: string; a: string; source?: string }[];
text: string;
icon: string;
tokens: number;
};
interface Props extends BoxProps { interface Props extends BoxProps {
fileExtension: string; fileExtension: string;
onPushFiles: (files: FileItemType[]) => void;
tipText?: string; tipText?: string;
onSelectFile: (files: File[]) => Promise<void>; chunkLen?: number;
isLoading?: boolean;
isCsv?: boolean; isCsv?: boolean;
showUrlFetch?: boolean;
showCreateFile?: boolean;
} }
const FileSelect = ({ const FileSelect = ({
fileExtension, fileExtension,
onSelectFile, onPushFiles,
isLoading,
tipText, tipText,
chunkLen = 500,
isCsv = false, isCsv = false,
showUrlFetch = true,
showCreateFile = true,
...props ...props
}: Props) => { }: Props) => {
const { Loading: FileSelectLoading } = useLoading(); const { Loading: FileSelectLoading } = useLoading();
const { t } = useTranslation(); const { t } = useTranslation();
const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
const { toast } = useToast(); const { toast } = useToast();
...@@ -35,6 +58,154 @@ const FileSelect = ({ ...@@ -35,6 +58,154 @@ const FileSelect = ({
}); });
const [isDragging, setIsDragging] = useState(false); const [isDragging, setIsDragging] = useState(false);
const [selecting, setSelecting] = useState(false);
const {
isOpen: isOpenUrlFetch,
onOpen: onOpenUrlFetch,
onClose: onCloseUrlFetch
} = useDisclosure();
const {
isOpen: isOpenCreateFile,
onOpen: onOpenCreateFile,
onClose: onCloseCreateFile
} = useDisclosure();
const onSelectFile = useCallback(
async (files: File[]) => {
setSelecting(true);
try {
// Parse file by file
let promise = Promise.resolve<FileItemType[]>([]);
files.forEach((file) => {
promise = promise.then(async (result) => {
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
/* text file */
const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
let text = await (async () => {
switch (extension) {
case 'txt':
case 'md':
return readTxtContent(file);
case 'pdf':
return readPdfContent(file);
case 'doc':
case 'docx':
return readDocContent(file);
}
return '';
})();
if (!icon) return result;
if (text) {
text = simpleText(text);
const splitRes = splitText2Chunks({
text,
maxLen: chunkLen
});
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
text,
tokens: splitRes.tokens,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: '',
source: file.name
}))
};
return [fileItem].concat(result);
}
/* csv file */
if (extension === 'csv') {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'question' || header[1] !== 'answer') {
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
}
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
tokens: 0,
text: '',
chunks: data.map((item) => ({
q: item[0],
a: item[1],
source: item[2] || file.name
}))
};
return [fileItem].concat(result);
}
return result;
});
});
const chunkFiles = await promise;
onPushFiles(chunkFiles);
} catch (error: any) {
console.log(error);
toast({
title: typeof error === 'string' ? error : '解析文件失败',
status: 'error'
});
}
setSelecting(false);
},
[chunkLen, onPushFiles, toast]
);
const onUrlFetch = useCallback(
(e: FetchResultItem[]) => {
const result = e.map(({ url, content }) => {
const splitRes = splitText2Chunks({
text: content,
maxLen: chunkLen
});
return {
id: nanoid(),
filename: url,
icon: '/imgs/files/url.svg',
text: content,
tokens: splitRes.tokens,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: '',
source: url
}))
};
});
onPushFiles(result);
},
[chunkLen, onPushFiles]
);
const onCreateFile = useCallback(
({ filename, content }: { filename: string; content: string }) => {
content = simpleText(content);
const splitRes = splitText2Chunks({
text: content,
maxLen: chunkLen
});
onPushFiles([
{
id: nanoid(),
filename,
icon: '/imgs/files/txt.svg',
text: content,
tokens: splitRes.tokens,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: '',
source: filename
}))
}
]);
},
[chunkLen, onPushFiles]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => { const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault(); e.preventDefault();
...@@ -46,7 +217,8 @@ const FileSelect = ({ ...@@ -46,7 +217,8 @@ const FileSelect = ({
setIsDragging(false); setIsDragging(false);
}; };
const handleDrop = useCallback(async (e: DragEvent<HTMLDivElement>) => { const handleDrop = useCallback(
async (e: DragEvent<HTMLDivElement>) => {
e.preventDefault(); e.preventDefault();
setIsDragging(false); setIsDragging(false);
...@@ -95,7 +267,19 @@ const FileSelect = ({ ...@@ -95,7 +267,19 @@ const FileSelect = ({
} }
onSelectFile(fileList); onSelectFile(fileList);
}, []); },
[onSelectFile, t, toast]
);
const SelectTextStyles: BoxProps = {
ml: 1,
as: 'span',
cursor: 'pointer',
color: 'myBlue.700',
_hover: {
textDecoration: 'underline'
}
};
return ( return (
<Box <Box
...@@ -120,10 +304,28 @@ const FileSelect = ({ ...@@ -120,10 +304,28 @@ const FileSelect = ({
t('file.Release the mouse to upload the file') t('file.Release the mouse to upload the file')
) : ( ) : (
<Box> <Box>
{t('file.Drag and drop')} {t('file.Drag and drop')},
<Text ml={1} as={'span'} cursor={'pointer'} color={'myBlue.700'} onClick={onOpen}> <MyTooltip label={t('file.max 10')}>
<Box {...SelectTextStyles} onClick={onOpen}>
{t('file.select a document')} {t('file.select a document')}
</Text> </Box>
</MyTooltip>
{showUrlFetch && (
<>
,
<Box {...SelectTextStyles} onClick={onOpenUrlFetch}>
{t('file.Fetch Url')}
</Box>
</>
)}
{showCreateFile && (
<>
,
<Box {...SelectTextStyles} onClick={onOpenCreateFile}>
{t('file.Create file')}
</Box>
</>
)}
</Box> </Box>
)} )}
</Flex> </Flex>
...@@ -151,8 +353,10 @@ const FileSelect = ({ ...@@ -151,8 +353,10 @@ const FileSelect = ({
{t('file.Click to download CSV template')} {t('file.Click to download CSV template')}
</Box> </Box>
)} )}
<FileSelectLoading loading={isLoading} fixed={false} /> <FileSelectLoading loading={selecting} fixed={false} />
<File onSelect={onSelectFile} /> <File onSelect={onSelectFile} />
{isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
{isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}
</Box> </Box>
); );
}; };
......
client/src/pages/kb/detail/components/Import/UrlFetchModal.tsx 0 → 100644
import React, { useRef } from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import type { FetchResultItem } from '@/types/plugin';
import { useRequest } from '@/hooks/useRequest';
import { fetchUrls } from '@/api/plugins/common';
const UrlFetchModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: FetchResultItem[]) => void;
}) => {
const { t } = useTranslation();
const Dom = useRef<HTMLTextAreaElement>(null);
const { mutate, isLoading } = useRequest({
mutationFn: async () => {
const val = Dom.current?.value || '';
const urls = val.split('\n').filter((e) => e);
const res = await fetchUrls(urls);
onSuccess(res);
onClose();
},
errorToast: '获取链接失败'
});
return (
<MyModal
title={
<>
<Box>{t('file.Fetch Url')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
目前仅支持读取静态链接，请注意检查结果
</Box>
</>
}
top={'15vh'}
isOpen
onClose={onClose}
w={'600px'}
>
<ModalBody>
<Textarea
ref={Dom}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={'最多10个链接，每行一个。'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
取消
</Button>
<Button isLoading={isLoading} onClick={mutate}>
确认
</Button>
</ModalFooter>
</MyModal>
);
};
export default UrlFetchModal;
client/src/types/plugin.d.ts
...@@ -28,3 +28,8 @@ export type KbTestItemType = { ...@@ -28,3 +28,8 @@ export type KbTestItemType = {
time: Date; time: Date;
results: (KbDataItemType & { score: number })[]; results: (KbDataItemType & { score: number })[];
}; };
export type FetchResultItem = {
url: string;
content: string;
};
client/src/utils/file.ts
...@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe ...@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
const overlapLen = Math.floor(maxLen * 0.3); // Overlap length const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
try { try {
const splitTexts = text.split(/(?<=[。！？.!?])/g); const splitTexts = text.split(/(?<=[。！？；.!?;])/g);
const chunks: string[] = []; const chunks: string[] = [];
let preChunk = ''; let preChunk = '';
...@@ -268,3 +268,11 @@ export const compressImg = ({ ...@@ -268,3 +268,11 @@ export const compressImg = ({
reject('压缩图片异常'); reject('压缩图片异常');
}; };
}); });
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/\s{2,}/g, ' ');
return text;
};
docSite/docs/develop/deploy/docker.md
...@@ -69,8 +69,8 @@ services: ...@@ -69,8 +69,8 @@ services:
- ./mongo/data:/data/db - ./mongo/data:/data/db
fastgpt: fastgpt:
container_name: fastgpt container_name: fastgpt
# image: c121914yu/fast-gpt:latest # docker hub # image: ghcr.io/labring/fastgpt:latest # git
image: ghcr.io/labring/fastgpt:latest # 阿里云 image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:latest # 阿里云
ports: ports:
- 3000:3000 - 3000:3000
networks: networks:
......
docSite/docs/develop/update/41init.md
...@@ -15,6 +15,6 @@ ...@@ -15,6 +15,6 @@
## 执行初始化 API ## 执行初始化 API
部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的） 部署新版项目，并发起 1 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）
https://xxxxx/api/admin/initChatItem https://xxxxx/api/admin/initChatItem
docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md
...@@ -15,6 +15,6 @@ ...@@ -15,6 +15,6 @@
## 执行初始化 API ## 执行初始化 API
部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的） 部署新版项目，并发起 1 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）
https://xxxxx/api/admin/initChatItem https://xxxxx/api/admin/initChatItem
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment