Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 1fcdd7cb by Archer Committed by GitHub
Options

feat: url fetch and create file (#199) 

* docs

* docs

* feat: url fetch and create file
parent 4054eb9d
Showing with 326 additions and 305 deletions
README.md
...@@ -119,3 +119,4 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开 ...@@ -119,3 +119,4 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
1. 允许作为后台服务直接商用，但不允许直接使用 saas 服务商用。 1. 允许作为后台服务直接商用，但不允许直接使用 saas 服务商用。
2. 需保留相关版权信息。 2. 需保留相关版权信息。
3. 完整请查看 [FstGPT Open Source License](./LICENSE) 3. 完整请查看 [FstGPT Open Source License](./LICENSE)
4. 联系方式：yujinlong@sealos.io, [点击查看定价策略](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
client/public/imgs/files/url.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1692418843591" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4084" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M511.5 82c-236.6 0-429 192.4-429 429 0 236.5 192.5 429 429 429 236.6 0 429-192.4 429-429 0-236.5-192.4-429-429-429z m377.6 403.8H734.3c-4-139.9-41.4-259.9-97.5-331.9C776.5 203 879 332 889.1 485.8z m-402.8-349v349h-147c5.5-175.5 68.6-322.6 147-349z m0 399.4v349c-78.4-26.4-141.4-173.5-147-349h147z m50.5 349v-349h147c-5.6 175.5-68.6 322.6-147 349z m0-399.4v-349c78.4 26.4 141.4 173.5 147 349h-147zM386.3 153.9c-56.1 72-93.5 192-97.5 331.9H133.9C144.1 332 246.5 203 386.3 153.9zM133.9 536.2h154.8c4 139.9 41.4 259.9 97.5 331.9C246.5 819 144.1 690 133.9 536.2z m502.8 331.9c56.1-72 93.5-192 97.5-331.9H889C879 690 776.5 819 636.7 868.1z" fill="#5F9BEB" p-id="4085"></path></svg>
\ No newline at end of file
client/public/locales/en/common.json
...@@ -54,9 +54,14 @@ ...@@ -54,9 +54,14 @@
}, },
"file": { "file": {
"Click to download CSV template": "Click to download CSV template", "Click to download CSV template": "Click to download CSV template",
"Drag and drop": "Drag and drop files here, or click", "Create File": "Create File",
"Create file": "Create file",
"Drag and drop": "Drag and drop files here",
"Fetch Url": "Fetch Url",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
"Release the mouse to upload the file": "Release the mouse to upload the file", "Release the mouse to upload the file": "Release the mouse to upload the file",
"Select a maximum of 10 files": "Select a maximum of 10 files",
"max 10": "Max 10 files",
"select a document": "select a document", "select a document": "select a document",
"support": "support {{fileExtension}} file", "support": "support {{fileExtension}} file",
"upload error description": "Only upload multiple files or one folder at a time" "upload error description": "Only upload multiple files or one folder at a time"
......
client/public/locales/zh/common.json
...@@ -54,9 +54,14 @@ ...@@ -54,9 +54,14 @@
}, },
"file": { "file": {
"Click to download CSV template": "点击下载 CSV 模板", "Click to download CSV template": "点击下载 CSV 模板",
"Drag and drop": "拖拽文件至此，或点击", "Create File": "创建新文件",
"Create file": "创建文件",
"Drag and drop": "拖拽文件至此",
"Fetch Url": "链接读取",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
"Release the mouse to upload the file": "松开鼠标上传文件", "Release the mouse to upload the file": "松开鼠标上传文件",
"Select a maximum of 10 files": "最多选择10个文件",
"max 10": "最多选择 10 个文件",
"select a document": "选择文件", "select a document": "选择文件",
"support": "支持 {{fileExtension}} 文件", "support": "支持 {{fileExtension}} 文件",
"upload error description": "单次只支持上传多个文件或者一个文件夹" "upload error description": "单次只支持上传多个文件或者一个文件夹"
......
client/src/api/plugins/common.ts 0 → 100644
import { GET, POST, PUT, DELETE } from '../request';
import type { FetchResultItem } from '@/types/plugin';
export const fetchUrls = (urlList: string[]) =>
POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });
client/src/constants/flow/ModuleTemplate.ts
...@@ -182,7 +182,7 @@ export const ChatModule: FlowModuleTemplateType = { ...@@ -182,7 +182,7 @@ export const ChatModule: FlowModuleTemplateType = {
{ {
key: TaskResponseKeyEnum.answerText, key: TaskResponseKeyEnum.answerText,
label: '模型回复', label: '模型回复',
description: '如果外接了内容，会在回复结束时自动添加\n\n', description: '将在 stream 回复完毕后触发',
valueType: FlowValueTypeEnum.string, valueType: FlowValueTypeEnum.string,
type: FlowOutputItemTypeEnum.source, type: FlowOutputItemTypeEnum.source,
targets: [] targets: []
......
client/src/hooks/useSelectFile.tsx
import React, { useRef, useCallback } from 'react'; import React, { useRef, useCallback } from 'react';
import { Box } from '@chakra-ui/react'; import { Box } from '@chakra-ui/react';
import { useToast } from './useToast';
import { useTranslation } from 'react-i18next';
export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => { export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
const { t } = useTranslation();
const { fileType = '*', multiple = false } = props || {}; const { fileType = '*', multiple = false } = props || {};
const { toast } = useToast();
const SelectFileDom = useRef<HTMLInputElement>(null); const SelectFileDom = useRef<HTMLInputElement>(null);
const File = useCallback( const File = useCallback(
...@@ -15,12 +19,18 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) ...@@ -15,12 +19,18 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean })
multiple={multiple} multiple={multiple}
onChange={(e) => { onChange={(e) => {
if (!e.target.files || e.target.files?.length === 0) return; if (!e.target.files || e.target.files?.length === 0) return;
if (e.target.files.length > 10) {
return toast({
status: 'warning',
title: t('file.Select a maximum of 10 files')
});
}
onSelect(Array.from(e.target.files)); onSelect(Array.from(e.target.files));
}} }}
/> />
</Box> </Box>
), ),
[fileType, multiple] [fileType, multiple, t, toast]
); );
const onOpen = useCallback(() => { const onOpen = useCallback(() => {
......
client/src/pages/api/plugins/urlFetch.ts
...@@ -5,12 +5,9 @@ import { JSDOM } from 'jsdom'; ...@@ -5,12 +5,9 @@ import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability'; import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import type { FetchResultItem } from '@/types/plugin';
import { simpleText } from '@/utils/file';
type FetchResultItem = {
url: string;
title: string;
content: string;
};
export type UrlFetchResponse = FetchResultItem[]; export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => { const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
...@@ -38,10 +35,11 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => { ...@@ -38,10 +35,11 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
const reader = new Readability(dom.window.document); const reader = new Readability(dom.window.document);
const article = reader.parse(); const article = reader.parse();
const content = article?.textContent || '';
return { return {
url, url,
title: article?.title || '', content: simpleText(`${article?.title}\n${content}`)
content: article?.textContent || ''
}; };
}) })
) )
......
client/src/pages/kb/detail/components/DataCard.tsx
...@@ -217,7 +217,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -217,7 +217,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
<Box color={'myGray.600'}>{item.a}</Box> <Box color={'myGray.600'}>{item.a}</Box>
</Box> </Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}> <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1}> <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
{item.source?.trim()} {item.source?.trim()}
</Box> </Box>
<IconButton <IconButton
......
client/src/pages/kb/detail/components/Import/CreateFileModal.tsx 0 → 100644
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
const CreateFileModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: { filename: string; content: string }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
filename: '',
content: ''
}
});
return (
<MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
<ModalBody>
<Box mb={1} fontSize={'sm'}>
文件名
</Box>
<Input
mb={5}
{...register('filename', {
required: '文件名不能为空'
})}
/>
<Box mb={1} fontSize={'sm'}>
文件内容
</Box>
<Textarea
{...register('content', {
required: '文件内容不能为空'
})}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
取消
</Button>
<Button
onClick={() => {
handleSubmit(onSuccess)();
onClose();
}}
>
确认
</Button>
</ModalFooter>
</MyModal>
);
};
export default CreateFileModal;
client/src/pages/kb/detail/components/Import/Csv.tsx
...@@ -8,28 +8,18 @@ import { getErrText } from '@/utils/tools'; ...@@ -8,28 +8,18 @@ import { getErrText } from '@/utils/tools';
import { vectorModelList } from '@/store/static'; import { vectorModelList } from '@/store/static';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete'; import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import { customAlphabet } from 'nanoid';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
import { readCsvContent } from '@/utils/file';
const fileExtension = '.csv'; const fileExtension = '.csv';
type FileItemType = {
id: string;
filename: string;
chunks: { q: string; a: string; source?: string }[];
};
const CsvImport = ({ kbId }: { kbId: string }) => { const CsvImport = ({ kbId }: { kbId: string }) => {
const model = vectorModelList[0]?.model; const model = vectorModelList[0]?.model;
const theme = useTheme(); const theme = useTheme();
const router = useRouter(); const router = useRouter();
const { toast } = useToast(); const { toast } = useToast();
const [selecting, setSelecting] = useState(false);
const [files, setFiles] = useState<FileItemType[]>([]); const [files, setFiles] = useState<FileItemType[]>([]);
const [successChunks, setSuccessChunks] = useState(0); const [successChunks, setSuccessChunks] = useState(0);
...@@ -43,58 +33,9 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -43,58 +33,9 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。` content: `该任务无法终止，需要一定时间生成索引，请确认导入。如果余额不足，未完成的任务会被暂停，充值后可继续进行。`
}); });
const onSelectFile = useCallback(
async (files: File[]) => {
setSelecting(true);
try {
let promise = Promise.resolve();
files
.filter((file) => /csv$/.test(file.name))
.forEach((file) => {
promise = promise.then(async () => {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'question' || header[1] !== 'answer') {
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
}
setFiles((state) => [
{
id: nanoid(),
filename: file.name,
chunks: data.map((item) => ({
q: item[0],
a: item[1],
source: item[2]
}))
},
...state
]);
});
});
await promise;
} catch (error: any) {
console.log(error);
toast({
title: getErrText(error, '解析文件失败'),
status: 'error'
});
}
setSelecting(false);
},
[toast]
);
const { mutate: onclickUpload, isLoading: uploading } = useMutation({ const { mutate: onclickUpload, isLoading: uploading } = useMutation({
mutationFn: async () => { mutationFn: async () => {
const chunks: { a: string; q: string; source: string }[] = []; const chunks = files.map((file) => file.chunks).flat();
files.forEach((file) =>
file.chunks.forEach((chunk) => {
chunks.push({
...chunk,
source: chunk.source || file.filename
});
})
);
// subsection import // subsection import
let success = 0; let success = 0;
...@@ -131,6 +72,10 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -131,6 +72,10 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
} }
}); });
const filenameStyles = {
className: 'textEllipsis',
maxW: '400px'
};
return ( return (
<Box display={['block', 'flex']} h={['auto', '100%']}> <Box display={['block', 'flex']} h={['auto', '100%']}>
<Flex <Flex
...@@ -143,11 +88,12 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -143,11 +88,12 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
> >
<FileSelect <FileSelect
fileExtension={fileExtension} fileExtension={fileExtension}
onSelectFile={onSelectFile}
isLoading={selecting}
tipText={ tipText={
'file.If the imported file is garbled, please convert CSV to UTF-8 encoding format' 'file.If the imported file is garbled, please convert CSV to UTF-8 encoding format'
} }
onPushFiles={(files) => setFiles((state) => files.concat(state))}
showUrlFetch={false}
showCreateFile={false}
py={emptyFiles ? '100px' : 5} py={emptyFiles ? '100px' : 5}
isCsv isCsv
/> />
...@@ -169,7 +115,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -169,7 +115,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
_hover={{ ...hoverDeleteStyles }} _hover={{ ...hoverDeleteStyles }}
> >
<Image src={'/imgs/files/csv.svg'} w={'16px'} alt={''} /> <Image src={'/imgs/files/csv.svg'} w={'16px'} alt={''} />
<Box ml={2} flex={'1 0 0'} pr={3} className="textEllipsis"> <Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
{item.filename} {item.filename}
</Box> </Box>
<MyIcon <MyIcon
...@@ -203,9 +149,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -203,9 +149,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
</Flex> </Flex>
{!emptyFiles && ( {!emptyFiles && (
<Box flex={'2 0 0'} w={['100%', 0]} h={'100%'} pt={[4, 8]} overflow={'overlay'}> <Box flex={'2 0 0'} w={['100%', 0]} h={'100%'} pt={[4, 8]} overflow={'overlay'}>
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'}> <Flex px={[4, 8]} alignItems={'center'}>
数据预览({totalChunk}组) <Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
</Box> 分段预览({totalChunk}组)
</Box>
{totalChunk > 100 && (
<Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
仅展示部分
</Box>
)}
</Flex>
<Box px={[4, 8]} overflow={'overlay'}> <Box px={[4, 8]} overflow={'overlay'}>
{files.map((file) => {files.map((file) =>
file.chunks.slice(0, 100).map((item, i) => ( file.chunks.slice(0, 100).map((item, i) => (
......
client/src/pages/kb/detail/components/Import/UrlFetchModal.tsx 0 → 100644
import React, { useRef } from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import type { FetchResultItem } from '@/types/plugin';
import { useRequest } from '@/hooks/useRequest';
import { fetchUrls } from '@/api/plugins/common';
const UrlFetchModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: FetchResultItem[]) => void;
}) => {
const { t } = useTranslation();
const Dom = useRef<HTMLTextAreaElement>(null);
const { mutate, isLoading } = useRequest({
mutationFn: async () => {
const val = Dom.current?.value || '';
const urls = val.split('\n').filter((e) => e);
const res = await fetchUrls(urls);
onSuccess(res);
onClose();
},
errorToast: '获取链接失败'
});
return (
<MyModal
title={
<>
<Box>{t('file.Fetch Url')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'} mt={1}>
目前仅支持读取静态链接，请注意检查结果
</Box>
</>
}
top={'15vh'}
isOpen
onClose={onClose}
w={'600px'}
>
<ModalBody>
<Textarea
ref={Dom}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={'最多10个链接，每行一个。'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'base'} mr={4} onClick={onClose}>
取消
</Button>
<Button isLoading={isLoading} onClick={mutate}>
确认
</Button>
</ModalFooter>
</MyModal>
);
};
export default UrlFetchModal;
client/src/types/plugin.d.ts
...@@ -28,3 +28,8 @@ export type KbTestItemType = { ...@@ -28,3 +28,8 @@ export type KbTestItemType = {
time: Date; time: Date;
results: (KbDataItemType & { score: number })[]; results: (KbDataItemType & { score: number })[];
}; };
export type FetchResultItem = {
url: string;
content: string;
};
client/src/utils/file.ts
...@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe ...@@ -151,7 +151,7 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
const overlapLen = Math.floor(maxLen * 0.3); // Overlap length const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
try { try {
const splitTexts = text.split(/(?<=[。！？.!?])/g); const splitTexts = text.split(/(?<=[。！？；.!?;])/g);
const chunks: string[] = []; const chunks: string[] = [];
let preChunk = ''; let preChunk = '';
...@@ -268,3 +268,11 @@ export const compressImg = ({ ...@@ -268,3 +268,11 @@ export const compressImg = ({
reject('压缩图片异常'); reject('压缩图片异常');
}; };
}); });
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/\s{2,}/g, ' ');
return text;
};
docSite/docs/develop/deploy/docker.md
...@@ -69,8 +69,8 @@ services: ...@@ -69,8 +69,8 @@ services:
- ./mongo/data:/data/db - ./mongo/data:/data/db
fastgpt: fastgpt:
container_name: fastgpt container_name: fastgpt
# image: c121914yu/fast-gpt:latest # docker hub # image: ghcr.io/labring/fastgpt:latest # git
image: ghcr.io/labring/fastgpt:latest # 阿里云 image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:latest # 阿里云
ports: ports:
- 3000:3000 - 3000:3000
networks: networks:
......
docSite/docs/develop/update/41init.md
...@@ -15,6 +15,6 @@ ...@@ -15,6 +15,6 @@
## 执行初始化 API ## 执行初始化 API
部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的） 部署新版项目，并发起 1 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）
https://xxxxx/api/admin/initChatItem https://xxxxx/api/admin/initChatItem
docSite/i18n/zh-Hans/docusaurus-plugin-content-docs/current/develop/update/41init.md
...@@ -15,6 +15,6 @@ ...@@ -15,6 +15,6 @@
## 执行初始化 API ## 执行初始化 API
部署新版项目，并发起 3 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的） 部署新版项目，并发起 1 个 HTTP 请求（记得携带 headers.rootkey，这个值是环境变量里的）
https://xxxxx/api/admin/initChatItem https://xxxxx/api/admin/initChatItem
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment