Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 2ae8d432 by archer
Options

fix: reg failed

parent 0ea464f3
Showing with 74 additions and 69 deletions
client/src/constants/common.ts
...@@ -6,11 +6,11 @@ export enum UserAuthTypeEnum { ...@@ -6,11 +6,11 @@ export enum UserAuthTypeEnum {
export const PRICE_SCALE = 100000; export const PRICE_SCALE = 100000;
export const fileImgs = [ export const fileImgs = [
{ reg: /pdf/gi, src: '/imgs/files/pdf.svg' }, { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
{ reg: /csv/gi, src: '/imgs/files/csv.svg' }, { suffix: 'csv', src: '/imgs/files/csv.svg' },
{ reg: /(doc|docs)/gi, src: '/imgs/files/doc.svg' }, { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
{ reg: /txt/gi, src: '/imgs/files/txt.svg' }, { suffix: 'txt', src: '/imgs/files/txt.svg' },
{ reg: /md/gi, src: '/imgs/files/markdown.svg' } { suffix: 'md', src: '/imgs/files/markdown.svg' }
]; ];
export const htmlTemplate = `<!DOCTYPE html> export const htmlTemplate = `<!DOCTYPE html>
......
client/src/pages/kb/detail/components/Import/FileSelect.tsx
...@@ -76,76 +76,76 @@ const FileSelect = ({ ...@@ -76,76 +76,76 @@ const FileSelect = ({
setSelecting(true); setSelecting(true);
try { try {
// Parse file by file // Parse file by file
let promise = Promise.resolve<FileItemType[]>([]); const chunkFiles: FileItemType[] = [];
files.forEach((file) => {
promise = promise.then(async (result) => {
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
/* text file */ for await (let file of files) {
const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src; const extension = file?.name?.split('.')?.pop()?.toLowerCase();
let text = await (async () => {
switch (extension) {
case 'txt':
case 'md':
return readTxtContent(file);
case 'pdf':
return readPdfContent(file);
case 'doc':
case 'docx':
return readDocContent(file);
}
return '';
})();
if (!icon) return result; /* text file */
const icon = fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.name))?.src;
if (text) { if (!icon) {
text = simpleText(text); continue;
const splitRes = splitText2Chunks({ }
text,
maxLen: chunkLen
});
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
text,
tokens: splitRes.tokens,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: '',
source: file.name
}))
};
return [fileItem].concat(result);
}
/* csv file */ let text = await (async () => {
if (extension === 'csv') { switch (extension) {
const { header, data } = await readCsvContent(file); case 'txt':
if (header[0] !== 'question' || header[1] !== 'answer') { case 'md':
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列'); return readTxtContent(file);
} case 'pdf':
const fileItem: FileItemType = { return readPdfContent(file);
id: nanoid(), case 'doc':
filename: file.name, case 'docx':
icon, return readDocContent(file);
tokens: 0,
text: '',
chunks: data.map((item) => ({
q: item[0],
a: item[1],
source: item[2] || file.name
}))
};
return [fileItem].concat(result);
} }
return result; return '';
}); })();
});
const chunkFiles = await promise; if (text) {
text = simpleText(text);
const splitRes = splitText2Chunks({
text,
maxLen: chunkLen
});
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
text,
tokens: splitRes.tokens,
chunks: splitRes.chunks.map((chunk) => ({
q: chunk,
a: '',
source: file.name
}))
};
chunkFiles.unshift(fileItem);
continue;
}
/* csv file */
if (extension === 'csv') {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'question' || header[1] !== 'answer') {
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
}
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
tokens: 0,
text: '',
chunks: data.map((item) => ({
q: item[0],
a: item[1],
source: item[2] || file.name
}))
};
chunkFiles.unshift(fileItem);
}
}
onPushFiles(chunkFiles); onPushFiles(chunkFiles);
} catch (error: any) { } catch (error: any) {
console.log(error); console.log(error);
......
client/src/service/events/generateQA.ts
...@@ -9,6 +9,7 @@ import { axiosConfig, getAIChatApi } from '../ai/openai'; ...@@ -9,6 +9,7 @@ import { axiosConfig, getAIChatApi } from '../ai/openai';
import { ChatCompletionRequestMessage } from 'openai'; import { ChatCompletionRequestMessage } from 'openai';
import { modelToolMap } from '@/utils/plugin'; import { modelToolMap } from '@/utils/plugin';
import { gptMessage2ChatType } from '@/utils/adapt'; import { gptMessage2ChatType } from '@/utils/adapt';
import { addLog } from '../utils/tools';
const reduceQueue = () => { const reduceQueue = () => {
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0; global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
...@@ -105,12 +106,16 @@ A2: ...@@ -105,12 +106,16 @@ A2:
const result = formatSplitText(answer || ''); // 格式化后的QA对 const result = formatSplitText(answer || ''); // 格式化后的QA对
console.log(`split result length: `, result.length); console.log(`split result length: `, result.length);
// 计费 // 计费
result.length > 0 && if (result.length > 0) {
pushQABill({ pushQABill({
userId: data.userId, userId: data.userId,
totalTokens, totalTokens,
appName: 'QA 拆分' appName: 'QA 拆分'
}); });
} else {
addLog.info(`QA result 0:`, { answer });
}
return { return {
rawContent: answer, rawContent: answer,
result result
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment