Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit b5f0ac3e by Archer Committed by GitHub
Options

Perf: read file woker (#1337) 

* perf: read file worker

* fix: Http node url input

* fix: htm2md

* fix: html2md

* fix: ts

* perf: Problem classification increases the matching order

* feat: tool response answer
parent 1529c1e9
Showing with 382 additions and 367 deletions
.github/ISSUE_TEMPLATE/bugs.md
......@@ -21,7 +21,7 @@ assignees: ''
- [ ] 公有云版本
- [ ] 私有部署版本, 具体版本号:
**问题描述**
**问题描述, 日志截图**
**复现步骤**
......
packages/global/core/workflow/template/system/tools.ts
......@@ -64,5 +64,14 @@ export const ToolModule: FlowNodeTemplateType = {
Input_Template_History,
Input_Template_UserChatInput
],
outputs: []
outputs: [
{
id: NodeOutputKeyEnum.answerText,
key: NodeOutputKeyEnum.answerText,
label: 'core.module.output.label.Ai response content',
description: 'core.module.output.description.Ai response content',
valueType: WorkflowIOValueTypeEnum.string,
type: FlowNodeOutputTypeEnum.static
}
]
};
packages/service/common/file/gridfs/controller.ts
......@@ -6,7 +6,6 @@ import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
import { MongoFileSchema } from './schema';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { ReadFileByBufferParams } from '../read/type';
import { MongoRwaTextBuffer } from '../../buffer/rawText/schema';
import { readFileRawContent } from '../read/utils';
import { PassThrough } from 'stream';
......@@ -197,19 +196,15 @@ export const readFileContentFromMongo = async ({
});
})();
const params: ReadFileByBufferParams = {
const { rawText } = await readFileRawContent({
extension,
csvFormat,
teamId,
buffer: fileBuffers,
encoding,
metadata: {
relatedId: fileId
}
};
const { rawText } = await readFileRawContent({
extension,
csvFormat,
params
});
if (rawText.trim()) {
......
packages/service/common/file/read/html.ts deleted 100644 → 0
import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
import { initMarkdownText } from './utils';
import { htmlToMarkdown } from '../../string/markdown';
import { readFileRawText } from './rawText';
export const readHtmlRawText = async (
params: ReadFileByBufferParams
): Promise<ReadFileResponse> => {
const { teamId, metadata } = params;
const { rawText: html } = readFileRawText(params);
const md = await htmlToMarkdown(html);
const rawText = await initMarkdownText({
teamId,
md,
metadata
});
return {
rawText
};
};
packages/service/common/file/read/markdown.ts deleted 100644 → 0
import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
import { initMarkdownText } from './utils';
import { readFileRawText } from './rawText';
export const readMarkdown = async (params: ReadFileByBufferParams): Promise<ReadFileResponse> => {
const { teamId, metadata } = params;
const { rawText: md } = readFileRawText(params);
const rawText = await initMarkdownText({
teamId,
md,
metadata
});
return {
rawText
};
};
packages/service/common/file/read/utils.ts
import { markdownProcess } from '@fastgpt/global/common/string/markdown';
import { markdownProcess, simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
import { uploadMongoImg } from '../image/controller';
import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
import { addHours } from 'date-fns';
import { ReadFileByBufferParams } from './type';
import { readFileRawText } from '../read/rawText';
import { readMarkdown } from '../read/markdown';
import { readHtmlRawText } from '../read/html';
import { readPdfFile } from '../read/pdf';
import { readWordFile } from '../read/word';
import { readCsvRawText } from '../read/csv';
import { readPptxRawText } from '../read/pptx';
import { readXlsxRawText } from '../read/xlsx';
import { WorkerNameEnum, runWorker } from '../../../worker/utils';
import { ReadFileResponse } from '../../../worker/file/type';
export const initMarkdownText = ({
teamId,
......@@ -36,46 +30,39 @@ export const initMarkdownText = ({
export const readFileRawContent = async ({
extension,
csvFormat,
params
teamId,
buffer,
encoding,
metadata
}: {
csvFormat?: boolean;
extension: string;
params: ReadFileByBufferParams;
teamId: string;
buffer: Buffer;
encoding: string;
metadata?: Record<string, any>;
}) => {
switch (extension) {
case 'txt':
return readFileRawText(params);
case 'md':
return readMarkdown(params);
case 'html':
return readHtmlRawText(params);
case 'pdf':
return readPdfFile(params);
case 'docx':
return readWordFile(params);
case 'pptx':
return readPptxRawText(params);
case 'xlsx':
const xlsxResult = await readXlsxRawText(params);
if (csvFormat) {
return {
rawText: xlsxResult.formatText || ''
};
}
return {
rawText: xlsxResult.rawText
};
case 'csv':
const csvResult = await readCsvRawText(params);
if (csvFormat) {
return {
rawText: csvResult.formatText || ''
};
}
return {
rawText: csvResult.rawText
};
default:
return Promise.reject('Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx');
const result = await runWorker<ReadFileResponse>(WorkerNameEnum.readFile, {
extension,
csvFormat,
encoding,
buffer
});
// markdown data format
if (['md', 'html', 'docx'].includes(extension)) {
result.rawText = await initMarkdownText({
teamId: teamId,
md: result.rawText,
metadata: metadata
});
}
return result;
};
export const htmlToMarkdown = async (html?: string | null) => {
const md = await runWorker<string>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
return simpleMarkdownText(md);
};
packages/service/common/string/cheerio.ts
import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api';
import * as cheerio from 'cheerio';
import axios from 'axios';
import { htmlToMarkdown } from './markdown';
import { htmlToMarkdown } from '../file/read/utils';
export const cheerioToHtml = ({
fetchUrl,
......@@ -77,7 +77,9 @@ export const urlsFetch = async ({
$,
selector
});
console.log('html====', html);
const md = await htmlToMarkdown(html);
console.log('html====', md);
return {
url,
......
packages/service/common/string/markdown.ts deleted 100644 → 0
import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
import { WorkerNameEnum, runWorker } from '../../worker/utils';
/* html string to markdown */
export const htmlToMarkdown = async (html?: string | null) => {
const md = await runWorker<string>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
return simpleMarkdownText(md);
};
packages/service/common/vectorStore/pg/controller.ts
......@@ -23,7 +23,7 @@ export async function initPg() {
`);
await PgClient.query(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);`
`CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 100);`
);
await PgClient.query(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);`
......
packages/service/core/workflow/dispatch/agent/classifyQuestion.ts
......@@ -131,7 +131,9 @@ const completions = async ({
console.log(answer, '----');
const id =
agents.find((item) => answer.includes(item.key) || answer.includes(item.value))?.key || '';
agents.find((item) => answer.includes(item.key))?.key ||
agents.find((item) => answer.includes(item.value))?.key ||
'';
return {
tokens: await countMessagesTokens(messages),
......
packages/service/core/workflow/dispatch/agent/runTool/index.ts
......@@ -23,7 +23,9 @@ import { runToolWithPromptCall } from './promptCall';
import { replaceVariable } from '@fastgpt/global/common/string/tools';
import { Prompt_Tool_Call } from './constants';
type Response = DispatchNodeResultType<{}>;
type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.answerText]: string;
}>;
export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<Response> => {
const {
......@@ -129,6 +131,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
const flatUsages = dispatchFlowResponse.map((item) => item.flowUsages).flat();
return {
[NodeOutputKeyEnum.answerText]: assistantResponses
.filter((item) => item.text?.content)
.map((item) => item.text?.content || '')
.join(''),
[DispatchNodeResponseKeyEnum.assistantResponses]: assistantResponses,
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints: totalPointsUsage,
......
packages/service/core/workflow/dispatch/index.ts
......@@ -142,10 +142,8 @@ export async function dispatchWorkFlow({
}
if (assistantResponses) {
chatAssistantResponse = chatAssistantResponse.concat(assistantResponses);
}
// save assistant text response
if (answerText) {
} else if (answerText) {
// save assistant text response
const isResponseAnswerText =
inputs.find((item) => item.key === NodeInputKeyEnum.aiChatIsResponseText)?.value ?? true;
if (isResponseAnswerText) {
......
packages/service/core/workflow/dispatch/tools/answer.ts
......@@ -19,24 +19,24 @@ export const dispatchAnswer = (props: Record<string, any>): AnswerResponse => {
res,
detail,
stream,
node: { name },
params: { text = '' }
} = props as AnswerProps;
const formatText = typeof text === 'string' ? text : JSON.stringify(text, null, 2);
const responseText = `\n${formatText}`;
if (res && stream) {
responseWrite({
res,
event: detail ? SseResponseEventEnum.fastAnswer : undefined,
data: textAdaptGptResponse({
text: `\n${formatText}`
text: responseText
})
});
}
return {
[NodeOutputKeyEnum.answerText]: formatText,
[NodeOutputKeyEnum.answerText]: responseText,
[DispatchNodeResponseKeyEnum.nodeResponse]: {
textOutput: formatText
}
......
packages/service/common/file/read/csv.ts → packages/service/worker/file/extension/csv.ts
import Papa from 'papaparse';
import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
import { readFileRawText } from './rawText';
// 加载源文件内容
export const readCsvRawText = async (params: ReadFileByBufferParams): Promise<ReadFileResponse> => {
export const readCsvRawText = async (params: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
const { rawText } = readFileRawText(params);
const csvArr = Papa.parse(rawText).data as string[][];
......
packages/service/common/file/read/word.ts → packages/service/worker/file/extension/docx.ts
import mammoth from 'mammoth';
import { htmlToMarkdown } from '../../string/markdown';
import { ReadFileByBufferParams, ReadFileResponse } from './type';
import { initMarkdownText } from './utils';
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
import { html2md } from '../../htmlStr2Md/utils';
/**
* read docx to markdown
*/
export const readWordFile = async ({
teamId,
buffer,
metadata = {}
}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
export const readDocsFile = async ({ buffer }: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
try {
const { value: html } = await mammoth.convertToHtml({
buffer
});
const md = await htmlToMarkdown(html);
const rawText = await initMarkdownText({
teamId,
md,
metadata
});
const rawText = html2md(html);
return {
rawText,
metadata: {}
rawText
};
} catch (error) {
console.log('error doc read:', error);
......
packages/service/worker/file/extension/html.ts 0 → 100644
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
import { readFileRawText } from './rawText';
import { html2md } from '../../htmlStr2Md/utils';
export const readHtmlRawText = async (params: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
const { rawText: html } = readFileRawText(params);
const rawText = html2md(html);
return {
rawText
};
};
packages/service/common/file/read/pdf.ts → packages/service/worker/file/extension/pdf.ts
import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';
// @ts-ignore
import('pdfjs-dist/legacy/build/pdf.worker.min.mjs');
import { ReadFileByBufferParams, ReadFileResponse } from './type';
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
type TokenType = {
str: string;
......@@ -13,9 +13,7 @@ type TokenType = {
hasEOL: boolean;
};
export const readPdfFile = async ({
buffer
}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
export const readPdfFile = async ({ buffer }: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
const readPDFPage = async (doc: any, pageNo: number) => {
const page = await doc.getPage(pageNo);
const tokenizedText = await page.getTextContent();
......@@ -65,7 +63,6 @@ export const readPdfFile = async ({
loadingTask.destroy();
return {
rawText: pageTexts.join(''),
metadata: {}
rawText: pageTexts.join('')
};
};
packages/service/common/file/read/pptx.ts → packages/service/worker/file/extension/pptx.ts
import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
// import { parseOfficeAsync } from 'officeparser';
import { parseOffice } from './parseOffice';
import { parseOffice } from '../parseOffice';
export const readPptxRawText = async ({
buffer,
encoding
}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
}: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
const result = await parseOffice({
buffer,
encoding: encoding as BufferEncoding,
......
packages/service/common/file/read/rawText.ts → packages/service/worker/file/extension/rawText.ts
import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
import iconv from 'iconv-lite';
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
const rawEncodingList = [
'ascii',
......@@ -17,7 +17,7 @@ const rawEncodingList = [
];
// 加载源文件内容
export const readFileRawText = ({ buffer, encoding }: ReadFileByBufferParams): ReadFileResponse => {
export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
const content = rawEncodingList.includes(encoding)
? buffer.toString(encoding as BufferEncoding)
: iconv.decode(buffer, 'gbk');
......
packages/service/common/file/read/xlsx.ts → packages/service/worker/file/extension/xlsx.ts
import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
import xlsx from 'node-xlsx';
import Papa from 'papaparse';
export const readXlsxRawText = async ({
buffer
}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
}: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
const result = xlsx.parse(buffer, {
skipHidden: false,
defval: ''
......
packages/service/common/file/read/parseOffice.ts → packages/service/worker/file/parseOffice.ts
......@@ -2,8 +2,8 @@ import { getNanoid } from '@fastgpt/global/common/string/tools';
import fs from 'fs';
import decompress from 'decompress';
import { DOMParser } from '@xmldom/xmldom';
import { clearDirFiles } from '../utils';
import { addLog } from '../../system/log';
import { clearDirFiles } from '../../common/file/utils';
import { addLog } from '../../common/system/log';
const DEFAULTDECOMPRESSSUBLOCATION = '/tmp';
......
packages/service/worker/file/read.ts 0 → 100644
import { parentPort } from 'worker_threads';
import { readFileRawText } from './extension/rawText';
import { ReadRawTextByBuffer, ReadRawTextProps } from './type';
import { readHtmlRawText } from './extension/html';
import { readPdfFile } from './extension/pdf';
import { readDocsFile } from './extension/docx';
import { readPptxRawText } from './extension/pptx';
import { readXlsxRawText } from './extension/xlsx';
import { readCsvRawText } from './extension/csv';
parentPort?.on('message', async (props: ReadRawTextProps<Uint8Array>) => {
const readFileRawContent = async (params: ReadRawTextByBuffer) => {
switch (params.extension) {
case 'txt':
case 'md':
return readFileRawText(params);
case 'html':
return readHtmlRawText(params);
case 'pdf':
return readPdfFile(params);
case 'docx':
return readDocsFile(params);
case 'pptx':
return readPptxRawText(params);
case 'xlsx':
const xlsxResult = await readXlsxRawText(params);
if (params.csvFormat) {
return {
rawText: xlsxResult.formatText || ''
};
}
return {
rawText: xlsxResult.rawText
};
case 'csv':
const csvResult = await readCsvRawText(params);
if (params.csvFormat) {
return {
rawText: csvResult.formatText || ''
};
}
return {
rawText: csvResult.rawText
};
default:
return Promise.reject('Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx');
}
};
// params.buffer: Uint8Array -> buffer
const buffer = Buffer.from(props.buffer);
const newProps: ReadRawTextByBuffer = {
...props,
buffer
};
try {
parentPort?.postMessage({
type: 'success',
data: await readFileRawContent(newProps)
});
} catch (error) {
console.log(error);
parentPort?.postMessage({
type: 'error',
data: error
});
}
global?.close?.();
});
packages/service/common/file/read/type.d.ts → packages/service/worker/file/type.d.ts
export type ReadFileByBufferParams = {
teamId: string;
buffer: Buffer;
import { ReadFileByBufferParams } from '../../common/file/read/type';
export type ReadRawTextProps<T> = {
csvFormat?: boolean;
extension: string;
buffer: T;
encoding: string;
metadata?: Record<string, any>;
};
export type ReadRawTextByBuffer = ReadRawTextProps<Buffer>;
export type ReadFileResponse = {
rawText: string;
formatText?: string;
metadata?: Record<string, any>;
};
packages/service/worker/htmlStr2Md/index.ts 0 → 100644
import { parentPort } from 'worker_threads';
import { html2md } from './utils';
parentPort?.on('message', (params: { html: string }) => {
try {
const md = html2md(params?.html || '');
parentPort?.postMessage({
type: 'success',
data: md
});
} catch (error) {
parentPort?.postMessage({
type: 'error',
data: error
});
}
global?.close?.();
});
packages/service/worker/htmlStr2Md.ts → packages/service/worker/htmlStr2Md/utils.ts
import { parentPort } from 'worker_threads';
import TurndownService from 'turndown';
//@ts-ignore
import domino from 'domino';
//@ts-ignore
import * as turndownPluginGfm from 'joplin-turndown-plugin-gfm';
const turndownService = new TurndownService({
headingStyle: 'atx',
bulletListMarker: '-',
codeBlockStyle: 'fenced',
fence: '```',
emDelimiter: '_',
strongDelimiter: '**',
linkStyle: 'inlined',
linkReferenceStyle: 'full'
});
parentPort?.on('message', (params: { html: string }) => {
const html2md = (html: string): string => {
try {
const window = domino.createWindow(html);
const document = window.document;
turndownService.remove(['i', 'script', 'iframe']);
turndownService.addRule('codeBlock', {
filter: 'pre',
replacement(_, node) {
const content = node.textContent?.trim() || '';
// @ts-ignore
const codeName = node?._attrsByQName?.class?.data?.trim() || '';
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
}
});
turndownService.use(turndownPluginGfm.gfm);
// @ts-ignore
return turndownService.turndown(document);
} catch (error) {
return '';
}
};
const domino = require('domino-ext');
const turndownPluginGfm = require('joplin-turndown-plugin-gfm');
export const html2md = (html: string): string => {
const turndownService = new TurndownService({
headingStyle: 'atx',
bulletListMarker: '-',
codeBlockStyle: 'fenced',
fence: '```',
emDelimiter: '_',
strongDelimiter: '**',
linkStyle: 'inlined',
linkReferenceStyle: 'full'
});
try {
const md = html2md(params?.html || '');
parentPort?.postMessage({
type: 'success',
data: md
const window = domino.createWindow(html);
const document = window.document;
turndownService.remove(['i', 'script', 'iframe']);
turndownService.addRule('codeBlock', {
filter: 'pre',
replacement(_, node) {
const content = node.textContent?.trim() || '';
// @ts-ignore
const codeName = node?._attrsByQName?.class?.data?.trim() || '';
return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
}
});
turndownService.use(turndownPluginGfm.gfm);
return turndownService.turndown(document);
} catch (error) {
parentPort?.postMessage({
type: 'error',
data: error
});
console.log('html 2 markdown error', error);
return '';
}
global?.close?.();
});
};
packages/service/worker/utils.ts
......@@ -2,6 +2,7 @@ import { Worker } from 'worker_threads';
import path from 'path';
export enum WorkerNameEnum {
readFile = 'readFile',
htmlStr2Md = 'htmlStr2Md',
countGptMessagesTokens = 'countGptMessagesTokens'
}
......
packages/web/hooks/useScrollPagination.tsx
......@@ -37,7 +37,7 @@ export function useScrollPagination<
const [data, setData] = useState<TData['list']>([]);
const [isLoading, { setTrue, setFalse }] = useBoolean(false);
const [list] = useVirtualList(data, {
const [list] = useVirtualList<TData['list'][0]>(data, {
containerTarget: containerRef,
wrapperTarget: wrapperRef,
itemHeight,
......
projects/app/next.config.js
......@@ -51,11 +51,15 @@ const nextConfig = {
...entries,
'worker/htmlStr2Md': path.resolve(
process.cwd(),
'../../packages/service/worker/htmlStr2Md.ts'
'../../packages/service/worker/htmlStr2Md/index.ts'
),
'worker/countGptMessagesTokens': path.resolve(
process.cwd(),
'../../packages/service/worker/tiktoken/countGptMessagesTokens.ts'
),
'worker/readFile': path.resolve(
process.cwd(),
'../../packages/service/worker/file/read.ts'
)
};
}
......@@ -82,7 +86,12 @@ const nextConfig = {
serverComponentsExternalPackages: ['mongoose', 'pg'],
// 指定导出包优化，按需引入包模块
optimizePackageImports: ['mongoose', 'pg'],
outputFileTracingRoot: path.join(__dirname, '../../')
outputFileTracingRoot: path.join(__dirname, '../../'),
outputFileTracingIncludes: {
'/api/common/file/previewContent.ts': [
path.resolve(process.cwd(), '../../packages/service/worker/**/*')
]
}
}
};
......
projects/app/package.json
{
"name": "app",
"version": "4.7.1",
"version": "4.8",
"private": false,
"scripts": {
"dev": "next dev",
......
projects/app/src/components/core/workflow/Flow/nodes/NodeHttp/index.tsx
......@@ -38,6 +38,7 @@ import IOTitle from '../../components/IOTitle';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '../../../context';
import { getWorkflowGlobalVariables } from '@/web/core/workflow/utils';
import { useMemoizedFn } from 'ahooks';
const CurlImportModal = dynamic(() => import('./CurlImportModal'));
export const HttpHeaders = [
......@@ -108,159 +109,136 @@ const RenderHttpMethodAndUrl = React.memo(function RenderHttpMethodAndUrl({
const requestMethods = inputs.find((item) => item.key === NodeInputKeyEnum.httpMethod);
const requestUrl = inputs.find((item) => item.key === NodeInputKeyEnum.httpReqUrl);
const onChangeUrl = useCallback(
(e: React.ChangeEvent<HTMLInputElement>) => {
const onChangeUrl = (e: React.ChangeEvent<HTMLInputElement>) => {
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpReqUrl,
value: {
...requestUrl,
value: e.target.value
}
});
};
const onBlurUrl = (e: React.ChangeEvent<HTMLInputElement>) => {
const val = e.target.value;
// 拆分params和url
const url = val.split('?')[0];
const params = val.split('?')[1];
if (params) {
const paramsArr = params.split('&');
const paramsObj = paramsArr.reduce((acc, cur) => {
const [key, value] = cur.split('=');
return {
...acc,
[key]: value
};
}, {});
const inputParams = inputs.find((item) => item.key === NodeInputKeyEnum.httpParams);
if (!inputParams || Object.keys(paramsObj).length === 0) return;
const concatParams: PropsArrType[] = inputParams?.value || [];
Object.entries(paramsObj).forEach(([key, value]) => {
if (!concatParams.find((item) => item.key === key)) {
concatParams.push({ key, value: value as string, type: 'string' });
}
});
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpParams,
value: {
...inputParams,
value: concatParams
}
});
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpReqUrl,
value: {
...requestUrl,
value: e.target.value
value: url
}
});
},
[nodeId, onChangeNode, requestUrl]
);
const onBlurUrl = useCallback(
(e: React.ChangeEvent<HTMLInputElement>) => {
const val = e.target.value;
// 拆分params和url
const url = val.split('?')[0];
const params = val.split('?')[1];
if (params) {
const paramsArr = params.split('&');
const paramsObj = paramsArr.reduce((acc, cur) => {
const [key, value] = cur.split('=');
return {
...acc,
[key]: value
};
}, {});
const inputParams = inputs.find((item) => item.key === NodeInputKeyEnum.httpParams);
if (!inputParams || Object.keys(paramsObj).length === 0) return;
const concatParams: PropsArrType[] = inputParams?.value || [];
Object.entries(paramsObj).forEach(([key, value]) => {
if (!concatParams.find((item) => item.key === key)) {
concatParams.push({ key, value: value as string, type: 'string' });
}
});
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpParams,
value: {
...inputParams,
value: concatParams
}
});
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpReqUrl,
value: {
...requestUrl,
value: url
}
});
toast({
status: 'success',
title: t('core.module.http.Url and params have been split')
});
}
},
[inputs, nodeId, onChangeNode, requestUrl, t, toast]
);
toast({
status: 'success',
title: t('core.module.http.Url and params have been split')
});
}
};
const Render = useMemo(() => {
return (
<Box>
<Box mb={2} display={'flex'} justifyContent={'space-between'}>
<Box fontWeight={'medium'} color={'myGray.600'}>
{t('core.module.Http request settings')}
</Box>
<Button variant={'link'} onClick={onOpenCurl}>
{t('core.module.http.curl import')}
</Button>
return (
<Box>
<Box mb={2} display={'flex'} justifyContent={'space-between'}>
<Box fontWeight={'medium'} color={'myGray.600'}>
{t('core.module.Http request settings')}
</Box>
<Flex alignItems={'center'} className="nodrag">
<MySelect
h={'34px'}
w={'88px'}
bg={'white'}
width={'100%'}
value={requestMethods?.value}
list={[
{
label: 'GET',
value: 'GET'
},
{
label: 'POST',
value: 'POST'
},
{
label: 'PUT',
value: 'PUT'
},
{
label: 'DELETE',
value: 'DELETE'
},
{
label: 'PATCH',
value: 'PATCH'
}
]}
onchange={(e) => {
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpMethod,
value: {
...requestMethods,
value: e
}
});
}}
/>
<Input
flex={'1 0 0'}
ml={2}
h={'34px'}
bg={'white'}
value={requestUrl?.value}
placeholder={t('core.module.input.label.Http Request Url')}
fontSize={'xs'}
onChange={onChangeUrl}
onBlur={onBlurUrl}
/>
</Flex>
{isOpenCurl && <CurlImportModal nodeId={nodeId} inputs={inputs} onClose={onCloseCurl} />}
<Button variant={'link'} onClick={onOpenCurl}>
{t('core.module.http.curl import')}
</Button>
</Box>
);
}, [
inputs,
isOpenCurl,
nodeId,
onBlurUrl,
onChangeNode,
onChangeUrl,
onCloseCurl,
onOpenCurl,
requestMethods,
requestUrl?.value,
t
]);
<Flex alignItems={'center'} className="nodrag">
<MySelect
h={'34px'}
w={'88px'}
bg={'white'}
width={'100%'}
value={requestMethods?.value}
list={[
{
label: 'GET',
value: 'GET'
},
{
label: 'POST',
value: 'POST'
},
{
label: 'PUT',
value: 'PUT'
},
{
label: 'DELETE',
value: 'DELETE'
},
{
label: 'PATCH',
value: 'PATCH'
}
]}
onchange={(e) => {
onChangeNode({
nodeId,
type: 'updateInput',
key: NodeInputKeyEnum.httpMethod,
value: {
...requestMethods,
value: e
}
});
}}
/>
<Input
flex={'1 0 0'}
ml={2}
h={'34px'}
bg={'white'}
value={requestUrl?.value || ''}
placeholder={t('core.module.input.label.Http Request Url')}
fontSize={'xs'}
onChange={onChangeUrl}
onBlur={onBlurUrl}
/>
</Flex>
return Render;
{isOpenCurl && <CurlImportModal nodeId={nodeId} inputs={inputs} onClose={onCloseCurl} />}
</Box>
);
});
export function RenderHttpProps({
......@@ -644,15 +622,17 @@ const NodeHttp = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
const splitToolInputs = useContextSelector(WorkflowContext, (v) => v.splitToolInputs);
const { toolInputs, commonInputs, isTool } = splitToolInputs(inputs, nodeId);
const CustomComponents = useMemo(
() => ({
[NodeInputKeyEnum.httpMethod]: () => (
<RenderHttpMethodAndUrl nodeId={nodeId} inputs={inputs} />
),
[NodeInputKeyEnum.httpHeaders]: () => <RenderHttpProps nodeId={nodeId} inputs={inputs} />
}),
[inputs, nodeId]
);
const HttpMethodAndUrl = useMemoizedFn(() => (
<RenderHttpMethodAndUrl nodeId={nodeId} inputs={inputs} />
));
const Headers = useMemoizedFn(() => <RenderHttpProps nodeId={nodeId} inputs={inputs} />);
const CustomComponents = useMemo(() => {
return {
[NodeInputKeyEnum.httpMethod]: HttpMethodAndUrl,
[NodeInputKeyEnum.httpHeaders]: Headers
};
}, [Headers, HttpMethodAndUrl]);
return (
<NodeCard minW={'350px'} selected={selected} {...data}>
......
projects/app/src/components/core/workflow/Flow/nodes/NodeTools.tsx
......@@ -10,10 +10,11 @@ import { ToolSourceHandle } from './render/Handle/ToolHandle';
import { Box } from '@chakra-ui/react';
import IOTitle from '../components/IOTitle';
import MyIcon from '@fastgpt/web/components/common/Icon';
import RenderOutput from './render/RenderOutput';
const NodeTools = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
const { t } = useTranslation();
const { nodeId, inputs } = data;
const { nodeId, inputs, outputs } = data;
return (
<NodeCard minW={'350px'} selected={selected} {...data}>
......@@ -21,7 +22,10 @@ const NodeTools = ({ data, selected }: NodeProps<FlowNodeItemType>) => {
<IOTitle text={t('common.Input')} />
<RenderInput nodeId={nodeId} flowInputList={inputs} />
</Container>
<Container>
<IOTitle text={t('common.Output')} />
<RenderOutput nodeId={nodeId} flowOutputList={outputs} />
</Container>
<Box position={'relative'}>
<Box borderBottomLeftRadius={'md'} borderBottomRadius={'md'} overflow={'hidden'}>
<Divider
......
projects/app/src/components/core/workflow/Flow/nodes/render/RenderInput/Label.tsx
......@@ -58,7 +58,7 @@ const InputLabel = ({ nodeId, input }: Props) => {
);
const RenderLabel = useMemo(() => {
const renderType = renderTypeList[selectedTypeIndex || 0];
const renderType = renderTypeList?.[selectedTypeIndex || 0];
return (
<Flex className="nodrag" cursor={'default'} alignItems={'center'} position={'relative'}>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment